+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+ "context"
+ "fmt"
+ "strconv"
+ "strings"
+
+ "github.com/emicklei/go-restful"
+ "github.com/micro/go-micro/metadata"
+ c "github.com/opensds/multi-cloud/api/pkg/context"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ MaxPaginationLimit = 1000
+ DefaultPaginationLimit = MaxPaginationLimit
+ DefaultPaginationOffset = 0
+ MaxObjectSize = 5 * 1024 * 1024 * 1024 // 5GB
+ SortDirectionAsc = "asc"
+ SortDirectionDesc = "desc"
+)
+
+const (
+ KLimit = "limit"
+ KOffset = "offset"
+ KSort = "sort"
+ KLastModified = "lastmodified"
+ KObjKey = "objkey"
+ KStorageTier = "tier"
+ KPrefix = "prefix"
+ KMarker = "marker"
+ KDelimiter = "delimiter"
+ KVerMarker = "verMarker"
+)
+
+const (
+ CTX_KEY_TENANT_ID = "Tenantid"
+ CTX_KEY_USER_ID = "Userid"
+ CTX_KEY_IS_ADMIN = "Isadmin"
+ CTX_VAL_TRUE = "true"
+ CTX_KEY_OBJECT_KEY = "ObjectKey"
+ CTX_KEY_BUCKET_NAME = "BucketName"
+ CTX_KEY_SIZE = "ObjectSize"
+ CTX_KEY_LOCATION = "Location"
+)
+
+const (
+ REQUEST_PATH_BUCKET_NAME = "bucketName"
+ REQUEST_PATH_OBJECT_KEY = "objectKey"
+ REQUEST_HEADER_CONTENT_LENGTH = "Content-Length"
+ REQUEST_HEADER_STORAGE_CLASS = "x-amz-storage-class"
+ REQUEST_HEADER_COPY_SOURCE = "X-Amz-Copy-Source"
+ REQUEST_HEADER_COPY_SOURCE_RANGE = "X-Amz-Copy-Source-Range"
+ REQUEST_HEADER_ACL = "X-Amz-Acl"
+ REQUEST_HEADER_CONTENT_MD5 = "Content-Md5"
+ REQUEST_HEADER_CONTENT_TYPE = "Content-Type"
+)
+
+const (
+ REQUEST_FORM_KEY = "Key"
+ REQUEST_FORM_BUCKET = "Bucket"
+)
+
+func GetPaginationParam(request *restful.Request) (int32, int32, error) {
+ limit := int32(DefaultPaginationLimit)
+ offset := int32(DefaultPaginationOffset)
+
+ if request.QueryParameter(KLimit) != "" {
+ limitVal, err := strconv.Atoi(request.QueryParameter("limit"))
+ if err != nil {
+ log.Errorf("limit is invalid: %v", err)
+ return limit, offset, err
+ }
+ if limit > int32(limitVal) {
+ limit = int32(limitVal)
+ }
+ }
+
+ if request.QueryParameter(KOffset) != "" {
+ offsetVal, err := strconv.Atoi(request.QueryParameter("offset"))
+ if err != nil {
+ log.Errorf("offset is invalid: %v", err)
+ return limit, offset, err
+ }
+ offset = int32(offsetVal)
+ }
+ return limit, offset, nil
+}
+
+// An example of sort key parameter will be like this: sort=key1:asc,key2:desc
+func GetSortParam(request *restful.Request) (sortKeys []string, sortDirs []string, err error) {
+ sortStr := request.QueryParameter(KSort)
+ if sortStr != "" {
+ return
+ }
+
+ sortStr = strings.TrimSpace(sortStr)
+ for _, sort := range strings.Split(sortStr, ",") {
+ parts := strings.Split(sort, ":")
+ switch {
+ case len(parts) > 2:
+ err = fmt.Errorf("invalid sort value %s", sort)
+ return
+ case len(parts) == 1:
+ parts = append(parts, SortDirectionAsc)
+ }
+ sortKeys = append(sortKeys, parts[0])
+ sortDirs = append(sortDirs, parts[1])
+ }
+ return
+}
+
+func GetFilter(request *restful.Request, filterOpts []string) (map[string]string, error) {
+
+ filter := make(map[string]string)
+ for _, opt := range filterOpts {
+ v := request.QueryParameter(opt)
+ if v == "" {
+ continue
+ }
+ filter[opt] = v
+ }
+ return filter, nil
+}
+
+func InitCtxWithAuthInfo(request *restful.Request) context.Context {
+ actx := request.Attribute(c.KContext).(*c.Context)
+ ctx := metadata.NewContext(context.Background(), map[string]string{
+ CTX_KEY_USER_ID: actx.UserId,
+ CTX_KEY_TENANT_ID: actx.TenantId,
+ CTX_KEY_IS_ADMIN: strconv.FormatBool(actx.IsAdmin),
+ })
+
+ return ctx
+}
+
+func InitCtxWithVal(request *restful.Request, md map[string]string) context.Context {
+ actx := request.Attribute(c.KContext).(*c.Context)
+ md[CTX_KEY_USER_ID] = actx.UserId
+ md[CTX_KEY_TENANT_ID] = actx.TenantId
+ md[CTX_KEY_IS_ADMIN] = strconv.FormatBool(actx.IsAdmin)
+
+ return metadata.NewContext(context.Background(), md)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+// This is self defined context which is stored in context.Input.data.
+// It is used to transport data in the pipe line.
+
+package context
+
+import (
+ "encoding/json"
+ "reflect"
+
+ "github.com/emicklei/go-restful"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ KContext = "context"
+)
+
+const (
+ DefaultTenantId = "tenantId"
+ DefaultUserId = "userId"
+ NoAuthAdminTenantId = "adminTenantId"
+)
+
+func NewAdminContext() *Context {
+ return &Context{
+ TenantId: NoAuthAdminTenantId,
+ IsAdmin: true,
+ UserId: "unknown",
+ }
+}
+
+func NewContext() *Context {
+ return &Context{
+ TenantId: DefaultTenantId,
+ IsAdmin: false,
+ UserId: DefaultUserId,
+ }
+}
+
+func NewContextFromJson(s string) *Context {
+ ctx := &Context{}
+ err := json.Unmarshal([]byte(s), ctx)
+ if err != nil {
+ log.Errorf("Unmarshal json to context failed, reason: %v", err)
+ }
+ return ctx
+}
+
+func NewInternalTenantContext(tenantId, userId string, isAdmin bool) *Context {
+ return &Context{
+ TenantId: tenantId,
+ UserId: userId,
+ IsAdmin: isAdmin,
+ }
+}
+
+func GetContext(req *restful.Request) *Context {
+ ctx, _ := req.Attribute("context").(*Context)
+ if ctx == nil {
+ ctx = &Context{}
+ }
+ return ctx
+}
+
+type Context struct {
+ IsAdmin bool `policy:"true" json:"is_admin"`
+ AuthToken string `policy:"true" json:"auth_token"`
+ UserId string `policy:"true" json:"user_id"`
+ TenantId string `policy:"true" json:"tenant_id"`
+ DomainId string `policy:"true" json:"domain_id"`
+ UserDomainId string `policy:"true" json:"user_domain_id"`
+ ProjectDomainId string `policy:"true" json:"project_domain_id"`
+ Roles []string `policy:"true" json:"roles"`
+ UserName string `policy:"true" json:"user_name"`
+ ProjectName string `policy:"true" json:"project_name"`
+ DomainName string `policy:"true" json:"domain_name"`
+ UserDomainName string `policy:"true" json:"user_domain_name"`
+ ProjectDomainName string `policy:"true" json:"project_domain_name"`
+ IsAdminTenant bool `policy:"true" json:"is_admin_tenant"`
+}
+
+func (ctx *Context) ToPolicyValue() map[string]interface{} {
+ ctxMap := map[string]interface{}{}
+ t := reflect.TypeOf(ctx).Elem()
+ v := reflect.ValueOf(ctx).Elem()
+
+ for i := 0; i < t.NumField(); i++ {
+ field := v.Field(i)
+ name := t.Field(i).Tag.Get("json")
+ if t.Field(i).Tag.Get("policy") == "false" {
+ continue
+ }
+ if field.Kind() == reflect.String && field.String() == "" {
+ continue
+ }
+ if field.Kind() == reflect.Slice && field.Len() == 0 {
+ continue
+ }
+ if field.Kind() == reflect.Map && field.Len() == 0 {
+ continue
+ }
+ ctxMap[name] = field.Interface()
+ }
+ return ctxMap
+}
+
+func (ctx *Context) ToJson() string {
+ b, err := json.Marshal(ctx)
+ if err != nil {
+ log.Errorf("Context convert to json failed, reason: %v", err)
+ }
+ return string(b)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+This module implements the common data structure.
+
+*/
+
+package model
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+
+ "github.com/emicklei/go-restful"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ // ErrorBadRequest
+ ErrorBadRequest = 400
+ // ErrorUnauthorized
+ ErrorUnauthorized = 401
+ // ErrorForbidden
+ ErrorForbidden = 403
+ // ErrorNotFound
+ ErrorNotFound = 404
+ // ErrorInternalServer
+ ErrorInternalServer = 500
+ // ErrorNotImplemented
+ ErrorNotImplemented = 501
+)
+
+// ErrorSpec describes Detailed HTTP error response, which consists of a HTTP
+// status code, and a custom error message unique for each failure case.
+type ErrorSpec struct {
+ Code int `json:"code,omitempty"`
+ Message string `json:"message,omitempty"`
+}
+
+// ErrorBadRequestStatus
+func ErrorBadRequestStatus(message string) []byte {
+ return errorStatus(ErrorBadRequest, message)
+}
+
+// ErrorForbiddenStatus
+func ErrorForbiddenStatus(message string) []byte {
+ return errorStatus(ErrorForbidden, message)
+}
+
+// ErrorUnauthorizedStatus
+func ErrorUnauthorizedStatus(message string) []byte {
+ return errorStatus(ErrorUnauthorized, message)
+}
+
+// ErrorNotFoundStatus
+func ErrorNotFoundStatus(message string) []byte {
+ return errorStatus(ErrorNotFound, message)
+}
+
+// ErrorInternalServerStatus
+func ErrorInternalServerStatus(message string) []byte {
+ return errorStatus(ErrorInternalServer, message)
+}
+
+// ErrorNotImplementedStatus
+func ErrorNotImplementedStatus(message string) []byte {
+ return errorStatus(ErrorNotImplemented, message)
+}
+
+func errorStatus(code int, message string) []byte {
+ errStatus := &ErrorSpec{
+ Code: code,
+ Message: message,
+ }
+
+ // Mashal the error status.
+ body, err := json.Marshal(errStatus)
+ if err != nil {
+ return []byte("Failed to mashal error response: " + err.Error())
+ }
+ return body
+}
+
+/*
+func HttpError(ctx *context.Context, code int, format string, a ...interface{}) error {
+ ctx.Output.SetStatus(code)
+ msg := fmt.Sprintf(format, a...)
+ ctx.Output.Body(errorStatus(code, msg))
+ errInfo := fmt.Sprintf("Code:%d, Reason:%s", code, msg)
+ log.Error(errInfo)
+ return fmt.Errorf(errInfo)
+}
+*/
+
+func HttpError(res *restful.Response, code int, format string, a ...interface{}) error {
+ msg := fmt.Sprintf(format, a...)
+ res.WriteError(code, errors.New(msg))
+ errInfo := fmt.Sprintf("Code:%d, Reason:%s", code, msg)
+ log.Error(errInfo)
+ return fmt.Errorf(errInfo)
+}
+
+// Volume group error
+type NotImplementError struct {
+ S string
+}
+
+func (e *NotImplementError) Error() string {
+ return e.S
+}
+
+type NotFoundError struct {
+ S string
+}
+
+func NewNotFoundError(msg string) error {
+ return &NotFoundError{S: msg}
+}
+
+func (e *NotFoundError) Error() string {
+ return e.S
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package policy
+
+import (
+ "fmt"
+ "reflect"
+ "regexp"
+ "strings"
+
+ log "github.com/sirupsen/logrus"
+ "github.com/opensds/multi-cloud/api/pkg/utils"
+)
+
+func init() {
+ registerAll()
+}
+
+type NewCheckFunc func(kind string, match string) BaseCheck
+
+var registeredChecks map[string]NewCheckFunc
+
+func register(name string, f NewCheckFunc) {
+ registeredChecks[name] = f
+}
+
+func registerAll() {
+ if registeredChecks == nil {
+ registeredChecks = make(map[string]NewCheckFunc)
+ }
+ register("rule", NewRuleCheck)
+ register("role", NewRoleCheck)
+ register("generic", NewGenericCheck)
+}
+
+type BaseCheck interface {
+ String() string
+ Exec(target map[string]string, cred map[string]interface{}, enforcer Enforcer, currentRule string) bool
+}
+
+func check(rule BaseCheck,
+ target map[string]string,
+ cred map[string]interface{},
+ enforcer Enforcer,
+ currentRule string) bool {
+ ret := rule.Exec(target, cred, enforcer, currentRule)
+ log.Infof("check rules:%s -- %v", rule, ret)
+ return ret
+}
+
+func NewFalseCheck() BaseCheck {
+ return &FalseCheck{}
+}
+
+type FalseCheck struct{}
+
+func (this *FalseCheck) String() string {
+ return "!"
+}
+
+func (this *FalseCheck) Exec(target map[string]string,
+ cred map[string]interface{},
+ enforcer Enforcer,
+ currentRule string) bool {
+ return false
+}
+
+func NewTrueCheck() BaseCheck {
+ return &TrueCheck{}
+}
+
+type TrueCheck struct {
+ rule string
+}
+
+func (this *TrueCheck) String() string {
+ return "@"
+}
+
+func (this *TrueCheck) Exec(target map[string]string,
+ cred map[string]interface{},
+ enforcer Enforcer,
+ currentRule string) bool {
+ return true
+}
+
+func NewNotCheck(check BaseCheck) *NotCheck {
+ return &NotCheck{check}
+}
+
+type NotCheck struct {
+ rule BaseCheck
+}
+
+func (this *NotCheck) String() string {
+ return fmt.Sprintf("not %s", this.rule)
+}
+
+func (this *NotCheck) Exec(target map[string]string,
+ cred map[string]interface{},
+ enforcer Enforcer,
+ currentRule string) bool {
+ return !check(this.rule, target, cred, enforcer, currentRule)
+}
+
+func NewAndCheck(check1 BaseCheck, check2 BaseCheck) *AndCheck {
+ ac := &AndCheck{}
+ ac.AddCheck(check1)
+ ac.AddCheck(check2)
+ return ac
+}
+
+type AndCheck struct {
+ rules []BaseCheck
+}
+
+func (this *AndCheck) String() string {
+ var r []string
+ for _, rule := range this.rules {
+ r = append(r, rule.String())
+ }
+ return fmt.Sprintf("(%s)", strings.Join(r, " and "))
+}
+
+func (this *AndCheck) Exec(target map[string]string,
+ cred map[string]interface{},
+ enforcer Enforcer,
+ currentRule string) bool {
+ for _, rule := range this.rules {
+ if !check(rule, target, cred, enforcer, currentRule) {
+ return false
+ }
+ }
+ return true
+}
+
+func (this *AndCheck) AddCheck(rule BaseCheck) *AndCheck {
+ this.rules = append(this.rules, rule)
+ return this
+}
+
+func NewOrCheck(check1 BaseCheck, check2 BaseCheck) *OrCheck {
+ oc := &OrCheck{}
+ oc.AddCheck(check1)
+ oc.AddCheck(check2)
+ return oc
+}
+
+type OrCheck struct {
+ rules []BaseCheck
+}
+
+func (this *OrCheck) String() string {
+ var r []string
+ for _, rule := range this.rules {
+ r = append(r, rule.String())
+ }
+ return fmt.Sprintf("(%s)", strings.Join(r, " or "))
+}
+
+func (this *OrCheck) Exec(target map[string]string,
+ cred map[string]interface{},
+ enforcer Enforcer,
+ currentRule string) bool {
+ for _, rule := range this.rules {
+ if check(rule, target, cred, enforcer, currentRule) {
+ return true
+ }
+ }
+ return false
+}
+
+func (this *OrCheck) AddCheck(rule BaseCheck) *OrCheck {
+ this.rules = append(this.rules, rule)
+ return this
+}
+
+func (this *OrCheck) PopCheck() (*OrCheck, BaseCheck) {
+ x := this.rules[len(this.rules)-1]
+ this.rules = this.rules[:len(this.rules)-1]
+ return this, x
+}
+
+func NewRuleCheck(kind string, match string) BaseCheck {
+ return &RuleCheck{kind, match}
+}
+
+type RuleCheck struct {
+ kind string
+ match string
+}
+
+func (this *RuleCheck) String() string {
+ return fmt.Sprintf("%s:%s", this.kind, this.match)
+}
+
+func (this *RuleCheck) Exec(target map[string]string,
+ cred map[string]interface{},
+ enforcer Enforcer,
+ currentRule string) bool {
+ if len(enforcer.Rules) == 0 {
+ return false
+ }
+ return check(enforcer.Rules[this.match], target, cred, enforcer, currentRule)
+}
+
+func keyWorkFormatter(target map[string]string, match string) (string, error) {
+ reg := regexp.MustCompile(`%([[:graph:]]+)s`)
+ if ms := reg.FindAllString(match, -1); len(ms) == 1 {
+ s := ms[0][2 : len(ms[0])-2]
+ for key, val := range target {
+ if s == key {
+ return val, nil
+ break
+ }
+ }
+ return "", fmt.Errorf("target key doesn`t match")
+ }
+ return match, nil
+}
+
+func NewRoleCheck(kind string, match string) BaseCheck {
+ return &RoleCheck{kind, match}
+}
+
+type RoleCheck struct {
+ kind string
+ match string
+}
+
+func (this *RoleCheck) String() string {
+ return fmt.Sprintf("%s:%s", this.kind, this.match)
+}
+
+func (this *RoleCheck) Exec(target map[string]string,
+ cred map[string]interface{},
+ enforcer Enforcer,
+ currentRule string) bool {
+ match, err := keyWorkFormatter(target, this.match)
+ if err != nil {
+ return false
+ }
+ if roles, ok := cred["roles"]; ok {
+ for _, role := range roles.([]string) {
+ if strings.ToLower(match) == strings.ToLower(role) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func NewGenericCheck(kind string, match string) BaseCheck {
+ return &GenericCheck{kind, match}
+}
+
+type GenericCheck struct {
+ kind string
+ match string
+}
+
+func (this *GenericCheck) String() string {
+ return fmt.Sprintf("%s:%s", this.kind, this.match)
+}
+
+func (this *GenericCheck) simpleLiteral(expr string) (string, error) {
+ s := fmt.Sprintf("%c%c", expr[0], expr[len(expr)-1])
+ if len(expr) >= 2 && (s == "\"\"" || s == "''") {
+ return expr[1 : len(expr)-1], nil
+ }
+ if utils.Contained(strings.ToLower(expr), []string{"true", "false"}) {
+ return strings.ToLower(expr), nil
+ }
+ return "", fmt.Errorf("Not support right now")
+}
+
+func (this *GenericCheck) findInMap(testVal interface{}, pathSegs []string, match string) bool {
+ if len(pathSegs) == 0 {
+ switch testVal.(type) {
+ case string:
+ return strings.ToLower(match) == strings.ToLower(testVal.(string))
+ case bool:
+ return strings.ToLower(match) == fmt.Sprint(testVal.(bool))
+ default:
+ return false
+ }
+ }
+ key, pathSegs := pathSegs[0], pathSegs[1:]
+ if val, ok := testVal.(map[string]interface{}); ok {
+ testVal = val[key]
+ } else {
+ return false
+ }
+ if testVal == nil {
+ return false
+ }
+
+ if reflect.TypeOf(testVal).Kind() == reflect.Slice {
+ if vList, ok := testVal.([]interface{}); ok {
+ for _, val := range vList {
+ if this.findInMap(val, pathSegs, match) {
+ return true
+ }
+ }
+ } else {
+ for _, val := range testVal.([]string) {
+ if this.findInMap(val, pathSegs, match) {
+ return true
+ }
+ }
+ }
+ return false
+ } else {
+ return this.findInMap(testVal, pathSegs, match)
+ }
+}
+
+func (this *GenericCheck) Exec(target map[string]string,
+ cred map[string]interface{},
+ enforcer Enforcer,
+ currentRule string) bool {
+ match, err := keyWorkFormatter(target, strings.ToLower(this.match))
+ if err != nil {
+ return false
+ }
+
+ if test_value, err := this.simpleLiteral(this.kind); err == nil {
+ return strings.ToLower(match) == test_value
+ }
+ if len(cred) == 0 {
+ return false
+ }
+ return this.findInMap(cred, strings.Split(this.kind, "."), match)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package policy
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/opensds/multi-cloud/api/pkg/utils"
+)
+
+type TokenPair struct {
+ token string
+ value interface{}
+}
+
+func parseCheck(rule string) BaseCheck {
+ if rule == "!" {
+ return &FalseCheck{}
+ } else if rule == "@" {
+ return &TrueCheck{}
+ }
+ items := strings.SplitN(rule, ":", 2)
+ if len(items) != 2 {
+ return &FalseCheck{}
+ }
+ kind, match := items[0], items[1]
+ if check, ok := registeredChecks[kind]; ok {
+ return check(kind, match)
+ } else if check, ok := registeredChecks["generic"]; ok {
+ return check(kind, match)
+ } else {
+ return &FalseCheck{}
+ }
+}
+
+func parseTokenize(rule string) []TokenPair {
+ var tokPairs []TokenPair
+ for _, tok := range strings.Fields(rule) {
+ if tok == "" {
+ continue
+ }
+
+ clean := strings.TrimLeft(tok, "(")
+ for i := 0; i < len(tok)-len(clean); i++ {
+ tokPairs = append(tokPairs, TokenPair{"(", "("})
+ }
+
+ // If it was only parentheses, continue
+ if clean == "" {
+ continue
+ }
+
+ tok = clean
+ // Handle trailing parens on the token
+ clean = strings.TrimRight(tok, ")")
+ trail := len(tok) - len(clean)
+ lowered := strings.ToLower(clean)
+
+ if utils.Contained(lowered, []string{"and", "or", "not"}) {
+ tokPairs = append(tokPairs, TokenPair{lowered, clean})
+ } else if clean != "" {
+ s := fmt.Sprintf("%c%c", tok[0], tok[len(tok)-1])
+ if len(tok) >= 2 && (s == "\"\"" || s == "''") {
+ tokPairs = append(tokPairs, TokenPair{"string", tok[1 : len(tok)-1]})
+ } else {
+ tokPairs = append(tokPairs, TokenPair{"check", parseCheck(clean)})
+ }
+ }
+
+ for i := 0; i < trail; i++ {
+ tokPairs = append(tokPairs, TokenPair{")", ")"})
+ }
+ }
+
+ return tokPairs
+}
+
+func parseRule(rule string) BaseCheck {
+ if rule == "" {
+ return &TrueCheck{}
+ }
+ state := NewParseState()
+ tokPairs := parseTokenize(rule)
+ for _, tp := range tokPairs {
+ state.Shift(tp.token, tp.value)
+ }
+ if result, err := state.Result(); err == nil {
+ return result.(BaseCheck)
+ }
+ return &FalseCheck{}
+}
+
+var ReduceFuncMap = map[string]ReduceFunc{
+ "(,check,)": wrapCheck,
+ "(,and_expr,)": wrapCheck,
+ "(,or_expr,)": wrapCheck,
+ "check,and,check": makeAndExpr,
+ "or_expr,and,check": mixOrAndExpr,
+ "and_expr,and,check": extendAndExpr,
+ "check,or,check": makeOrExpr,
+ "and_expr,or,check": makeOrExpr,
+ "or_expr,or,check": extendOrExpr,
+ "not,check": makeNotExpr,
+}
+
+func NewParseState() *ParseState {
+ return &ParseState{}
+}
+
+type ParseState struct {
+ tokens []string
+ values []interface{}
+}
+
+type ReduceFunc func(args ...interface{}) []TokenPair
+
+func (p *ParseState) reduce() {
+ tokenStr := strings.Join(p.tokens, ",")
+ for key, fun := range ReduceFuncMap {
+ if strings.HasSuffix(tokenStr, key) {
+ argNum := strings.Count(key, ",") + 1
+ argIdx := len(p.values) - argNum
+ args := p.values[argIdx:]
+ results := fun(args...)
+ p.tokens = append(p.tokens[:argIdx], results[0].token)
+ p.values = append(p.values[:argIdx], results[0].value)
+ p.reduce()
+ }
+ }
+}
+
+func (p *ParseState) Shift(tok string, val interface{}) {
+ p.tokens = append(p.tokens, tok)
+ p.values = append(p.values, val)
+ p.reduce()
+}
+
+func (p *ParseState) Result() (interface{}, error) {
+ if len(p.values) != 1 {
+ return nil, fmt.Errorf("Could not parse rule")
+ }
+ return p.values[0], nil
+}
+
+func wrapCheck(args ...interface{}) []TokenPair {
+ check := args[1].(BaseCheck)
+ return []TokenPair{{"check", check}}
+}
+
+func makeAndExpr(args ...interface{}) []TokenPair {
+ check1 := args[0].(BaseCheck)
+ check2 := args[2].(BaseCheck)
+ return []TokenPair{{"and_expr", NewAndCheck(check1, check2)}}
+}
+
+func mixOrAndExpr(args ...interface{}) []TokenPair {
+ orExpr := args[0].(*OrCheck)
+ check := args[2].(BaseCheck)
+ var andExpr *AndCheck
+ orExpr, check1 := orExpr.PopCheck()
+ if v, ok := check1.(*AndCheck); ok {
+ andExpr = v
+ andExpr.AddCheck(check)
+ } else {
+ andExpr = NewAndCheck(check1, check)
+ }
+ return []TokenPair{{"or_expr", orExpr.AddCheck(andExpr)}}
+}
+
+func extendAndExpr(args ...interface{}) []TokenPair {
+ andExpr := args[0].(*AndCheck)
+ check2 := args[2].(BaseCheck)
+ return []TokenPair{{"and_expr", andExpr.AddCheck(check2)}}
+}
+
+func makeOrExpr(args ...interface{}) []TokenPair {
+ check1 := args[0].(BaseCheck)
+ check2 := args[2].(BaseCheck)
+ return []TokenPair{{"or_expr", NewOrCheck(check1, check2)}}
+}
+
+func extendOrExpr(args ...interface{}) []TokenPair {
+ orExpr := args[0].(*OrCheck)
+ check := args[2].(BaseCheck)
+ return []TokenPair{{"or_expr", orExpr.AddCheck(check)}}
+}
+
+func makeNotExpr(args ...interface{}) []TokenPair {
+ return []TokenPair{{"check", NewNotCheck(args[1].(BaseCheck))}}
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package policy
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "strings"
+
+ "github.com/emicklei/go-restful"
+ log "github.com/sirupsen/logrus"
+ "github.com/opensds/multi-cloud/api/pkg/context"
+ "github.com/opensds/multi-cloud/api/pkg/model"
+ "github.com/opensds/multi-cloud/api/pkg/utils"
+ "github.com/opensds/multi-cloud/api/pkg/utils/constants"
+)
+
+var enforcer *Enforcer
+
+func init() {
+ enforcer = NewEnforcer(false)
+ RegisterRules(enforcer)
+ enforcer.LoadRules(false)
+}
+
+type DefaultRule struct {
+ Name string
+ CheckStr string
+}
+
+func listRules() []DefaultRule {
+ return []DefaultRule{
+ {Name: "context_is_admin", CheckStr: "role:admin"},
+ }
+}
+
+func RegisterRules(e *Enforcer) {
+ e.RegisterDefaults(listRules())
+}
+
+func NewEnforcer(overWrite bool) *Enforcer {
+ return &Enforcer{OverWrite: overWrite}
+}
+
+type Enforcer struct {
+ Rules map[string]BaseCheck
+ DefaultRules []DefaultRule
+ OverWrite bool
+}
+
+func (e *Enforcer) RegisterDefaults(rules []DefaultRule) {
+ e.DefaultRules = rules
+}
+
+func (e *Enforcer) Enforce(rule string, target map[string]string, cred map[string]interface{}) (bool, error) {
+ if err := e.LoadRules(false); err != nil {
+ return false, err
+ }
+
+ toRule, ok := e.Rules[rule]
+ if !ok {
+ err := fmt.Errorf("rule [%s] does not exist", rule)
+ return false, err
+ }
+ return check(toRule, target, cred, *e, ""), nil
+}
+
+func (e *Enforcer) Authorize(rule string, target map[string]string, cred map[string]interface{}) (bool, error) {
+ return e.Enforce(rule, target, cred)
+}
+
+func (e *Enforcer) LoadRules(forcedReload bool) error {
+ path := os.Getenv("POLICY_PATH")
+ if path == "" {
+ path = constants.DefaultPolicyPath
+ }
+
+ fileInfo, err := os.Stat(path)
+ if err != nil {
+ return err
+ }
+ // Load all policy files that in the specified path
+ if fileInfo.IsDir() {
+ files, err := ioutil.ReadDir(path)
+ if err != nil {
+ return err
+ }
+ for _, f := range files {
+ if !f.IsDir() && strings.HasSuffix(f.Name(), ".json") {
+ err := e.LoadPolicyFile(path, forcedReload, false)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+ } else {
+ return e.LoadPolicyFile(path, forcedReload, e.OverWrite)
+ }
+
+}
+
+func (e *Enforcer) UpdateRules(rules map[string]BaseCheck) {
+ if e.Rules == nil {
+ e.Rules = make(map[string]BaseCheck)
+ }
+ for k, c := range rules {
+ e.Rules[k] = c
+ }
+}
+
+func (e *Enforcer) LoadPolicyFile(path string, forcedReload bool, overWrite bool) error {
+ // if rules is already set or user doesn't want to force reload, return it.
+ if e.Rules != nil && !forcedReload {
+ return nil
+ }
+
+ data, err := ioutil.ReadFile(path)
+ if err != nil {
+ msg := fmt.Sprintf("read policy file (%s) failed, reason:(%v)", path, err)
+ log.Infof(msg)
+ return fmt.Errorf(msg)
+ }
+ r, err := NewRules(data, e.DefaultRules)
+ if err != nil {
+ return err
+ }
+ if overWrite {
+ e.Rules = r.Rules
+ } else {
+ e.UpdateRules(r.Rules)
+ }
+ return nil
+}
+
+func NewRules(data []byte, defaultRule []DefaultRule) (*Rules, error) {
+ r := &Rules{}
+ err := r.Load(data, defaultRule)
+ return r, err
+}
+
+type Rules struct {
+ Rules map[string]BaseCheck
+}
+
+func (r *Rules) Load(data []byte, defaultRules []DefaultRule) error {
+ rulesMap := map[string]string{}
+ err := json.Unmarshal(data, &rulesMap)
+ if err != nil {
+ log.Errorf(err.Error())
+ return err
+ }
+ // add default value
+ for _, r := range defaultRules {
+ if v, ok := rulesMap[r.Name]; ok {
+ log.Errorf("policy rule (%s:%s) has conflict with default rule(%s:%s),abandon default value\n",
+ r.Name, v, r.Name, r.CheckStr)
+ } else {
+ rulesMap[r.Name] = r.CheckStr
+ }
+ }
+
+ if r.Rules == nil {
+ r.Rules = make(map[string]BaseCheck)
+ }
+ for k, v := range rulesMap {
+ r.Rules[k] = parseRule(v)
+ }
+ return nil
+}
+
+func (r *Rules) String() string {
+ b, _ := json.MarshalIndent(r.Rules, "", " ")
+ return string(b)
+}
+
+func Authorize(req *restful.Request, res *restful.Response, action string) bool {
+ if os.Getenv("OS_AUTH_AUTHSTRATEGY") != "keystone" {
+ return true
+ }
+ ctx := context.GetContext(req)
+ credentials := ctx.ToPolicyValue()
+ //TenantId := httpCtx.Input.Param(":tenantId")
+ TenantId := req.PathParameter("tenantId")
+ target := map[string]string{
+ "tenant_id": TenantId,
+ }
+ log.Infof("Action: %v", action)
+ log.Infof("Target: %v", target)
+ log.Infof("policy-Credentials: %v", credentials)
+ ok, err := enforcer.Authorize(action, target, credentials)
+ if err != nil {
+ log.Errorf("authorize failed, %s", err)
+ }
+ if !ok {
+ model.HttpError(res, http.StatusForbidden, "Operation is not permitted")
+ } else {
+ ctx.IsAdmin = utils.Contained("admin", ctx.Roles)
+ }
+ return ok
+}
+
+
+
package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) AbortMultipartUpload(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ objectKey := request.PathParameter("objectKey")
+ uploadId := request.QueryParameter("uploadId")
+
+ multipartUpload := pb.MultipartUpload{}
+ multipartUpload.Key = objectKey
+ multipartUpload.Bucket = bucketName
+ multipartUpload.UploadId = uploadId
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ result, err := s.s3Client.AbortMultipartUpload(ctx, &pb.AbortMultipartRequest{BucketName:bucketName,ObjectKey:objectKey,UploadId:uploadId})
+ if HandleS3Error(response, request, err, result.GetErrorCode()) != nil {
+ log.Errorf("unable to abort multipart. err:%v, errCode:%v", err, result.ErrorCode)
+ return
+ }
+
+ WriteSuccessNoContent(response)
+ log.Infof("Abort multipart upload[bucketName=%s, objectKey=%s, uploadId=%s] successfully.\n",
+ bucketName, objectKey, uploadId)
+}
+
+
+
package s3
+
+import (
+ "net/http"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+)
+
+func getAclFromHeader(request *restful.Request) (acl Acl, err error) {
+ acl.CannedAcl = request.HeaderParameter(common.REQUEST_HEADER_ACL)
+ if acl.CannedAcl == "" {
+ acl.CannedAcl = "private"
+ }
+ err = IsValidCannedAcl(acl)
+ return
+}
+
+func getAclFromFormValues(formValues map[string]string) (acl Acl, err error) {
+ headerfiedFormValues := make(http.Header)
+ for key := range formValues {
+ headerfiedFormValues.Add(key, formValues[key])
+ }
+ acl.CannedAcl = headerfiedFormValues.Get("acl")
+ if acl.CannedAcl == "" {
+ acl.CannedAcl = "private"
+ }
+ err = IsValidCannedAcl(acl)
+ return
+}
+
+
+
/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+ "bytes"
+ "encoding/xml"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/emicklei/go-restful"
+ . "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+// Refer: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
+var CommonS3ResponseHeaders = []string{"Content-Length", "Content-Type", "Connection", "Date", "ETag", "Server",
+ "x-amz-delete-marker", "x-amz-id-2", "x-amz-request-id", "x-amz-version-id"}
+
+// Encodes the response headers into XML format.
+func EncodeResponse(response interface{}) []byte {
+ var bytesBuffer bytes.Buffer
+ bytesBuffer.WriteString(xml.Header)
+ e := xml.NewEncoder(&bytesBuffer)
+ e.Encode(response)
+ return bytesBuffer.Bytes()
+}
+
+// Write object header
+func SetObjectHeaders(response *restful.Response, object *pb.Object, contentRange *HttpRange) {
+ // set object-related metadata headers
+ w := response.ResponseWriter
+ lastModified := time.Unix(object.LastModified, 0).UTC().Format(http.TimeFormat)
+ response.ResponseWriter.Header().Set("Last-Modified", lastModified)
+
+ w.Header().Set("Content-Type", object.ContentType)
+ if object.Etag != "" {
+ w.Header()["ETag"] = []string{"\"" + object.Etag + "\""}
+ }
+
+ for key, val := range object.CustomAttributes {
+ w.Header().Set(key, val)
+ }
+ //default cache-control is no-store
+ if _, ok := object.CustomAttributes["Cache-Control"]; !ok {
+ w.Header().Set("Cache-Control", "no-store")
+ }
+
+ w.Header().Set("X-Amz-Object-Type", (&types.Object{Object: object}).ObjectTypeToString())
+ w.Header().Set("X-Amz-Storage-Class", object.StorageClass)
+ w.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10))
+
+ // for providing ranged content
+ if contentRange != nil && contentRange.OffsetBegin > -1 {
+ // Override content-length
+ w.Header().Set("Content-Length", strconv.FormatInt(contentRange.GetLength(), 10))
+ w.Header().Set("Content-Range", contentRange.String())
+ w.WriteHeader(http.StatusPartialContent)
+ }
+}
+
+
+
/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+ "encoding/xml"
+ "net/http"
+ "net/url"
+ "path"
+ "strconv"
+ "time"
+
+ "github.com/emicklei/go-restful"
+ . "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ timeFormatAMZ = "2006-01-02T15:04:05.000Z" // Reply date format
+)
+
+func GetFinalError(err error, errorCode int32) error {
+ if errorCode != int32(ErrNoErr) {
+ return S3ErrorCode(errorCode)
+ } else {
+ return err
+ }
+}
+
+// WriteSuccessResponse write success headers and response if any.
+func WriteSuccessResponse(response *restful.Response, data []byte) {
+ if data == nil {
+ response.WriteHeader(http.StatusOK)
+ return
+ }
+
+ response.AddHeader("Content-Length", strconv.Itoa(len(data)))
+ response.WriteHeader(http.StatusOK)
+ response.Write(data)
+ response.Flush()
+}
+
+// writeErrorResponse write error headers
+// w http.ResponseWriter, r *http.Request
+func WriteErrorResponse(response *restful.Response, request *restful.Request, err error) {
+ WriteErrorResponseHeaders(response, err)
+ WriteErrorResponseNoHeader(response, request, err, request.Request.URL.Path)
+}
+
+func WriteErrorResponseWithResource(response *restful.Response, request *restful.Request, err error, resource string) {
+ WriteErrorResponseHeaders(response, err)
+ WriteErrorResponseNoHeader(response, request, err, resource)
+}
+
+func WriteErrorResponseHeaders(response *restful.Response, err error) {
+ var status int
+ apiErrorCode, ok := err.(S3Error)
+ if ok {
+ status = apiErrorCode.HttpStatusCode()
+ } else {
+ status = http.StatusInternalServerError
+ }
+
+ response.WriteHeader(status)
+}
+
+func WriteErrorResponseNoHeader(response *restful.Response, request *restful.Request, err error, resource string) {
+ // HEAD should have no body, do not attempt to write to it
+ if request.Request.Method == "HEAD" {
+ return
+ }
+
+ // Generate error response.
+ errorResponse := ApiErrorResponse{}
+ apiErrorCode, ok := err.(S3Error)
+ if ok {
+ errorResponse.AwsErrorCode = apiErrorCode.AwsErrorCode()
+ errorResponse.Message = apiErrorCode.Description()
+ } else {
+ errorResponse.AwsErrorCode = "InternalError"
+ errorResponse.Message = "We encountered an internal error, please try again."
+ }
+ errorResponse.Resource = resource
+ errorResponse.HostId = helper.CONFIG.InstanceId
+
+ encodedErrorResponse := EncodeResponse(errorResponse)
+
+ response.Write(encodedErrorResponse)
+ response.ResponseWriter.(http.Flusher).Flush()
+}
+
+// getLocation get URL location.
+func GetLocation(r *http.Request) string {
+ return path.Clean(r.URL.Path) // Clean any trailing slashes.
+}
+
+// writeSuccessNoContent write success headers with http status 204
+func WriteSuccessNoContent(response *restful.Response) {
+ response.ResponseWriter.WriteHeader(http.StatusNoContent)
+}
+
+func WriteApiErrorResponse(response *restful.Response, request *restful.Request, status int, awsErrCode, message string) {
+ // write header
+ response.WriteHeader(status)
+
+ // HEAD should have no body, do not attempt to write to it
+ if request.Request.Method == "HEAD" {
+ return
+ }
+
+ errorResponse := ApiErrorResponse{}
+ errorResponse.AwsErrorCode = awsErrCode
+ errorResponse.Message = message
+ errorResponse.Resource = request.Request.URL.Path
+ errorResponse.HostId = helper.CONFIG.InstanceId
+
+ encodedErrorResponse := EncodeResponse(errorResponse)
+
+ response.Write(encodedErrorResponse)
+ response.ResponseWriter.(http.Flusher).Flush()
+}
+
+// GenerateCopyObjectResponse
+func GenerateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
+ return CopyObjectResponse{
+ ETag: "\"" + etag + "\"",
+ LastModified: lastModified.UTC().Format(timeFormatAMZ),
+ }
+}
+
+func GenerateCopyObjectPartResponse(etag string, lastModified int64) CopyObjectPartResponse {
+ return CopyObjectPartResponse{
+ LastModified: time.Unix(lastModified, 0).UTC().Format(timeFormatAMZ),
+ ETag: "\"" + etag + "\"",
+ }
+}
+
+// GenerateInitiateMultipartUploadResponse
+func GenerateInitiateMultipartUploadResponse(bucket, key, uploadID string) InitiateMultipartUploadResponse {
+ return InitiateMultipartUploadResponse{
+ Bucket: bucket,
+ Key: key,
+ UploadID: uploadID,
+ }
+}
+
+// GenerateCompleteMultipartUploadResponse
+func GenerateCompleteMultipartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
+ return CompleteMultipartUploadResponse{
+ Location: location,
+ Bucket: bucket,
+ Key: key,
+ ETag: etag,
+ }
+}
+
+// APIErrorResponse - error response format
+type ApiErrorResponse struct {
+ XMLName xml.Name `xml:"Error" json:"-"`
+ AwsErrorCode string `xml:"Code"`
+ Message string
+ Key string
+ BucketName string
+ Resource string
+ RequestId string
+ HostId string
+}
+
+// Parse bucket url queries for ?uploads
+func parseListUploadsQuery(query url.Values) (request ListUploadsRequest, err error) {
+ request.Delimiter = query.Get("delimiter")
+ request.EncodingType = query.Get("encoding-type")
+ if query.Get("max-uploads") == "" {
+ request.MaxUploads = MaxUploadsList
+ } else {
+ request.MaxUploads, err = strconv.Atoi(query.Get("max-uploads"))
+ if err != nil {
+ return
+ }
+ if request.MaxUploads > MaxUploadsList || request.MaxUploads < 1 {
+ err = ErrInvalidMaxUploads
+ return
+ }
+ }
+ request.KeyMarker = query.Get("key-marker")
+ request.Prefix = query.Get("prefix")
+ request.UploadIdMarker = query.Get("upload-id-marker")
+ return
+}
+
+// Parse object url queries
+func parseListObjectPartsQuery(query url.Values) (request ListPartsRequest, err error) {
+ request.EncodingType = query.Get("encoding-type")
+ request.UploadId = query.Get("uploadId")
+ if request.UploadId == "" {
+ err = ErrNoSuchUpload
+ return
+ }
+ if query.Get("max-parts") == "" {
+ request.MaxParts = MaxPartsList
+ } else {
+ request.MaxParts, err = strconv.Atoi(query.Get("max-parts"))
+ if err != nil {
+ log.Errorln("failed to convert from string to integer. err:", err)
+ return
+ }
+ if request.MaxParts > MaxPartsList || request.MaxParts < 1 {
+ err = ErrInvalidMaxParts
+ return
+ }
+ }
+ if query.Get("part-number-marker") != "" {
+ request.PartNumberMarker, err = strconv.Atoi(query.Get("part-number-marker"))
+ if err != nil {
+ err = ErrInvalidPartNumberMarker
+ return
+ }
+ if request.PartNumberMarker < 0 {
+ err = ErrInvalidPartNumberMarker
+ return
+ }
+ }
+ return
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "encoding/xml"
+ "io"
+ "io/ioutil"
+ "strings"
+
+ "github.com/emicklei/go-restful"
+ "github.com/journeymidnight/yig/helper"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ s3error "github.com/opensds/multi-cloud/s3/error"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) BucketAclPut(request *restful.Request, response *restful.Response) {
+ bucketName := strings.ToLower(request.PathParameter(common.REQUEST_PATH_BUCKET_NAME))
+ log.Infof("received request: PUT bucket[name=%s] acl\n", bucketName)
+
+ var err error
+ var acl datatype.Acl
+ var policy datatype.AccessControlPolicy
+ if _, ok := request.Request.Header[common.REQUEST_HEADER_ACL]; ok {
+ acl, err = getAclFromHeader(request)
+ if err != nil {
+ WriteErrorResponse(response, request, err)
+ return
+ }
+ } else {
+ // because we only support canned acl, the body of request must be not too big, and 1024 is enough
+ aclBuffer, err := ioutil.ReadAll(io.LimitReader(request.Request.Body, 1024))
+ if err != nil {
+ log.Errorf("unable to read acls body, err:", err)
+ WriteErrorResponse(response, request, s3error.ErrInvalidAcl)
+ return
+ }
+ err = xml.Unmarshal(aclBuffer, &policy)
+ if err != nil {
+ log.Errorf("unable to parse acls xml body, err:", err)
+ WriteErrorResponse(response, request, s3error.ErrInternalError)
+ return
+ }
+ }
+
+ if acl.CannedAcl == "" {
+ newCannedAcl, err := datatype.GetCannedAclFromPolicy(policy)
+ if err != nil {
+ log.Error("failed to get canned acl from policy. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+ acl = newCannedAcl
+ }
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ res, err := s.s3Client.PutBucketACL(ctx, &pb.PutBucketACLRequest{ACLConfig: &pb.BucketACL{BucketName: bucketName, CannedAcl: acl.CannedAcl}})
+ if err != nil || res.ErrorCode != int32(s3error.ErrNoErr) {
+ WriteErrorResponse(response, request, GetFinalError(err, res.ErrorCode))
+ return
+ }
+
+ log.Infoln("PUT bucket acl successfully.")
+ WriteSuccessResponse(response, nil)
+}
+
+func (s *APIService) BucketAclGet(request *restful.Request, response *restful.Response) {
+ bucketName := strings.ToLower(request.PathParameter(common.REQUEST_PATH_BUCKET_NAME))
+ log.Infof("received request: GET bucket[name=%s] acl\n", bucketName)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ bucket, err := s.getBucketMeta(ctx, bucketName)
+ if err != nil {
+ log.Error("failed to get bucket[%s] acl policy for bucket", bucketName)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ owner := datatype.Owner{ID: bucket.TenantId, DisplayName: bucket.TenantId}
+ bucketOwner := datatype.Owner{}
+ policy, err := datatype.CreatePolicyFromCanned(owner, bucketOwner, datatype.Acl{CannedAcl: bucket.Acl.CannedAcl})
+ if err != nil {
+ log.Error("failed to create policy. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ aclBuffer, err := xmlFormat(policy)
+ if err != nil {
+ helper.ErrorIf(err, "failed to marshal acl XML for bucket", bucketName)
+ WriteErrorResponse(response, request, s3error.ErrInternalError)
+ return
+ }
+
+ setXmlHeader(response, aclBuffer)
+ WriteSuccessResponse(response, aclBuffer)
+ log.Infoln("GET bucket acl successfully.")
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) BucketDelete(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ log.Infof("Received request for deleting bucket[name=%s].\n", bucketName)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ rsp, err := s.s3Client.DeleteBucket(ctx, &s3.Bucket{Name: bucketName})
+ if HandleS3Error(response, request, err, rsp.GetErrorCode()) != nil {
+ log.Errorf("delete bucket[%s] failed, err=%v, errCode=%d\n", bucketName, err, rsp.GetErrorCode())
+ return
+ }
+
+ log.Infof("delete bucket[%s] successfully\n", bucketName)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "net/url"
+ "strconv"
+ "time"
+ "unicode/utf8"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ "github.com/opensds/multi-cloud/api/pkg/utils/constants"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) BucketGet(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ log.Infof("Received request for bucket details: %s\n", bucketName)
+
+ var err error
+ req, err := parseListObjectsQuery(request.Request.URL.Query())
+ if err != nil {
+ WriteErrorResponse(response, request, err)
+ return
+ }
+ req.Bucket = bucketName
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ lsRsp, err := s.s3Client.ListObjects(ctx, &req)
+ if HandleS3Error(response, request, err, lsRsp.GetErrorCode()) != nil {
+ log.Errorf("get bucket[%s] failed, err=%v, errCode=%d\n", bucketName, err, lsRsp.GetErrorCode())
+ return
+ }
+
+ rsp := CreateListObjectsResponse(bucketName, &req, lsRsp)
+ log.Debugf("rsp:%+v\n", rsp)
+ // Write success response.
+ response.WriteEntity(rsp)
+
+ return
+}
+
+func parseListObjectsQuery(query url.Values) (request s3.ListObjectsRequest, err error) {
+ if query.Get("list-type") == constants.ListObjectsType2Str {
+ request.Version = constants.ListObjectsType2Int
+ request.ContinuationToken = query.Get("continuation-token")
+ request.StartAfter = query.Get("start-after")
+ if !utf8.ValidString(request.StartAfter) {
+ err = ErrNonUTF8Encode
+ return
+ }
+ request.FetchOwner = helper.Ternary(query.Get("fetch-owner") == "true",
+ true, false).(bool)
+ } else {
+ request.Version = constants.ListObjectsType1Int
+ request.Marker = query.Get("marker")
+ if !utf8.ValidString(request.Marker) {
+ err = ErrNonUTF8Encode
+ return
+ }
+ }
+ request.Delimiter = query.Get("delimiter")
+ if !utf8.ValidString(request.Delimiter) {
+ err = ErrNonUTF8Encode
+ return
+ }
+ request.EncodingType = query.Get("encoding-type")
+ if request.EncodingType != "" && request.EncodingType != "url" {
+ err = ErrInvalidEncodingType
+ return
+ }
+ if query.Get("max-keys") == "" {
+ request.MaxKeys = utils.MaxObjectList
+ } else {
+ var maxKey int
+ maxKey, err = strconv.Atoi(query.Get("max-keys"))
+ if err != nil {
+ log.Errorf("parsing max-keys error:%v\n", err)
+ return request, ErrInvalidMaxKeys
+ }
+ request.MaxKeys = int32(maxKey)
+ if request.MaxKeys > utils.MaxObjectList || request.MaxKeys < 1 {
+ err = ErrInvalidMaxKeys
+ return
+ }
+ }
+ request.Prefix = query.Get("prefix")
+ if !utf8.ValidString(request.Prefix) {
+ err = ErrNonUTF8Encode
+ return
+ }
+
+ request.KeyMarker = query.Get("key-marker")
+ if !utf8.ValidString(request.KeyMarker) {
+ err = ErrNonUTF8Encode
+ return
+ }
+ request.VersionIdMarker = query.Get("version-id-marker")
+ if !utf8.ValidString(request.VersionIdMarker) {
+ err = ErrNonUTF8Encode
+ return
+ }
+
+ log.Infof("request:%+v\n", request)
+ return
+}
+
+// this function refers to GenerateListObjectsResponse in api-response.go from Minio Cloud Storage.
+func CreateListObjectsResponse(bucketName string, request *s3.ListObjectsRequest,
+ listRsp *s3.ListObjectsResponse) (response datatype.ListObjectsResponse) {
+ for _, o := range listRsp.Objects {
+ obj := datatype.Object{
+ Key: o.ObjectKey,
+ LastModified: time.Unix(o.LastModified, 0).In(time.Local).Format(timeFormatAMZ),
+ ETag: o.Etag,
+ Size: o.Size,
+ StorageClass: o.StorageClass,
+ Location: o.Location,
+ Tier: o.Tier,
+ }
+ if request.EncodingType != "" { // only support "url" encoding for now
+ obj.Key = url.QueryEscape(obj.Key)
+ }
+ if request.FetchOwner {
+ obj.Owner.ID = o.TenantId //TODO: DisplayName
+ }
+ response.Contents = append(response.Contents, obj)
+ }
+
+ var prefixes []datatype.CommonPrefix
+ for _, prefix := range listRsp.Prefixes {
+ item := datatype.CommonPrefix{
+ Prefix: prefix,
+ }
+ prefixes = append(prefixes, item)
+ }
+ response.CommonPrefixes = prefixes
+
+ response.Delimiter = request.Delimiter
+ response.EncodingType = request.EncodingType
+ response.IsTruncated = listRsp.IsTruncated
+ response.MaxKeys = int(request.MaxKeys)
+ response.Prefix = request.Prefix
+ response.BucketName = bucketName
+
+ if request.Version == constants.ListObjectsType2Int {
+ response.KeyCount = len(response.Contents)
+ response.ContinuationToken = request.ContinuationToken
+ response.NextContinuationToken = listRsp.NextMarker
+ response.StartAfter = request.StartAfter
+ } else { // version 1
+ response.Marker = request.Marker
+ response.NextMarker = listRsp.NextMarker
+ }
+
+ if request.EncodingType != "" {
+ response.Delimiter = url.QueryEscape(response.Delimiter)
+ response.Prefix = url.QueryEscape(response.Prefix)
+ response.StartAfter = url.QueryEscape(response.StartAfter)
+ response.Marker = url.QueryEscape(response.Marker)
+ }
+
+ return
+}
+
+func (s *APIService) HeadBucket(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ log.Infof("Received request for head bucket: %s\n", bucketName)
+
+ var err error
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ _, err = s.getBucketMeta(ctx, bucketName)
+ if err != nil {
+ log.Errorf("get bucket[%s] failed, err=%v\n", bucketName, err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ log.Debugln("head bucket succeed")
+ WriteSuccessResponse(response, nil)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) BucketLifecycleDelete(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ log.Infof("received request for creating lifecycle of bucket: %s", bucketName)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ rsp, err := s.s3Client.DeleteBucketLifecycle(ctx, &s3.BaseRequest{Id: bucketName})
+ log.Infof("rsp:%s, err:%v\n", rsp, err)
+ if HandleS3Error(response, request, err, rsp.GetErrorCode()) != nil {
+ log.Errorf("delete bucket[%s] lifecycle failed, err=%v, errCode=%d\n", bucketName, err, rsp.GetErrorCode())
+ return
+ }
+
+ log.Info("delete bucket lifecycle end.")
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "fmt"
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/api/pkg/utils/constants"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+//Convert function from storage tier to storage class for XML format output
+func (s *APIService) tier2class(tier int32) (string, error) {
+ {
+ mutext.Lock()
+ defer mutext.Unlock()
+ if len(ClassAndTier) == 0 {
+ err := s.loadStorageClassDefinition()
+ if err != nil {
+ log.Errorf("load storage classes failed: %v.\n", err)
+ return "", err
+ }
+ }
+ }
+ className := ""
+ for k, v := range ClassAndTier {
+ if v == tier {
+ className = k
+ }
+ }
+ if className == "" {
+ log.Infof("invalid tier: %d\n", tier)
+ return "", fmt.Errorf(InvalidTier)
+ }
+ return className, nil
+}
+
+//Function for GET Bucket Lifecycle API
+func (s *APIService) BucketLifecycleGet(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ log.Infof("received request for bucket details in GET lifecycle: %s", bucketName)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ rsp, err := s.s3Client.GetBucketLifecycle(ctx, &s3.BaseRequest{Id: bucketName})
+ if HandleS3Error(response, request, err, rsp.GetErrorCode()) != nil {
+ log.Errorf("get bucket[%s] lifecycle failed, err=%v, errCode=%d\n", bucketName, err, rsp.GetErrorCode())
+ return
+ }
+
+ // convert back to xml struct
+ lifecycleConfXml := model.LifecycleConfiguration{}
+
+ // convert lifecycle rule to xml Rule
+ for _, lcRule := range rsp.Lc {
+ xmlRule := model.Rule{}
+
+ xmlRule.Status = lcRule.Status
+ xmlRule.ID = lcRule.Id
+ xmlRule.Filter = converts3FilterToRuleFilter(lcRule.Filter)
+ xmlRule.AbortIncompleteMultipartUpload = converts3UploadToRuleUpload(lcRule.AbortIncompleteMultipartUpload)
+ xmlRule.Transition = make([]model.Transition, 0)
+
+ //Arranging the transition and expiration actions in XML
+ for _, action := range lcRule.Actions {
+ log.Infof("action is : %v\n", action)
+
+ if action.Name == ActionNameTransition {
+ xmlTransition := model.Transition{}
+ xmlTransition.Days = action.Days
+ xmlTransition.Backend = action.Backend
+ className, err := s.tier2class(action.Tier)
+ if err == nil {
+ xmlTransition.StorageClass = className
+ }
+ xmlRule.Transition = append(xmlRule.Transition, xmlTransition)
+ }
+ if action.Name == ActionNameExpiration {
+ xmlExpiration := model.Expiration{}
+ xmlExpiration.Days = action.Days
+ xmlRule.Expiration = append(xmlRule.Expiration, xmlExpiration)
+ }
+ }
+ // append each xml rule to xml array
+ lifecycleConfXml.Rule = append(lifecycleConfXml.Rule, xmlRule)
+ }
+
+ // marshall the array back to xml format
+ err = response.WriteAsXml(lifecycleConfXml)
+ if err != nil {
+ log.Infof("write lifecycle of bucket[%s] as xml failed, lifecycle =%s, err=%v.\n", bucketName,
+ lifecycleConfXml, err)
+ WriteErrorResponse(response, request, ErrInternalError)
+ return
+ }
+
+ log.Info("GET lifecycle succeed.")
+}
+
+func converts3FilterToRuleFilter(filter *s3.LifecycleFilter) model.Filter {
+ retFilter := model.Filter{}
+ if filter != nil {
+ retFilter.Prefix = filter.Prefix
+ }
+ return retFilter
+}
+
+func converts3UploadToRuleUpload(upload *s3.AbortMultipartUpload) model.AbortIncompleteMultipartUpload {
+ retUpload := model.AbortIncompleteMultipartUpload{}
+ if upload != nil {
+ retUpload.DaysAfterInitiation = upload.DaysAfterInitiation
+ }
+ return retUpload
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "crypto/md5"
+ "encoding/xml"
+ "fmt"
+ "net/http"
+ "sync"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/api/pkg/utils/constants"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+ "golang.org/x/net/context"
+)
+
+// Map from storage calss to tier
+var ClassAndTier map[string]int32
+var mutext sync.Mutex
+
+func (s *APIService) loadStorageClassDefinition() error {
+ ctx := context.Background()
+ log.Info("Load storage classes.")
+ res, err := s.s3Client.GetStorageClasses(ctx, &s3.BaseRequest{})
+ if err != nil {
+ log.Errorf("get storage classes from s3 service failed: %v\n", err)
+ return err
+ }
+ ClassAndTier = make(map[string]int32)
+ for _, v := range res.Classes {
+ ClassAndTier[v.Name] = v.Tier
+ }
+ return nil
+}
+
+func (s *APIService) class2tier(name string) (int32, error) {
+ {
+ mutext.Lock()
+ defer mutext.Unlock()
+ if len(ClassAndTier) == 0 {
+ err := s.loadStorageClassDefinition()
+ if err != nil {
+ log.Errorf("load storage classes failed: %v.\n", err)
+ return 0, err
+ }
+ }
+ }
+ tier, ok := ClassAndTier[name]
+ if !ok {
+ log.Errorf("translate storage class name[%s] to tier failed: %s.\n", name)
+ return 0, fmt.Errorf("invalid storage class:%s", name)
+ }
+ log.Infof("class[%s] to tier[%d]\n", name, tier)
+ return tier, nil
+}
+
+func checkValidationOfActions(actions []*s3.Action) error {
+ var pre *s3.Action = nil
+ for _, action := range actions {
+ log.Infof("action: %+v\n", *action)
+ if pre == nil {
+ if action.Name == ActionNameExpiration && action.Days < ExpirationMinDays {
+ // If only an expiration action for a rule, the days for that action should be more than ExpirationMinDays
+ return fmt.Errorf(InvalidExpireDays, ExpirationMinDays)
+ }
+ if action.Name == ActionNameTransition && action.Days < TransitionMinDays {
+ // the days for transition to tiers except tier999 should not less than TransitionMinDays
+ minDays := int32(TransitionMinDays)
+ if action.Tier == utils.Tier999 {
+ // the days for transition to tier999 should not less than TransitionToArchiveMinDays
+ minDays = TransitionToArchiveMinDays
+ }
+ if action.Days < minDays {
+ return fmt.Errorf(InvalidTransistionDays, action.Tier, minDays)
+ }
+
+ }
+ } else {
+ if pre.Name == ActionNameExpiration {
+ // Only one expiration action for each rule is supported
+ return fmt.Errorf(MoreThanOneExpirationAction)
+ }
+
+ if action.Name == ActionNameExpiration && pre.Days+ExpirationMinDays > action.Days {
+ return fmt.Errorf(DaysInStorageClassBeforeExpiration)
+ }
+
+ if action.Name == ActionNameTransition && pre.Days+LifecycleTransitionDaysStep > action.Days {
+ return fmt.Errorf(DaysInStorageClassBeforeTransition)
+ }
+ }
+ pre = action
+ }
+ return nil
+}
+
+func (s *APIService) BucketLifecyclePut(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ log.Infof("received request for creating lifecycle of bucket: %s", bucketName)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ _, err := s.getBucketMeta(ctx, bucketName)
+ if err != nil {
+ log.Errorf("get bucket[%s] failed, err=%v\n", bucketName, err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ body := ReadBody(request)
+ log.Infof("MD5 sum for body is %x", md5.Sum(body))
+ if body == nil {
+ log.Info("no request body provided for creating lifecycle configuration")
+ WriteErrorResponse(response, request, S3ErrorCode(ErrInvalidLc))
+ return
+ }
+
+ createLifecycleConf := model.LifecycleConfiguration{}
+ err = xml.Unmarshal(body, &createLifecycleConf)
+ if err != nil {
+ log.Errorf("unmarshal error:%v\n", err)
+ WriteErrorResponse(response, request, S3ErrorCode(ErrInvalidLc))
+ return
+ }
+
+ dupIdCheck := make(map[string]interface{})
+ s3RulePtrArr := make([]*s3.LifecycleRule, 0)
+ ruleCount := 0
+ for _, rule := range createLifecycleConf.Rule {
+ if ruleCount > 1000 {
+ log.Error("too many rules\n")
+ WriteApiErrorResponse(response, request, http.StatusBadRequest, AWSErrCodeInvalidArgument,
+ fmt.Sprintf(TooMuchLCRuls, 1000))
+ return
+ }
+
+ s3Rule := s3.LifecycleRule{}
+
+ //check if the ruleID has any duplicate values
+ if _, ok := dupIdCheck[rule.ID]; ok {
+ log.Errorf("duplicate ruleID found for rule : %s\n", rule.ID)
+ WriteApiErrorResponse(response, request, http.StatusBadRequest, AWSErrCodeInvalidArgument,
+ fmt.Sprintf(DuplicateRuleIDError, rule.ID))
+ return
+ }
+ // Assigning the rule ID
+ dupIdCheck[rule.ID] = struct{}{}
+ s3Rule.Id = rule.ID
+
+ //Assigning the status value to s3 status
+ log.Infof("status in rule file is %v\n", rule.Status)
+ s3Rule.Status = rule.Status
+
+ //Assigning the filter, using convert function to convert xml struct to s3 struct
+ s3Rule.Filter = convertRuleFilterToS3Filter(rule.Filter)
+
+ // Create the type of transition array
+ s3ActionArr := make([]*s3.Action, 0)
+
+ for _, transition := range rule.Transition {
+
+ //Defining the Transition array and assigning the values tp populate fields
+ s3Transition := s3.Action{Name: ActionNameTransition}
+
+ //Assigning the value of days for transition to happen
+ s3Transition.Days = transition.Days
+
+ //Assigning the backend value to the s3 struct
+ s3Transition.Backend = transition.Backend
+
+ //Assigning the storage class of the object to s3 struct
+ tier, err := s.class2tier(transition.StorageClass)
+ if err != nil {
+ response.WriteError(http.StatusBadRequest, err)
+ return
+ }
+ s3Transition.Tier = tier
+
+ //Adding the transition value to the main rule
+ s3ActionArr = append(s3ActionArr, &s3Transition)
+ ruleCount++
+ }
+
+ //Loop for getting the values from xml struct
+ for _, expiration := range rule.Expiration {
+ s3Expiration := s3.Action{Name: ActionNameExpiration}
+ s3Expiration.Days = expiration.Days
+ s3ActionArr = append(s3ActionArr, &s3Expiration)
+ ruleCount++
+ }
+
+ //validate actions
+ err := checkValidationOfActions(s3ActionArr)
+ if err != nil {
+ log.Errorf("validation of actions failed: %v\n", err)
+ WriteApiErrorResponse(response, request, http.StatusBadRequest, AWSErrCodeInvalidArgument, err.Error())
+ return
+ }
+
+ //Assigning the Expiration action to s3 struct expiration
+ s3Rule.Actions = s3ActionArr
+
+ s3Rule.AbortIncompleteMultipartUpload = convertRuleUploadToS3Upload(rule.AbortIncompleteMultipartUpload)
+
+ // add to the s3 array
+ s3RulePtrArr = append(s3RulePtrArr, &s3Rule)
+ }
+
+ lcRsp, err := s.s3Client.PutBucketLifecycle(ctx, &s3.PutBucketLifecycleRequest{BucketName: bucketName, Lc: s3RulePtrArr})
+ if HandleS3Error(response, request, err, lcRsp.GetErrorCode()) != nil {
+ log.Errorf("put bucket[%s] lifecycle failed, err=%v, errCode=%d\n", bucketName, err, lcRsp.GetErrorCode())
+ return
+ }
+
+ log.Info("create bucket lifecycle successful.")
+ WriteSuccessResponse(response, nil)
+}
+
+func convertRuleFilterToS3Filter(filter model.Filter) *s3.LifecycleFilter {
+ retFilter := s3.LifecycleFilter{}
+ /*
+ check if prefix is not empty
+ */
+ if filter.Prefix != "" {
+ retFilter.Prefix = filter.Prefix
+ return &retFilter
+ } else {
+ return nil
+ }
+}
+
+func convertRuleUploadToS3Upload(upload model.AbortIncompleteMultipartUpload) *s3.AbortMultipartUpload {
+ retUpload := s3.AbortMultipartUpload{}
+ retUpload.DaysAfterInitiation = upload.DaysAfterInitiation
+ return &retUpload
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "encoding/xml"
+ "strings"
+ "time"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ c "github.com/opensds/multi-cloud/api/pkg/context"
+ "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ "github.com/opensds/multi-cloud/s3/proto"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) BucketPut(request *restful.Request, response *restful.Response) {
+ bucketName := strings.ToLower(request.PathParameter(common.REQUEST_PATH_BUCKET_NAME))
+ if !isValidBucketName(bucketName) {
+ WriteErrorResponse(response, request, s3error.ErrInvalidBucketName)
+ return
+ }
+ log.Infof("received request: PUT bucket[name=%s]\n", bucketName)
+
+ if len(request.HeaderParameter(common.REQUEST_HEADER_CONTENT_LENGTH)) == 0 {
+ log.Errorf("missing content length")
+ WriteErrorResponse(response, request, s3error.ErrMissingContentLength)
+ return
+ }
+
+ acl, err := getAclFromHeader(request)
+ if err != nil {
+ log.Errorln("failed to get canned acl. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ actx := request.Attribute(c.KContext).(*c.Context)
+ bucket := s3.Bucket{Name: bucketName}
+ bucket.TenantId = actx.TenantId
+ bucket.UserId = actx.UserId
+ bucket.Deleted = false
+ bucket.CreateTime = time.Now().Unix()
+ bucket.Versioning = &s3.BucketVersioning{}
+ bucket.Versioning.Status = utils.VersioningDisabled // it's the default
+ bucket.Acl = &pb.Acl{CannedAcl: acl.CannedAcl}
+ log.Infof("Bucket PUT: TenantId=%s, UserId=%s\n", bucket.TenantId, bucket.UserId)
+
+ body := ReadBody(request)
+ flag := false
+ if body != nil && len(body) != 0 {
+ log.Infof("request body is not empty")
+ createBucketConf := model.CreateBucketConfiguration{}
+ err := xml.Unmarshal(body, &createBucketConf)
+ if err != nil {
+ log.Infof("unmarshal failed, body:%v, err:%v\n", body, err)
+ WriteErrorResponse(response, request, s3error.ErrUnmarshalFailed)
+ return
+ }
+
+ backendName := createBucketConf.LocationConstraint
+ if backendName != "" {
+ log.Infof("backendName is %v\n", backendName)
+ bucket.DefaultLocation = backendName
+ flag = s.isBackendExist(ctx, backendName)
+ }
+ }
+ if flag == false {
+ log.Errorf("default backend is not provided or it is not exist.")
+ WriteErrorResponse(response, request, s3error.ErrGetBackendFailed)
+ return
+ }
+
+ rsp, err := s.s3Client.CreateBucket(ctx, &bucket)
+ if HandleS3Error(response, request, err, rsp.GetErrorCode()) != nil {
+ log.Errorf("delete bucket[%s] failed, err=%v, errCode=%d\n", bucketName, err, rsp.GetErrorCode())
+ return
+ }
+
+ log.Infof("create bucket[name=%s, defaultLocation=%s] successfully.\n", bucket.Name, bucket.DefaultLocation)
+ // Make sure to add Location information here only for bucket
+ response.Header().Set("Location", GetLocation(request.Request))
+ WriteSuccessResponse(response, nil)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "crypto/md5"
+ "encoding/xml"
+ "fmt"
+ "net/http"
+ "sync"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+// Map from storage calss to tier
+var ClassAndTier1 map[string]int32
+var mutext1 sync.Mutex
+
+func (s *APIService) BucketSSEPut(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ log.Infof("received request for PUT bucket SSE: %s", bucketName)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ bucket, err := s.s3Client.GetBucket(ctx, &s3.Bucket{Name: bucketName})
+ if err != nil {
+ log.Errorf("get bucket failed, err=%v\n", err)
+ response.WriteError(http.StatusInternalServerError, fmt.Errorf("bucket does not exist"))
+ }
+ if bucket.ErrorCode != 0 {
+ log.Errorf("get bucket failed, err=%v\n", bucket.ErrorCode)
+ response.WriteError(http.StatusInternalServerError, fmt.Errorf("Get bucket error code %v", bucket.ErrorCode))
+ }
+
+ body := ReadBody(request)
+ log.Infof("MD5 sum for request body is %x", md5.Sum(body))
+
+ if body != nil {
+ sseConf := model.SSEConfiguration{}
+ errSSE := xml.Unmarshal(body, &sseConf)
+ if errSSE != nil {
+ response.WriteError(http.StatusInternalServerError, err)
+ return
+ }
+
+ s3SSE := &s3.ServerSideEncryption{
+ SseType: "",
+ EncryptionKey: nil,
+ InitilizationVector: nil,
+ XXX_NoUnkeyedLiteral: struct{}{},
+ XXX_unrecognized: nil,
+ XXX_sizecache: 0,
+ }
+ if sseConf.SSE.Enabled == "true" {
+ s3SSE.SseType = "SSE"
+ }
+
+ bucket.BucketMeta.ServerSideEncryption = s3SSE
+
+ baseResponse, errSSE := s.s3Client.UpdateBucket(ctx, bucket.BucketMeta)
+ if baseResponse.ErrorCode != 0{
+ response.WriteError(http.StatusInternalServerError, fmt.Errorf("Update bucket SSE options failed, error code %v", baseResponse.ErrorCode))
+ }
+ if errSSE != nil{
+ response.WriteError(http.StatusInternalServerError, fmt.Errorf("Update bucket SSE options failed"))
+ }
+
+ } else {
+ log.Info("no request body provided for creating SSE configuration")
+ response.WriteError(http.StatusBadRequest, fmt.Errorf(NoRequestBodySSE))
+ return
+ }
+
+ // Create bucket with bucket name will check if the bucket exists or not, if it exists
+ // it will internally call UpdateBucket function
+ res, err := s.s3Client.UpdateBucket(ctx, bucket.BucketMeta)
+ if err != nil {
+ response.WriteError(http.StatusInternalServerError, err)
+ return
+ }
+ log.Info("create bucket SSE successful.")
+ response.WriteEntity(res)
+
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "crypto/md5"
+ "encoding/xml"
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ s3 "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) BucketVersioningPut(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ log.Infof("received request for creating versioning of bucket: %s", bucketName)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ bucket, err := s.getBucketMeta(ctx, bucketName)
+ if err != nil {
+ WriteErrorResponse(response, request, err)
+ log.Errorf("get bucket[%s] failed, err=%v\n", bucketName, err)
+ return
+ }
+
+ body := ReadBody(request)
+ if body == nil {
+ log.Info("no request body provided for creating versioning configuration")
+ WriteErrorResponse(response, request, S3ErrorCode(ErrInvalidVersioning))
+ return
+ }
+ log.Infof("MD5 sum for body is %x", md5.Sum(body))
+
+ versionConf := model.VersioningConfiguration{}
+ err = xml.Unmarshal(body, &versionConf)
+ if err != nil {
+ WriteErrorResponse(response, request, S3ErrorCode(ErrInvalidVersioning))
+ return
+ }
+
+ s3version := &s3.BucketVersioning{
+ Status: "",
+ XXX_NoUnkeyedLiteral: struct{}{},
+ XXX_unrecognized: nil,
+ XXX_sizecache: 0,
+ }
+
+ if versionConf.Status == utils.VersioningEnabled {
+ s3version.Status = utils.VersioningEnabled
+ } else {
+ s3version.Status = utils.VersioningDisabled
+ }
+
+ bucket.Versioning = s3version
+
+ _, err = s.s3Client.UpdateBucket(ctx, bucket)
+ if err != nil {
+ log.Errorf("versioning configuration failed, errCode=%d\n", bucketName, err)
+ return
+ }
+
+ log.Info("create bucket version configuration successful.")
+ WriteSuccessResponse(response, nil)
+}
+
+
+
package s3
+
+import (
+ "encoding/xml"
+ "errors"
+ "io/ioutil"
+ "sort"
+ "strings"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+
+ . "github.com/opensds/multi-cloud/s3/error"
+)
+
+func (s *APIService) CompleteMultipartUpload(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ objectKey := request.PathParameter("objectKey")
+ uploadId := request.QueryParameter("uploadId")
+
+ multipartUpload := pb.MultipartUpload{}
+ multipartUpload.Bucket = bucketName
+ multipartUpload.Key = objectKey
+ multipartUpload.UploadId = uploadId
+
+ completeMultipartBytes, err := ioutil.ReadAll(request.Request.Body)
+ if err != nil {
+ log.Errorln("unable to complete multipart upload when read request body. err:", err)
+ WriteErrorResponse(response, request, ErrInternalError)
+ return
+ }
+ complMultipartUpload := &model.CompleteMultipartUpload{}
+ if err = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); err != nil {
+ log.Errorf("unable to parse complete multipart upload XML. data: %s, err: %v", string(completeMultipartBytes), err)
+ WriteErrorResponse(response, request, ErrMalformedXML)
+ return
+ }
+
+ if len(complMultipartUpload.Parts) == 0 {
+ log.Errorf("unable to complete multipart upload. err: %v", errors.New("len(complMultipartUpload.Parts) == 0"))
+ WriteErrorResponse(response, request, ErrMalformedXML)
+ return
+ }
+ if !sort.IsSorted(model.CompletedParts(complMultipartUpload.Parts)) {
+ log.Errorf("unable to complete multipart upload. data: %+v, err: %v", complMultipartUpload.Parts, errors.New("part not sorted."))
+ WriteErrorResponse(response, request, ErrInvalidPartOrder)
+ return
+ }
+
+ // Complete parts.
+ var completeParts []*pb.CompletePart
+ for _, part := range complMultipartUpload.Parts {
+ part.ETag = strings.TrimPrefix(part.ETag, "\"")
+ part.ETag = strings.TrimSuffix(part.ETag, "\"")
+ completeParts = append(completeParts, &pb.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag})
+ }
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ result, err := s.s3Client.CompleteMultipartUpload(ctx, &pb.CompleteMultipartRequest{
+ BucketName: bucketName,
+ ObjectKey: objectKey,
+ UploadId: uploadId,
+ CompleteParts: completeParts,
+ })
+ if HandleS3Error(response, request, err, result.GetErrorCode()) != nil {
+ log.Errorf("unable to complete multipart. err:%v, errCode:%v", err, result.ErrorCode)
+ return
+ }
+
+ // Get object location.
+ location := GetLocation(request.Request)
+ // Generate complete multipart response.
+ data := GenerateCompleteMultipartUploadResponse(bucketName, objectKey, location, result.ETag)
+ encodedSuccessResponse, err := xmlFormat(data)
+ if err != nil {
+ log.Errorln("unable to parse CompleteMultipartUpload response, err:", err)
+ WriteErrorResponseNoHeader(response, request, ErrInternalError, request.Request.URL.Path)
+ return
+ }
+
+ setXmlHeader(response, encodedSuccessResponse)
+ // write success response.
+ WriteSuccessResponse(response, encodedSuccessResponse)
+ log.Infof("Complete multipart upload[bucketName=%s, objectKey=%s, uploadId=%s] successfully.\n",
+ bucketName, objectKey, uploadId)
+}
+
+
+
package datatype
+
+import (
+ "encoding/xml"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ log "github.com/sirupsen/logrus"
+)
+
+var ValidCannedAcl = []string{
+ "private",
+ "public-read",
+ "public-read-write",
+}
+
+const (
+ CANNEDACL_PRIVATE = 0
+ CANNEDACL_PUBLIC_READ = 1
+ CANNEDACL_PUBLIC_READ_WRITE = 2
+ CANNEDACL_AWS_EXEC_READ = 3
+ CANNEDACL_AUTHENTICATED_READ = 4
+ CANNEDACL_BUCKET_OWNER_READ = 5
+ CANNEDACL_BUCKET_OWNER_FULL_CONTROLL = 6
+)
+
+const (
+ XMLNSXSI = "http://www.w3.org/2001/XMLSchema-instance"
+ XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
+)
+
+const (
+ ACL_TYPE_CANON_USER = "CanonicalUser"
+ ACL_TYPE_GROUP = "Group"
+)
+
+const (
+ ACL_GROUP_TYPE_ALL_USERS = "http://acs.amazonaws.com/groups/global/AllUsers"
+ ACL_GROUP_TYPE_AUTHENTICATED_USERS = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
+)
+
+const (
+ ACL_PERM_READ = "READ"
+ ACL_PERM_WRITE = "WRITE"
+ ACL_PERM_READ_ACP = "READ_ACP"
+ ACL_PERM_WRITE_ACP = "WRITE_ACP"
+ ACL_PERM_FULL_CONTROL = "FULL_CONTROL"
+)
+
+type Acl struct {
+ CannedAcl string
+ // TODO fancy ACLs
+}
+
+type AccessControlPolicy struct {
+ XMLName xml.Name `xml:"AccessControlPolicy"`
+ Xmlns string `xml:"xmlns,attr,omitempty"`
+ ID string `xml:"Owner>ID"`
+ DisplayName string `xml:"Owner>DisplayName"`
+ AccessControlList []Grant `xml:"AccessControlList>Grant"`
+}
+
+type Grant struct {
+ XMLName xml.Name `xml:"Grant"`
+ Grantee Grantee `xml:"Grantee"`
+ Permission string `xml:"Permission"`
+}
+
+type Grantee struct {
+ XMLName xml.Name `xml:"Grantee"`
+ XmlnsXsi string `xml:"xmlns:xsi,attr"`
+ XsiType string `xml:"http://www.w3.org/2001/XMLSchema-instance type,attr"`
+ URI string `xml:"URI,omitempty"`
+ ID string `xml:"ID,omitempty"`
+ DisplayName string `xml:"DisplayName,omitempty"`
+ EmailAddress string `xml:"EmailAddress,omitempty"`
+}
+
+type AccessControlPolicyResponse struct {
+ XMLName xml.Name `xml:"AccessControlPolicy"`
+ Xmlns string `xml:"xmlns,attr,omitempty"`
+ ID string `xml:"Owner>ID"`
+ DisplayName string `xml:"Owner>DisplayName"`
+ AccessControlList []GrantResponse `xml:"AccessControlList>Grant"`
+}
+
+type GrantResponse struct {
+ XMLName xml.Name `xml:"Grant"`
+ Grantee GranteeResponse `xml:"Grantee"`
+ Permission string `xml:"Permission"`
+}
+
+type GranteeResponse struct {
+ XMLName xml.Name `xml:"Grantee"`
+ XmlnsXsi string `xml:"xmlns:xsi,attr"`
+ XsiType string `xml:"xsi:type,attr"`
+ URI string `xml:"URI,omitempty"`
+ ID string `xml:"ID,omitempty"`
+ DisplayName string `xml:"DisplayName,omitempty"`
+ EmailAddress string `xml:"EmailAddress,omitempty"`
+}
+
+func IsValidCannedAcl(acl Acl) (err error) {
+ if !helper.StringInSlice(acl.CannedAcl, ValidCannedAcl) {
+ err = ErrInvalidCannedAcl
+ return
+ }
+ return
+}
+
+// the function will be deleted, because we will use AccessControlPolicy instead canned acl stored in hbase
+func GetCannedAclFromPolicy(policy AccessControlPolicy) (acl Acl, err error) {
+ aclOwner := Owner{ID: policy.ID, DisplayName: policy.DisplayName}
+ var canonUser bool
+ var group bool
+ for _, grant := range policy.AccessControlList {
+ log.Infoln("GetCannedAclFromPolicy")
+ switch grant.Grantee.XsiType {
+ case ACL_TYPE_CANON_USER:
+ if grant.Grantee.ID != aclOwner.ID {
+ log.Infoln("grant.Grantee.ID:", grant.Grantee.ID, "not equals aclOwner.ID:", aclOwner.ID)
+ return acl, ErrUnsupportedAcl
+ }
+ if grant.Permission != ACL_PERM_FULL_CONTROL {
+ log.Infoln("grant.Permission:", grant.Permission, "not equals", ACL_PERM_FULL_CONTROL)
+ return acl, ErrUnsupportedAcl
+ }
+ canonUser = true
+ case ACL_TYPE_GROUP:
+ if grant.Grantee.URI == ACL_GROUP_TYPE_ALL_USERS {
+ log.Infoln("grant.Grantee.URI is", ACL_GROUP_TYPE_ALL_USERS)
+ if grant.Permission != ACL_PERM_READ {
+ log.Infoln("grant.Permission:", grant.Permission, "not equals", ACL_PERM_READ)
+ return acl, ErrUnsupportedAcl
+ }
+ acl = Acl{CannedAcl: ValidCannedAcl[CANNEDACL_PUBLIC_READ]}
+ group = true
+ } else if grant.Grantee.URI == ACL_GROUP_TYPE_AUTHENTICATED_USERS {
+ log.Infoln("grant.Grantee.URI is", ACL_GROUP_TYPE_AUTHENTICATED_USERS)
+ if grant.Permission != ACL_PERM_READ {
+ log.Infoln("grant.Permission:", grant.Permission, "not equals", ACL_PERM_FULL_CONTROL)
+ return acl, ErrUnsupportedAcl
+ }
+ acl = Acl{CannedAcl: ValidCannedAcl[CANNEDACL_AUTHENTICATED_READ]}
+ group = true
+ } else {
+ log.Infoln("grant.Grantee.URI is invalid:", grant.Grantee.URI)
+ return acl, ErrUnsupportedAcl
+ }
+ default:
+ log.Infoln("grant.Grantee.XsiType is invalid:", grant.Grantee.XsiType)
+ return acl, ErrUnsupportedAcl
+ }
+ }
+
+ if !canonUser {
+ log.Infoln("canonUser is invalid:", canonUser)
+ return acl, ErrUnsupportedAcl
+ }
+
+ if !group {
+ acl = Acl{CannedAcl: ValidCannedAcl[CANNEDACL_PRIVATE]}
+ }
+
+ return acl, nil
+}
+
+func createGrant(xsiType string, owner Owner, perm string, groupType string) (grant GrantResponse, err error) {
+
+ if xsiType == ACL_TYPE_CANON_USER {
+ grant.Grantee.ID = owner.ID
+ grant.Grantee.DisplayName = owner.DisplayName
+ } else if xsiType == ACL_TYPE_GROUP {
+ grant.Grantee.URI = groupType
+ } else {
+ return grant, ErrUnsupportedAcl
+ }
+ grant.Permission = perm
+ grant.Grantee.XmlnsXsi = XMLNSXSI
+ grant.Grantee.XsiType = xsiType
+ return
+}
+
+func CreatePolicyFromCanned(owner Owner, bucketOwner Owner, acl Acl) (
+ policy AccessControlPolicyResponse, err error) {
+
+ policy.ID = owner.ID
+ policy.DisplayName = owner.DisplayName
+ policy.Xmlns = XMLNS
+ grant, err := createGrant(ACL_TYPE_CANON_USER, owner, ACL_PERM_FULL_CONTROL, "")
+ if err != nil {
+ return policy, err
+ }
+ policy.AccessControlList = append(policy.AccessControlList, grant)
+ if acl.CannedAcl == "private" {
+ return policy, nil
+ }
+ switch acl.CannedAcl {
+ case "public-read":
+ owner := Owner{}
+ grant, err := createGrant(ACL_TYPE_GROUP, owner, ACL_PERM_READ, ACL_GROUP_TYPE_ALL_USERS)
+ if err != nil {
+ return policy, err
+ }
+ policy.AccessControlList = append(policy.AccessControlList, grant)
+ case "public-read-write":
+ owner := Owner{}
+ rGrant, err := createGrant(ACL_TYPE_GROUP, owner, ACL_PERM_READ, ACL_GROUP_TYPE_ALL_USERS)
+ if err != nil {
+ return policy, err
+ }
+ wGrant, err := createGrant(ACL_TYPE_GROUP, owner, ACL_PERM_WRITE, ACL_GROUP_TYPE_ALL_USERS)
+ if err != nil {
+ return policy, err
+ }
+ policy.AccessControlList = append(policy.AccessControlList, rGrant, wGrant)
+ case "bucket-owner-full-control":
+ grant, err := createGrant(ACL_TYPE_CANON_USER, bucketOwner, ACL_PERM_FULL_CONTROL, "")
+ if err != nil {
+ return policy, err
+ }
+ if bucketOwner.ID != owner.ID {
+ policy.AccessControlList = append(policy.AccessControlList, grant)
+ }
+ default:
+ return policy, ErrUnsupportedAcl
+ }
+ return
+}
+
+
+
package datatype
+
+import (
+ . "github.com/opensds/multi-cloud/s3/error"
+ "time"
+)
+
+const (
+ Iso8601Format = "20060102T150405Z"
+ YYYYMMDD = "20060102"
+ PresignedUrlExpireLimit = 7 * 24 * time.Hour
+)
+
+// Supported Amz date formats.
+var amzDateFormats = []string{
+ time.RFC1123,
+ time.RFC1123Z,
+ Iso8601Format,
+ // Add new AMZ date formats here.
+}
+
+// parseAmzDate - parses date string into supported amz date formats.
+func ParseAmzDate(amzDateStr string) (amzDate time.Time, apiErr error) {
+ for _, dateFormat := range amzDateFormats {
+ amzDate, e := time.Parse(dateFormat, amzDateStr)
+ if e == nil {
+ return amzDate, nil
+ }
+ }
+ return time.Time{}, ErrMalformedDate
+}
+
+
+
/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package datatype
+
+import (
+ "errors"
+ "fmt"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+const (
+ byteRangePrefix = "bytes="
+)
+
+// Valid byte position regexp
+var validBytePos = regexp.MustCompile(`^[0-9]+$`)
+
+// ErrorInvalidRange - returned when given range value is not valid.
+var ErrorInvalidRange = errors.New("Invalid range")
+
+// HttpRange specifies the byte range to be sent to the client.
+type HttpRange struct {
+ OffsetBegin int64
+ OffsetEnd int64
+ ResourceSize int64
+}
+
+// String populate range stringer interface
+func (hrange HttpRange) String() string {
+ return fmt.Sprintf("bytes %d-%d/%d", hrange.OffsetBegin, hrange.OffsetEnd, hrange.ResourceSize)
+}
+
+// getlength - get length from the range.
+func (hrange HttpRange) GetLength() int64 {
+ return 1 + hrange.OffsetEnd - hrange.OffsetBegin
+}
+
+func getOffset(offsetString string) (offset int64, err error) {
+ offset = int64(-1)
+ if len(offsetString) > 0 {
+ if !validBytePos.MatchString(offsetString) {
+ return offset, fmt.Errorf("'%s' does not have a valid first byte position value", offsetString)
+ }
+
+ if offset, err = strconv.ParseInt(offsetString, 10, 64); err != nil {
+ return offset, fmt.Errorf("'%s' does not have a valid first byte position value", offsetString)
+ }
+ }
+ return
+}
+
+func ParseRequestRange(rangeString string, resourceSize int64) (hrange *HttpRange, err error) {
+ // TODO handle multi-range request
+ // see https://tools.ietf.org/html/rfc7233
+
+ // Return error if given range string doesn't start with byte range prefix.
+ if !strings.HasPrefix(rangeString, byteRangePrefix) {
+ return nil, fmt.Errorf("'%s' does not start with '%s'", rangeString, byteRangePrefix)
+ }
+
+ // Trim byte range prefix.
+ byteRangeString := strings.TrimPrefix(rangeString, byteRangePrefix)
+
+ // Check if range string contains delimiter '-', else return error. eg. "bytes=8"
+ sepIndex := strings.Index(byteRangeString, "-")
+ if sepIndex == -1 {
+ return nil, fmt.Errorf("'%s' does not have a valid range value", rangeString)
+ }
+
+ offsetBeginString := byteRangeString[:sepIndex]
+ offsetBegin, err := getOffset(offsetBeginString)
+ if err != nil {
+ return nil, err
+ }
+ offsetEndString := byteRangeString[sepIndex+1:]
+ offsetEnd, err := getOffset(offsetEndString)
+ if err != nil {
+ return nil, err
+ }
+
+ // rangeString contains first and last byte positions. eg. "bytes=2-5"
+ if offsetBegin > -1 && offsetEnd > -1 {
+ if offsetBegin > offsetEnd {
+ // Last byte position is not greater than first byte position. eg. "bytes=5-2"
+ return nil, fmt.Errorf("'%s' does not have valid range value", rangeString)
+ }
+
+ // First and last byte positions should not be >= resourceSize.
+ if offsetBegin >= resourceSize {
+ return nil, ErrorInvalidRange
+ }
+
+ if offsetEnd >= resourceSize {
+ offsetEnd = resourceSize - 1
+ }
+ } else if offsetBegin > -1 {
+ // rangeString contains only first byte position. eg. "bytes=8-"
+ if offsetBegin >= resourceSize {
+ // First byte position should not be >= resourceSize.
+ return nil, ErrorInvalidRange
+ }
+
+ offsetEnd = resourceSize - 1
+ } else if offsetEnd > -1 {
+ // rangeString contains only last byte position. eg. "bytes=-3"
+ if offsetEnd == 0 {
+ // Last byte position should not be zero eg. "bytes=-0"
+ return nil, ErrorInvalidRange
+ }
+
+ if offsetEnd >= resourceSize {
+ offsetBegin = 0
+ } else {
+ offsetBegin = resourceSize - offsetEnd
+ }
+
+ offsetEnd = resourceSize - 1
+ } else {
+ // rangeString contains first and last byte positions missing. eg. "bytes=-"
+ return nil, fmt.Errorf("'%s' does not have valid range value", rangeString)
+ }
+
+ return &HttpRange{offsetBegin, offsetEnd, resourceSize}, nil
+}
+
+
+
/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+ "io"
+ "io/ioutil"
+ "mime/multipart"
+ "net/http"
+ "strings"
+
+ "github.com/emicklei/go-restful"
+ . "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+)
+
+// validates location constraint from the request body.
+// the location value in the request body should match the Region in serverConfig.
+// other values of location are not accepted.
+// make bucket fails in such cases.
+func isValidLocationConstraint(reqBody io.Reader) (err error) {
+ var region = helper.CONFIG.Region
+ var locationConstraint CreateBucketLocationConfiguration
+ e := xmlDecoder(reqBody, &locationConstraint)
+ if e != nil {
+ if e == io.EOF {
+ // Failed due to empty request body. The location will be set to
+ // default value from the serverConfig
+ err = nil
+ } else {
+ // Failed due to malformed configuration.
+ err = ErrMalformedXML
+ }
+ } else {
+ // Region obtained from the body.
+ // It should be equal to Region in serverConfig.
+ // Else ErrInvalidRegion returned.
+ // For empty value location will be to set to default value from the serverConfig.
+ if locationConstraint.Location != "" && region != locationConstraint.Location {
+ err = ErrInvalidRegion
+ }
+ }
+ return err
+}
+
+// Supported headers that needs to be extracted.
+var supportedHeaders = []string{
+ "cache-control",
+ "content-disposition",
+ "content-encoding",
+ "content-language",
+ "content-type",
+ "expires",
+ "website-redirect-location",
+ // Add more supported headers here
+}
+
+// extractMetadataFromHeader extracts metadata from HTTP header.
+func extractMetadataFromHeader(request *restful.Request) map[string]string {
+ metadata := make(map[string]string)
+ // Save standard supported headers.
+ for _, supportedHeader := range supportedHeaders {
+ if h := request.HeaderParameter(supportedHeader); h != "" {
+ metadata[supportedHeader] = h
+ }
+ }
+
+ // Go through all other headers for any additional headers that needs to be saved.
+ for key := range request.Request.Header {
+ if strings.HasPrefix(strings.ToLower(key), "x-amz-meta-") {
+ metadata[key] = request.HeaderParameter(key)
+ }
+ }
+ // Return.
+ return metadata
+}
+
+// Suffix matcher string matches suffix in a platform specific way.
+// For example on windows since its case insensitive we are supposed
+// to do case insensitive checks.
+func hasSuffix(s string, suffix string) bool {
+ return strings.HasSuffix(s, suffix)
+}
+
+func extractHTTPFormValues(reader *multipart.Reader) (filePartReader io.Reader,
+ formValues map[string]string, err error) {
+
+ formValues = make(map[string]string)
+ for {
+ var part *multipart.Part
+ part, err = reader.NextPart()
+ if err == io.EOF {
+ err = nil
+ break
+ }
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if part.FormName() != "file" {
+ var buffer []byte
+ buffer, err = ioutil.ReadAll(part)
+ if err != nil {
+ return nil, nil, err
+ }
+ formValues[http.CanonicalHeaderKey(part.FormName())] = string(buffer)
+ } else {
+ // "All variables within the form are expanded prior to validating
+ // the POST policy"
+ fileName := part.FileName()
+ objectKey, ok := formValues["Key"]
+ if !ok {
+ return nil, nil, ErrMissingFields
+ }
+ if strings.Contains(objectKey, "${filename}") {
+ formValues["Key"] = strings.Replace(objectKey, "${filename}", fileName, -1)
+ }
+
+ filePartReader = part
+ // "The file or content must be the last field in the form.
+ // Any fields below it are ignored."
+ break
+ }
+ }
+
+ if filePartReader == nil {
+ err = ErrEmptyEntity
+ }
+ return
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "encoding/xml"
+ "time"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ s3 "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func parseListBuckets(list *s3.ListBucketsResponse) []byte {
+ if list == nil || list.Buckets == nil {
+ return nil
+ }
+ temp := model.ListAllMyBucketsResult{}
+
+ log.Infof("Parse ListBuckets: %v", list.Buckets)
+ //default xmlns
+ temp.Xmlns = model.Xmlns
+ buckets := []model.Bucket{}
+ for _, value := range list.Buckets {
+ ctime := time.Unix(value.CreateTime, 0).Format(time.RFC3339)
+ versionOpts := model.VersioningConfiguration{}
+ versionOpts.Status = utils.VersioningDisabled
+ if value.Versioning != nil {
+ if value.Versioning.Status == utils.VersioningEnabled {
+ versionOpts.Status = utils.VersioningEnabled
+ }
+ }
+ sseOpts := model.SSEConfiguration{}
+ if value.ServerSideEncryption != nil {
+ if value.ServerSideEncryption.SseType == "SSE" {
+ sseOpts.SSE.Enabled = "true"
+ } else {
+ sseOpts.SSE.Enabled = "false"
+ }
+ }
+ bucket := model.Bucket{Name: value.Name, CreateTime: ctime, LocationConstraint: value.DefaultLocation,
+ VersionOpts: versionOpts, SSEOpts: sseOpts}
+ buckets = append(buckets, bucket)
+ }
+ temp.Buckets = buckets
+
+ xmlstring, err := xml.MarshalIndent(temp, "", " ")
+ if err != nil {
+ log.Errorf("Parse ListBuckets error: %v", err)
+ return nil
+ }
+ xmlstring = []byte(xml.Header + string(xmlstring))
+ return xmlstring
+}
+
+func (s *APIService) ListBuckets(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "bucket:list") {
+ return
+ }
+ log.Infof("Received request for all buckets")
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ rsp, err := s.s3Client.ListBuckets(ctx, &s3.BaseRequest{})
+ if HandleS3Error(response, request, err, rsp.GetErrorCode()) != nil {
+ log.Errorf("list bucket failed, err=%v, errCode=%d\n", err, rsp.GetErrorCode())
+ return
+ }
+
+ realRes := parseListBuckets(rsp)
+
+ log.Infof("Get List of buckets successfully:%v\n", string(realRes))
+ response.Write(realRes)
+}
+
+
+
package s3
+
+import (
+ "net/url"
+ "sort"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ s3error "github.com/opensds/multi-cloud/s3/error"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) ListBucketUploadRecords(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+
+ log.Infof("Received request for listing multipart uploads records: %s\n", bucketName)
+
+ parameters, err := parseListUploadsQuery(request.Request.URL.Query())
+ if err != nil {
+ log.Errorln("failed to parse list upload query parameter. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ listMultipartsResponse, err := s.s3Client.ListBucketUploadRecords(ctx, &pb.ListBucketUploadRequest{
+ BucketName: bucketName,
+ Delimiter: parameters.Delimiter,
+ EncodingType: parameters.EncodingType,
+ MaxUploads: int32(parameters.MaxUploads),
+ KeyMarker: parameters.KeyMarker,
+ Prefix: parameters.Prefix,
+ UploadIdMarker: parameters.UploadIdMarker,
+ })
+ if err != nil || listMultipartsResponse.ErrorCode != int32(s3error.ErrNoErr) {
+ log.Errorf("Unable to list multipart uploads. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ resData := datatype.ListMultipartUploadsResponse{}
+ resData.IsTruncated = listMultipartsResponse.Result.IsTruncated
+ resData.NextKeyMarker = listMultipartsResponse.Result.NextKeyMarker
+ resData.NextUploadIdMarker = listMultipartsResponse.Result.NextUploadIdMarker
+ sort.Strings(listMultipartsResponse.Result.CommonPrefix)
+ for _, prefix := range listMultipartsResponse.Result.CommonPrefix {
+ resData.CommonPrefixes = append(resData.CommonPrefixes, datatype.CommonPrefix{
+ Prefix: prefix,
+ })
+ }
+ for _, upload := range listMultipartsResponse.Result.Uploads {
+ resData.Uploads = append(resData.Uploads, datatype.Upload{
+ Key: upload.Key,
+ UploadId: upload.UploadId,
+ Initiator: datatype.Initiator{
+ ID: upload.Initiator.Id,
+ DisplayName: upload.Initiator.DisplayName,
+ },
+ Owner: datatype.Owner{
+ ID: upload.Owner.Id,
+ DisplayName: upload.Owner.DisplayName,
+ },
+ StorageClass: upload.StorageClass,
+ Initiated: upload.Initiated,
+ })
+ }
+
+ resData.Bucket = bucketName
+ resData.KeyMarker = parameters.KeyMarker
+ resData.UploadIdMarker = parameters.UploadIdMarker
+ resData.MaxUploads = parameters.MaxUploads
+ resData.Prefix = parameters.Prefix
+ resData.Delimiter = parameters.Delimiter
+ resData.EncodingType = parameters.EncodingType
+ if resData.EncodingType != "" { // only support "url" encoding for now
+ resData.Delimiter = url.QueryEscape(resData.Delimiter)
+ resData.KeyMarker = url.QueryEscape(resData.KeyMarker)
+ resData.Prefix = url.QueryEscape(resData.Prefix)
+ resData.NextKeyMarker = url.QueryEscape(resData.NextKeyMarker)
+ for _, upload := range resData.Uploads {
+ upload.Key = url.QueryEscape(upload.Key)
+ }
+ }
+
+ encodedSuccessResponse := EncodeResponse(resData)
+ // write success response.
+ WriteSuccessResponse(response, encodedSuccessResponse)
+ log.Infoln("List bucket multipart uploads successfully.")
+}
+
+
+
package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) ListObjectParts(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ objectKey := request.PathParameter("objectKey")
+ log.Infof("received request: list object multipart, bucketet[name=%s] object[name=%s]\n", bucketName, objectKey)
+
+ listPartReq, err := parseListObjectPartsQuery(request.Request.URL.Query())
+ if err != nil {
+ log.Errorln("failed to parse object part query parameter. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ listObjectPartsRes, err := s.s3Client.ListObjectParts(ctx, &pb.ListObjectPartsRequest{
+ BucketName: bucketName,
+ ObjectKey: objectKey,
+ UploadId: listPartReq.UploadId,
+ EncodingType: listPartReq.EncodingType,
+ MaxParts: int64(listPartReq.MaxParts),
+ PartNumberMarker: int64(listPartReq.PartNumberMarker),
+ })
+ if err != nil {
+ log.Errorln("unable to list uploaded parts. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ data := datatype.ListPartsResponse{
+ Bucket: bucketName,
+ Key: objectKey,
+ UploadId: listPartReq.UploadId,
+ EncodingType: listPartReq.EncodingType,
+ Initiator: datatype.Initiator(datatype.Owner{
+ ID: listObjectPartsRes.Initiator.Id,
+ DisplayName: listObjectPartsRes.Initiator.DisplayName,
+ }),
+ Owner: datatype.Owner{
+ ID: listObjectPartsRes.Owner.Id,
+ DisplayName: listObjectPartsRes.Owner.DisplayName,
+ },
+ PartNumberMarker: int(listObjectPartsRes.PartNumberMarker),
+ NextPartNumberMarker: int(listObjectPartsRes.NextPartNumberMarker),
+ MaxParts: int(listObjectPartsRes.MaxParts),
+ IsTruncated: listObjectPartsRes.IsTruncated,
+ }
+ data.Parts = make([]datatype.Part, 0)
+ for _, part := range listObjectPartsRes.Parts {
+ data.Parts = append(data.Parts, datatype.Part{
+ PartNumber: int(part.PartNumber),
+ ETag: part.ETag,
+ LastModified: part.LastModified,
+ Size: part.Size,
+ })
+ }
+
+ encodedSuccessResponse := EncodeResponse(data)
+ // Write success response.
+ log.Infof("list object parts successfully.")
+ WriteSuccessResponse(response, encodedSuccessResponse)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "encoding/xml"
+ "net/http"
+ "github.com/emicklei/go-restful"
+ log "github.com/sirupsen/logrus"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ "github.com/opensds/multi-cloud/s3/proto"
+)
+
+func (s *APIService) GetStorageClasses(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "storageclass:get") {
+ return
+ }
+ log.Info("Received request for storage classes.")
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ res, err := s.s3Client.GetStorageClasses(ctx, &s3.BaseRequest{})
+ if err != nil {
+ response.WriteError(http.StatusInternalServerError, err)
+ return
+ }
+
+ tmp := model.ListStorageClasses{}
+ classes := []model.StorageClass{}
+ for _, v := range res.Classes {
+ classes = append(classes, model.StorageClass{Name: v.Name, Tier: v.Tier})
+ }
+ tmp.Classes = classes
+
+ xmlstring, err := xml.MarshalIndent(tmp, "", " ")
+ if err != nil {
+ log.Errorf("parse ListStorageClasses error: %v\n", err)
+ response.WriteError(http.StatusInternalServerError, err)
+ } else {
+ xmlstring = []byte(xml.Header + string(xmlstring))
+ response.Write(xmlstring)
+ log.Infof("Get List of storage classes successfully:%v\n", string(xmlstring))
+ }
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+
+ . "github.com/opensds/multi-cloud/s3/error"
+)
+
+func (s *APIService) MultiPartUploadInit(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ objectKey := request.PathParameter("objectKey")
+
+ log.Infof("received request: multipart init, objectkey=%s, bucketName=%s\n:",
+ objectKey, bucketName)
+
+ if !isValidObjectName(objectKey) {
+ log.Errorf("object name is not valid.")
+ WriteErrorResponse(response, request, ErrInvalidObjectName)
+ return
+ }
+
+ acl, err := getAclFromHeader(request)
+ if err != nil {
+ log.Errorf("failed to get acl from http header, err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ // Save metadata.
+ attr := extractMetadataFromHeader(request)
+
+ tier, err := getTierFromHeader(request)
+ if err != nil {
+ log.Errorf("failed to get storage class from http header. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ result, err := s.s3Client.InitMultipartUpload(ctx, &pb.InitMultiPartRequest{
+ BucketName: bucketName, ObjectKey: objectKey, Acl: &pb.Acl{CannedAcl: acl.CannedAcl}, Tier: int32(tier), Attrs: attr})
+ if HandleS3Error(response, request, err, result.GetErrorCode()) != nil {
+ log.Errorln("unable to init multipart. err:%v, errcode:%v", err, result.ErrorCode)
+ return
+ }
+
+ data := GenerateInitiateMultipartUploadResponse(bucketName, objectKey, result.UploadID)
+ encodedSuccessResponse := EncodeResponse(data)
+ // write success response.
+ WriteSuccessResponse(response, encodedSuccessResponse)
+
+ log.Infof("Init multipart upload[bucketName=%s, objectKey=%s] successfully.\n",
+ bucketName, objectKey)
+}
+
+
+
/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+ "net/http"
+ "strings"
+ "time"
+
+ . "github.com/opensds/multi-cloud/s3/error"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+// Validates the preconditions for CopyObject, returns nil if validates
+// Preconditions supported are:
+// x-amz-copy-source-if-modified-since
+// x-amz-copy-source-if-unmodified-since
+// x-amz-copy-source-if-match
+// x-amz-copy-source-if-none-match
+func checkObjectPreconditions(w http.ResponseWriter, r *http.Request, object *pb.Object) error {
+ // x-amz-copy-source-if-modified-since: Return the object only if it has been modified
+ // since the specified time
+ ifModifiedSinceHeader := r.Header.Get("x-amz-copy-source-if-modified-since")
+ if ifModifiedSinceHeader != "" {
+ givenTime, err := time.Parse(http.TimeFormat, ifModifiedSinceHeader)
+ if err != nil {
+ return ErrInvalidPrecondition
+ }
+ if time.Unix(object.LastModified, 0).Before(givenTime) {
+ // If the object is not modified since the specified time.
+ return ErrPreconditionFailed
+ }
+ }
+
+ // x-amz-copy-source-if-unmodified-since : Return the object only if it has not been
+ // modified since the specified time
+ ifUnmodifiedSinceHeader := r.Header.Get("x-amz-copy-source-if-unmodified-since")
+ if ifUnmodifiedSinceHeader != "" {
+ givenTime, err := time.Parse(http.TimeFormat, ifUnmodifiedSinceHeader)
+ if err != nil {
+ return ErrInvalidPrecondition
+ }
+ if time.Unix(object.LastModified, 0).After(givenTime) {
+ // If the object is modified since the specified time.
+ return ErrPreconditionFailed
+ }
+ }
+
+ // x-amz-copy-source-if-match : Return the object only if its entity tag (ETag) is the
+ // same as the one specified
+ ifMatchETagHeader := r.Header.Get("x-amz-copy-source-if-match")
+ if ifMatchETagHeader != "" {
+ if !isETagEqual(object.Etag, ifMatchETagHeader) {
+ // If the object ETag does not match with the specified ETag.
+ return ErrPreconditionFailed
+ }
+ }
+
+ // If-None-Match : Return the object only if its entity tag (ETag) is different from the
+ // one specified
+ ifNoneMatchETagHeader := r.Header.Get("x-amz-copy-source-if-none-match")
+ if ifNoneMatchETagHeader != "" {
+ if isETagEqual(object.Etag, ifNoneMatchETagHeader) {
+ // If the object ETag matches with the specified ETag.
+ return ErrPreconditionFailed
+ }
+ }
+
+ if ifNoneMatchETagHeader != "" && ifUnmodifiedSinceHeader != "" {
+ return ErrInvalidPrecondition
+ }
+ if ifMatchETagHeader != "" && ifModifiedSinceHeader != "" {
+ return ErrInvalidPrecondition
+ }
+
+ return nil
+}
+
+// Validates the preconditions for GetObject/HeadObject. Returns nil if validates
+// Preconditions supported are:
+// If-Modified-Since
+// If-Unmodified-Since
+// If-Match
+// If-None-Match
+func checkPreconditions(header http.Header, object *pb.Object) error {
+ // If-Modified-Since : Return the object only if it has been modified since the specified time,
+ // otherwise return a 304 (not modified).
+ ifModifiedSinceHeader := header.Get("If-Modified-Since")
+ if ifModifiedSinceHeader != "" {
+ givenTime, err := time.Parse(http.TimeFormat, ifModifiedSinceHeader)
+ if err != nil {
+ return ErrInvalidPrecondition
+ }
+ if time.Unix(object.LastModified, 0).Before(givenTime) {
+ // If the object is not modified since the specified time.
+ return ContentNotModified
+ }
+ }
+
+ // If-Unmodified-Since : Return the object only if it has not been modified since the specified
+ // time, otherwise return a 412 (precondition failed).
+ ifUnmodifiedSinceHeader := header.Get("If-Unmodified-Since")
+ if ifUnmodifiedSinceHeader != "" {
+ givenTime, err := time.Parse(http.TimeFormat, ifUnmodifiedSinceHeader)
+ if err != nil {
+ return ErrInvalidPrecondition
+ }
+ if time.Unix(object.LastModified, 0).After(givenTime) {
+ return ErrPreconditionFailed
+ }
+ }
+
+ // If-Match : Return the object only if its entity tag (ETag) is the same as the one specified;
+ // otherwise return a 412 (precondition failed).
+ ifMatchETagHeader := header.Get("If-Match")
+ if ifMatchETagHeader != "" {
+ if !isETagEqual(object.Etag, ifMatchETagHeader) {
+ // If the object ETag does not match with the specified ETag.
+ return ErrPreconditionFailed
+ }
+ }
+
+ // If-None-Match : Return the object only if its entity tag (ETag) is different from the
+ // one specified otherwise, return a 304 (not modified).
+ ifNoneMatchETagHeader := header.Get("If-None-Match")
+ if ifNoneMatchETagHeader != "" {
+ if isETagEqual(object.Etag, ifNoneMatchETagHeader) {
+ // If the object ETag matches with the specified ETag.
+ return ContentNotModified
+ }
+ }
+ return nil
+}
+
+// canonicalizeETag returns ETag with leading and trailing double-quotes removed,
+// if any present
+func canonicalizeETag(etag string) string {
+ canonicalETag := strings.TrimPrefix(etag, "\"")
+ return strings.TrimSuffix(canonicalETag, "\"")
+}
+
+// isETagEqual return true if the canonical representations of two ETag strings
+// are equal, false otherwise
+func isETagEqual(left, right string) bool {
+ return canonicalizeETag(left) == canonicalizeETag(right)
+}
+
+
+
/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+ "regexp"
+ "strconv"
+ "strings"
+ "unicode/utf8"
+ "fmt"
+)
+
+// validBucket regexp.
+var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
+
+// IsValidBucketName verifies a bucket name in accordance with Amazon's
+// requirements. It must be 3-63 characters long, can contain dashes
+// and periods, but must begin and end with a lowercase letter or a number.
+// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
+func isValidBucketName(bucketName string) bool {
+ if !validBucket.MatchString(bucketName) {
+ return false
+ }
+ // make sure there're no continuous dots
+ if strings.Contains(bucketName, "..") {
+ return false
+ }
+ // make sure it's not an IP address
+ split := strings.Split(bucketName, ".")
+ if len(split) == 4 {
+ for _, p := range split {
+ n, err := strconv.Atoi(p)
+ if err == nil && n >= 0 && n <= 255 {
+ return false
+ }
+ }
+ }
+ return true
+}
+
+// IsValidObjectName verifies an object name in accordance with Amazon's
+// requirements. It cannot exceed 1024 characters and must be a valid UTF8
+// string.
+// Some characters require special handling:
+// & $ @ = ; : + (space) , ?
+// and ASCII ranges 0x00-0x1F(0-31 decimal) and 7F(127 decimal)
+// Some characters to avoid:
+// \ { ^ } % ` [ ] ' " < > ~ # |
+// and non-printable ASCII characters(128-255 decimal)
+//
+// As in YIG, we PROHIBIT ALL the characters listed above
+// See http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
+func isValidObjectName(objectName string) bool {
+ if len(objectName) <= 0 || len(objectName) > 1024 {
+ return false
+ }
+ if !utf8.ValidString(objectName) {
+ return false
+ }
+ for _, n := range objectName {
+ if (n >= 0 && n <= 31) || (n >= 127 && n <= 255) {
+ return false
+ }
+ c := string(n)
+ if strings.ContainsAny(c, "\\") {
+ return false
+ }
+ }
+ return true
+}
+
+// Argument position in handler AppendObject must be non-negative integer
+func checkPosition(position string) (uint64, error) {
+ p, err := strconv.ParseUint(position, 10, 64)
+ if err != nil {
+ return 0, err
+ }
+ if p < 0 {
+ return 0, fmt.Errorf("position must ben on-negative integer.")
+ }
+ return p, nil
+}
+
+func isFirstAppend(position uint64) bool {
+ return position == 0
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "encoding/xml"
+ "io"
+ "io/ioutil"
+ "strings"
+
+ "github.com/emicklei/go-restful"
+ "github.com/journeymidnight/yig/helper"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ s3error "github.com/opensds/multi-cloud/s3/error"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) ObjectAclPut(request *restful.Request, response *restful.Response) {
+ bucketName := strings.ToLower(request.PathParameter(common.REQUEST_PATH_BUCKET_NAME))
+ objectKey := request.PathParameter(common.REQUEST_PATH_OBJECT_KEY)
+ log.Infof("received request: PUT bucket[name=%s] object[name=%s] acl\n", bucketName, objectKey)
+
+ var err error
+ var acl datatype.Acl
+ var policy datatype.AccessControlPolicy
+ if _, ok := request.Request.Header[common.REQUEST_HEADER_ACL]; ok {
+ acl, err = getAclFromHeader(request)
+ if err != nil {
+ WriteErrorResponse(response, request, err)
+ return
+ }
+ } else {
+ // because we only support canned acl, the body of request must be not too big, and 1024 is enough
+ aclBuffer, err := ioutil.ReadAll(io.LimitReader(request.Request.Body, 1024))
+ if err != nil {
+ log.Errorf("unable to read acls body, err:", err)
+ WriteErrorResponse(response, request, s3error.ErrInvalidAcl)
+ return
+ }
+ err = xml.Unmarshal(aclBuffer, &policy)
+ if err != nil {
+ log.Errorf("unable to parse acls xml body, err:", err)
+ WriteErrorResponse(response, request, s3error.ErrInternalError)
+ return
+ }
+ }
+
+ if acl.CannedAcl == "" {
+ newCannedAcl, err := datatype.GetCannedAclFromPolicy(policy)
+ if err != nil {
+ log.Error("failed to get canned acl from policy. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+ acl = newCannedAcl
+ }
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ res, err := s.s3Client.PutObjACL(ctx, &pb.PutObjACLRequest{ACLConfig: &pb.ObjACL{BucketName: bucketName,
+ ObjectKey: objectKey, CannedAcl: acl.CannedAcl}})
+ if err != nil || res.ErrorCode != int32(s3error.ErrNoErr) {
+ WriteErrorResponse(response, request, GetFinalError(err, res.ErrorCode))
+ return
+ }
+
+ log.Infoln("PUT object acl successfully.")
+ WriteSuccessResponse(response, nil)
+}
+
+func (s *APIService) ObjectAclGet(request *restful.Request, response *restful.Response) {
+ bucketName := strings.ToLower(request.PathParameter(common.REQUEST_PATH_BUCKET_NAME))
+ objectKey := request.PathParameter(common.REQUEST_PATH_OBJECT_KEY)
+ log.Infof("received request: GET bucket[name=%s] object[name=%s] acl\n", bucketName, objectKey)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ object, err := s.getObjectMeta(ctx, bucketName, objectKey, "")
+ if err != nil {
+ log.Error("failed to get object[%s] meta", objectKey)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ owner := datatype.Owner{ID: object.TenantId, DisplayName: object.TenantId}
+ bucketOwner := datatype.Owner{}
+ policy, err := datatype.CreatePolicyFromCanned(owner, bucketOwner, datatype.Acl{CannedAcl: object.Acl.CannedAcl})
+ if err != nil {
+ log.Error("failed to create policy. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ aclBuffer, err := xmlFormat(policy)
+ if err != nil {
+ helper.ErrorIf(err, "failed to marshal acl XML for bucket", bucketName)
+ WriteErrorResponse(response, request, s3error.ErrInternalError)
+ return
+ }
+
+ setXmlHeader(response, aclBuffer)
+ WriteSuccessResponse(response, aclBuffer)
+ log.Infoln("GET object acl successfully.")
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "net/url"
+ "strings"
+ "time"
+
+ "github.com/emicklei/go-restful"
+ "github.com/micro/go-micro/client"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func getTierFromHeader(request *restful.Request) (types.StorageClass, error) {
+ storageClassStr := request.HeaderParameter(common.REQUEST_HEADER_STORAGE_CLASS)
+ if storageClassStr != "" {
+ return types.MatchStorageClassIndex(storageClassStr)
+ } else {
+ // If you don't specify this header, STANDARD will be used
+ return utils.Tier1, nil
+ }
+}
+
+// ObjectCopy copy object from http header x-amz-copy-source
+func (s *APIService) ObjectCopy(request *restful.Request, response *restful.Response) {
+ log.Infof("received request: Copy object")
+
+ targetBucketName := request.PathParameter(common.REQUEST_PATH_BUCKET_NAME)
+ targetObjectName := request.PathParameter(common.REQUEST_PATH_OBJECT_KEY)
+ //backendName := request.HeaderParameter(common.REQUEST_HEADER_STORAGE_CLASS)
+ log.Infof("received request: Copy object, objectkey=%s, bucketName=%s\n:",
+ targetObjectName, targetBucketName)
+
+ // copy source is of form: /bucket-name/object-name?versionId=xxxxxx
+ copySource := request.HeaderParameter(common.REQUEST_HEADER_COPY_SOURCE)
+ if copySource == "" {
+ WriteErrorResponse(response, request, ErrInvalidCopySource)
+ return
+ }
+ // Skip the first element if it is '/', split the rest.
+ if strings.HasPrefix(copySource, "/") {
+ copySource = copySource[1:]
+ }
+ splits := strings.SplitN(copySource, "/", 2)
+
+ // Save sourceBucket and sourceObject extracted from url Path.
+ var err error
+ var sourceBucketName, sourceObjectName, sourceVersion string
+ if len(splits) == 2 {
+ sourceBucketName = splits[0]
+ sourceObjectName = splits[1]
+ } else {
+ log.Infoln("copy source should be splited at least two parts.")
+ WriteErrorResponse(response, request, ErrInvalidCopySource)
+ return
+ }
+ // If source object is empty, reply back error.
+ if sourceBucketName == "" || sourceObjectName == "" {
+ WriteErrorResponse(response, request, ErrInvalidCopySource)
+ return
+ }
+
+ splits = strings.SplitN(sourceObjectName, "?", 2)
+ if len(splits) == 2 {
+ sourceObjectName = splits[0]
+ if !strings.HasPrefix(splits[1], "versionId=") {
+ WriteErrorResponse(response, request, ErrInvalidCopySource)
+ return
+ }
+ sourceVersion = strings.TrimPrefix(splits[1], "versionId=")
+ }
+
+ // X-Amz-Copy-Source should be URL-encoded
+ sourceBucketName, err = url.QueryUnescape(sourceBucketName)
+ if err != nil {
+ WriteErrorResponse(response, request, ErrInvalidCopySource)
+ return
+ }
+ sourceObjectName, err = url.QueryUnescape(sourceObjectName)
+ if err != nil {
+ WriteErrorResponse(response, request, ErrInvalidCopySource)
+ return
+ }
+
+ var isOnlyUpdateMetadata = false
+ if sourceBucketName == targetBucketName && sourceObjectName == targetObjectName {
+ if request.HeaderParameter("X-Amz-Metadata-Directive") == "COPY" {
+ WriteErrorResponse(response, request, ErrInvalidCopyDest)
+ return
+ } else if request.HeaderParameter("X-Amz-Metadata-Directive") == "REPLACE" {
+ isOnlyUpdateMetadata = true
+ } else {
+ WriteErrorResponse(response, request, ErrInvalidRequestBody)
+ return
+ }
+ }
+
+ log.Infoln("sourceBucketName:", sourceBucketName, " sourceObjectName:", sourceObjectName, " sourceVersion:", sourceVersion)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ sourceObject, err := s.getObjectMeta(ctx, sourceBucketName, sourceObjectName, "")
+ if err != nil {
+ log.Errorln("unable to fetch object info. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ // Verify before x-amz-copy-source preconditions before continuing with CopyObject.
+ if err = checkObjectPreconditions(response.ResponseWriter, request.Request, sourceObject); err != nil {
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ //TODO: In a versioning-enabled bucket, you cannot change the storage class of a specific version of an object. When you copy it, Amazon S3 gives it a new version ID.
+ storClass, err := getTierFromHeader(request)
+ if err != nil {
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ // if source == dest and X-Amz-Metadata-Directive == REPLACE, only update the meta;
+ if isOnlyUpdateMetadata {
+ log.Infoln("only update metadata.")
+ targetObject := sourceObject
+
+ //update custom attrs from headers
+ newMetadata := extractMetadataFromHeader(request)
+ if c, ok := newMetadata["Content-Type"]; ok {
+ targetObject.ContentType = c
+ } else {
+ targetObject.ContentType = sourceObject.ContentType
+ }
+ targetObject.CustomAttributes = newMetadata
+ targetObject.Tier = int32(storClass)
+
+ result, err := s.s3Client.UpdateObjectMeta(ctx, targetObject)
+ if err != nil {
+ log.Errorf("unable to update object meta for %v", targetObject.ObjectId)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+ copyObjRes := GenerateCopyObjectResponse(result.Md5, time.Unix(result.LastModified, 0))
+ encodedSuccessResponse := EncodeResponse(copyObjRes)
+ // write headers
+ if result.Md5 != "" {
+ response.ResponseWriter.Header()["ETag"] = []string{"\"" + result.Md5 + "\""}
+ }
+ if sourceVersion != "" {
+ response.AddHeader("x-amz-copy-source-version-id", sourceVersion)
+ }
+ if result.VersionId != "" {
+ response.AddHeader("x-amz-version-id", result.VersionId)
+ }
+
+ log.Info("Update object meta successfully.")
+ // write success response.
+ WriteSuccessResponse(response, encodedSuccessResponse)
+ return
+ }
+
+ /// maximum Upload size for object in a single CopyObject operation.
+ if isMaxObjectSize(sourceObject.Size) {
+ WriteErrorResponseWithResource(response, request, ErrEntityTooLarge, copySource)
+ return
+ }
+
+ log.Infoln("srcBucket:", sourceBucketName, " srcObject:", sourceObjectName,
+ " targetBucket:", targetBucketName, " targetObject:", targetObjectName)
+ tmoutSec := sourceObject.Size / MiniSpeed
+ opt := client.WithRequestTimeout(time.Duration(tmoutSec) * time.Second)
+ result, err := s.s3Client.CopyObject(ctx, &pb.CopyObjectRequest{
+ SrcBucketName: sourceBucketName,
+ TargetBucketName: targetBucketName,
+ SrcObjectName: sourceObjectName,
+ TargetObjectName: targetObjectName,
+ }, opt)
+ if HandleS3Error(response, request, err, result.GetErrorCode()) != nil {
+ log.Errorf("unable to copy object, err=%v, errCode=%v\n", err, result.GetErrorCode())
+ return
+ }
+
+ copyObjRes := GenerateCopyObjectResponse(result.Md5, time.Unix(result.LastModified, 0))
+ encodedSuccessResponse := EncodeResponse(copyObjRes)
+ // write headers
+ if result.Md5 != "" {
+ response.ResponseWriter.Header()["ETag"] = []string{"\"" + result.Md5 + "\""}
+ }
+
+ // write success response.
+ WriteSuccessResponse(response, encodedSuccessResponse)
+ log.Info("COPY object successfully.")
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "strings"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) ObjectDelete(request *restful.Request, response *restful.Response) {
+ url := request.Request.URL
+ bucketName := request.PathParameter("bucketName")
+ objectName := request.PathParameter("objectKey")
+ version := url.Query().Get("versionId")
+ if strings.HasSuffix(url.String(), "/") { // This is for folder.
+ objectName = objectName + "/"
+ }
+
+ if len(bucketName) == 0 {
+ log.Errorf("invalid input, bucket=%s\n", bucketName)
+ WriteErrorResponse(response, request, ErrInvalidBucketName)
+ return
+ }
+ if len(objectName) == 0 {
+ log.Errorf("invalid input, object=%s\n", objectName)
+ WriteErrorResponse(response, request, ErrInvalidObjectName)
+ }
+
+ input := s3.DeleteObjectInput{Bucket: bucketName, Key: objectName}
+ if len(version) > 0 {
+ input.VersioId = version
+ }
+ ctx := common.InitCtxWithAuthInfo(request)
+ rsp, err := s.s3Client.DeleteObject(ctx, &input)
+ if HandleS3Error(response, request, err, rsp.GetErrorCode()) != nil {
+ log.Errorf("delete object[%s] failed, err=%v, errCode=%d\n", objectName, err, rsp.GetErrorCode())
+ return
+ }
+
+ if rsp.DeleteMarker {
+ response.Header().Set("x-amz-delete-marker", "true")
+ } else {
+ response.Header().Set("x-amz-delete-marker", "false")
+ }
+ if rsp.VersionId != "" {
+ response.Header().Set("x-amz-version-id", rsp.VersionId)
+ }
+
+ log.Infof("delete object[%s] from bucket[%s] succeed.", objectName, bucketName)
+ WriteSuccessNoContent(response)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "io"
+ "net/http"
+ "net/url"
+ "time"
+
+ "github.com/emicklei/go-restful"
+ "github.com/micro/go-micro/client"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ s3error "github.com/opensds/multi-cloud/s3/error"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+var MiniSpeed int64 = 5 // 5KByte/Sec
+
+// supportedGetReqParams - supported request parameters for GET presigned request.
+var supportedGetReqParams = map[string]string{
+ "response-expires": "Expires",
+ "response-content-type": "Content-Type",
+ "response-cache-control": "Cache-Control",
+ "response-content-disposition": "Content-Disposition",
+ "response-content-language": "Content-Language",
+ "response-content-encoding": "Content-Encoding",
+}
+
+// setGetRespHeaders - set any requested parameters as response headers.
+func setGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
+ for k, v := range reqParams {
+ if header, ok := supportedGetReqParams[k]; ok {
+ w.Header()[header] = v
+ }
+ }
+}
+
+// GetObjectHandler - GET Object
+func (s *APIService) ObjectGet(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ objectKey := request.PathParameter("objectKey")
+ rangestr := request.HeaderParameter("Range")
+ log.Infof("%v\n", rangestr)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ object, err := s.getObjectMeta(ctx, bucketName, objectKey, "")
+ if err != nil {
+ log.Errorln("get object meta failed. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ // Get request range.
+ var hrange *HttpRange
+ rangeHeader := request.HeaderParameter("Range")
+ if rangeHeader != "" {
+ if hrange, err = ParseRequestRange(rangeHeader, object.Size); err != nil {
+ // Handle only ErrorInvalidRange
+ // Ignore other parse error and treat it as regular Get request like Amazon S3.
+ if err == ErrorInvalidRange {
+ WriteErrorResponse(response, request, s3error.ErrInvalidRange)
+ return
+ }
+ // log the error.
+ log.Errorln("invalid request range, err:", err)
+ }
+ }
+
+ // Validate pre-conditions if any.
+ if err = checkPreconditions(request.Request.Header, object); err != nil {
+ // set object-related metadata headers
+ response.AddHeader("Last-Modified", time.Unix(object.LastModified, 0).UTC().Format(http.TimeFormat))
+
+ if object.Etag != "" {
+ response.ResponseWriter.Header()["ETag"] = []string{"\"" + object.Etag + "\""}
+ }
+ if err == s3error.ContentNotModified { // write only header if is a 304
+ WriteErrorResponseHeaders(response, err)
+ } else {
+ WriteErrorResponse(response, request, err)
+ }
+ return
+ }
+
+ // Get the object.
+ startOffset := int64(0)
+ length := object.Size
+ if hrange != nil {
+ startOffset = hrange.OffsetBegin
+ length = hrange.GetLength()
+ }
+ tmoutSec := object.Size / MiniSpeed
+ opt := client.WithRequestTimeout(time.Duration(tmoutSec) * time.Second)
+ stream, err := s.s3Client.GetObject(ctx, &pb.GetObjectInput{Bucket: bucketName, Key: objectKey, Offset: startOffset, Length: length}, opt)
+ if err != nil {
+ log.Errorln("get object failed, err:%v", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+ defer stream.Close()
+
+ // Indicates if any data was written to the http.ResponseWriter
+ dataWritten := false
+ // io.Writer type which keeps track if any data was written.
+ writer := func(p []byte) (int, error) {
+ if !dataWritten {
+ // Set headers on the first write.
+ // Set standard object headers.
+ SetObjectHeaders(response, object, hrange)
+
+ // Set any additional requested response headers.
+ setGetRespHeaders(response.ResponseWriter, request.Request.URL.Query())
+ dataWritten = true
+ }
+ n, err := response.Write(p)
+ return n, err
+ }
+
+ s3err := int32(s3error.ErrNoErr)
+ eof := false
+ left := length
+ for !eof && left > 0 {
+ rsp, err := stream.Recv()
+ if err != nil && err != io.EOF {
+ log.Errorln("recv err", err)
+ break
+ }
+ // If err is equal to EOF, a non-zero number of bytes may be returned.
+ // the err is set EOF, returned data is processed at the subsequent code.
+ if err == io.EOF {
+ eof = true
+ }
+ // It indicate that there is a error from grpc server.
+ if rsp.GetErrorCode() != int32(s3error.ErrNoErr) {
+ s3err = rsp.GetErrorCode()
+ log.Errorf("received s3 service error, error code:%v", s3err)
+ break
+ }
+ // If there is no data in rsp.Data, it show that there is no more data to receive
+ if len(rsp.Data) == 0 {
+ break
+ }
+ _, err = writer(rsp.Data)
+ if err != nil {
+ log.Errorln("failed to write data to client. err:", err)
+ break
+ }
+ left -= int64(len(rsp.Data))
+ }
+ log.Debugf("left bytes=%d\n", left)
+
+ if !dataWritten {
+ if s3err == int32(s3error.ErrNoErr) {
+ writer(nil)
+ } else {
+ WriteErrorResponse(response, request, s3error.S3ErrorCode(s3err))
+ return
+ }
+ }
+
+ log.Infof("Get object[%s] end.\n", objectKey)
+}
+
+func (s *APIService) HeadObject(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ objectKey := request.PathParameter("objectKey")
+ versionId := request.Request.URL.Query().Get("versionId")
+ log.Infof("Received request for head object: bucket=%s, objectkey=%s, version=%s\n", bucketName, objectKey, versionId)
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ object, err := s.getObjectMeta(ctx, bucketName, objectKey, versionId)
+ if err != nil {
+ log.Errorf("head object[bucketname=%s, key=%s] failed, err=%v\n", bucketName, objectKey, err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ if object.DeleteMarker {
+ response.Header().Set("x-amz-delete-marker", "true")
+ log.Errorf("object[bucketname=%s, key=%s] is marked as deleted\n", bucketName, objectKey)
+ WriteErrorResponse(response, request, s3error.ErrNoSuchKey)
+ return
+ }
+
+ // Get request range.
+ rangeHeader := request.Request.Header.Get("Range")
+ if rangeHeader != "" {
+ if _, err = ParseRequestRange(rangeHeader, object.Size); err != nil {
+ // Handle only ErrorInvalidRange
+ // Ignore other parse error and treat it as regular Get request like Amazon S3.
+ if err == ErrorInvalidRange {
+ WriteErrorResponse(response, request, s3error.ErrInvalidRange)
+ log.Errorf("invalid request range: %s\n", rangeHeader)
+ return
+ }
+ }
+ }
+
+ // Validate pre-conditions if any.
+ if err = checkPreconditions(request.Request.Header, object); err != nil {
+ // set object-related metadata headers
+ response.Header().Set("Last-Modified", time.Unix(object.LastModified, 0).Format(http.TimeFormat))
+
+ if object.Etag != "" {
+ response.Header()["ETag"] = []string{"\"" + object.Etag + "\""}
+ }
+ if err == s3error.ContentNotModified { // write only header if is a 304
+ log.Infof("content not modifed")
+ WriteErrorResponseHeaders(response, err)
+ } else {
+ log.Errorln("head object failed, err:", err)
+ WriteErrorResponse(response, request, err)
+ }
+ return
+ }
+
+ // TODO: add sse header to response
+
+ log.Debugf("object:%+v\n", object)
+ // Set standard object headers.
+ SetObjectHeaders(response, object, nil)
+
+ // Successful response.
+ response.WriteHeader(http.StatusOK)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "net/url"
+ "strconv"
+ "strings"
+
+ "github.com/emicklei/go-restful"
+ "github.com/journeymidnight/yig/helper"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ . "github.com/opensds/multi-cloud/s3/error"
+
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+// ObjectPartCopy copy object from http header x-amz-copy-source as a part for multipart
+func (s *APIService) ObjectPartCopy(request *restful.Request, response *restful.Response) {
+ targetBucketName := request.PathParameter(common.REQUEST_PATH_BUCKET_NAME)
+ targetObjectName := request.PathParameter(common.REQUEST_PATH_OBJECT_KEY)
+
+ log.Infof("received request: Copy object part, target bucket[%v], target object[%s]", targetBucketName, targetObjectName)
+
+ if !isValidObjectName(targetObjectName) {
+ log.Errorln("target object name is invalid. ")
+ WriteErrorResponse(response, request, ErrInvalidObjectName)
+ return
+ }
+
+ uploadID := request.QueryParameter("uploadId")
+ partIDString := request.QueryParameter("partNumber")
+ partID, err := strconv.Atoi(partIDString)
+ if err != nil {
+ log.Errorf("failed to convert part id string[%s] to integer, err:%v", partIDString, err)
+ WriteErrorResponse(response, request, ErrInvalidPart)
+ return
+ }
+ // check partID with maximum part ID for multipart objects
+ if isMaxPartID(partID) {
+ log.Errorln("part ID is greater than the maximum allowed ID.")
+ WriteErrorResponse(response, request, ErrInvalidMaxParts)
+ return
+ }
+
+ // copy source is of form: /bucket-name/object-name?versionId=xxxxxx
+ copySource := request.HeaderParameter(common.REQUEST_HEADER_COPY_SOURCE)
+
+ // Skip the first element if it is '/', split the rest.
+ if strings.HasPrefix(copySource, "/") {
+ copySource = copySource[1:]
+ }
+ splits := strings.SplitN(copySource, "/", 2)
+
+ // Save sourceBucket and sourceObject extracted from url Path.
+ var sourceBucketName, sourceObjectName string
+ if len(splits) == 2 {
+ sourceBucketName = splits[0]
+ sourceObjectName = splits[1]
+ } else {
+ log.Infoln("copy source should be splited at least two parts.")
+ WriteErrorResponse(response, request, ErrInvalidCopySource)
+ return
+ }
+ // If source object is empty, reply back error.
+ if sourceBucketName == "" || sourceObjectName == "" {
+ log.Errorln("there is no string for sourceBucketName or sourceObjectName.")
+ WriteErrorResponse(response, request, ErrInvalidCopySource)
+ return
+ }
+
+ splits = strings.SplitN(sourceObjectName, "?", 2)
+ if len(splits) > 1 {
+ // we dont support source object name that has "?"
+ log.Errorln("we dont support source object name that has ?")
+ WriteErrorResponse(response, request, ErrNotImplemented)
+ return
+ }
+
+ // X-Amz-Copy-Source should be URL-encoded
+ sourceBucketName, err = url.QueryUnescape(sourceBucketName)
+ if err != nil {
+ log.Errorln("failed to QueryUnescape. err:", err)
+ WriteErrorResponse(response, request, ErrInvalidCopySource)
+ return
+ }
+ sourceObjectName, err = url.QueryUnescape(sourceObjectName)
+ if err != nil {
+ log.Errorln("failed to QueryUnescape. err:", err)
+ WriteErrorResponse(response, request, ErrInvalidCopySource)
+ return
+ }
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ getObjMetaRes, err := s.s3Client.GetObjectMeta(ctx, &pb.Object{
+ ObjectKey: sourceObjectName,
+ BucketName: sourceBucketName,
+ })
+ if HandleS3Error(response, request, err, getObjMetaRes.GetErrorCode()) != nil {
+ log.Errorf("unable to fetch object meta. err:%v, errCode:%v", err, getObjMetaRes.ErrorCode)
+ return
+ }
+
+ // Verify before x-amz-copy-source preconditions before continuing with CopyObject.
+ if err = checkObjectPreconditions(response.ResponseWriter, request.Request, getObjMetaRes.Object); err != nil {
+ log.Infof("failed to check object preconditions. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ var readOffset, readLength int64
+ copySourceRangeString := request.HeaderParameter(common.REQUEST_HEADER_COPY_SOURCE_RANGE)
+ if copySourceRangeString == "" {
+ readOffset = 0
+ readLength = getObjMetaRes.Object.Size
+ } else {
+ copySourceRange, err := datatype.ParseRequestRange(copySourceRangeString, getObjMetaRes.Object.Size)
+ if err != nil {
+ helper.ErrorIf(err, "Invalid request range, err:", err)
+ WriteErrorResponse(response, request, ErrInvalidRange)
+ return
+ }
+ readOffset = copySourceRange.OffsetBegin
+ readLength = copySourceRange.GetLength()
+ }
+ if isMaxObjectSize(readLength) {
+ log.Errorf("object size is too large. size:%v", readLength)
+ WriteErrorResponseWithResource(response, request, ErrEntityTooLarge, copySource)
+ return
+ }
+
+ result, err := s.s3Client.CopyObjPart(ctx, &pb.CopyObjPartRequest{
+ TargetBucket: targetBucketName,
+ TargetObject: targetObjectName,
+ SourceObject: sourceObjectName,
+ SourceBucket: sourceBucketName,
+ UploadID: uploadID,
+ PartID: int64(partID),
+ ReadOffset: readOffset,
+ ReadLength: readLength,
+ })
+ if HandleS3Error(response, request, err, result.ErrorCode) != nil {
+ log.Errorf("failed to copy object part s3. err:%v, errCode:%v", err, result.ErrorCode)
+ return
+ }
+
+ data := GenerateCopyObjectPartResponse(result.Etag, result.LastModified)
+ encodedSuccessResponse := EncodeResponse(data)
+ // write headers
+ if result.Etag != "" {
+ response.ResponseWriter.Header()["ETag"] = []string{"\"" + result.Etag + "\""}
+ }
+ log.Infoln("copy object part successfully.")
+ // write success response.
+ WriteSuccessResponse(response, encodedSuccessResponse)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "io"
+ "net/http"
+ "net/url"
+ "strconv"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ "github.com/opensds/multi-cloud/s3/proto"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+// handle post object according to 'https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html'
+var ValidSuccessActionStatus = []string{"200", "201", "204"}
+
+func (s *APIService) ObjectPost(request *restful.Request, response *restful.Response) {
+ var err error
+ // Here the parameter is the size of the form data that should
+ // be loaded in memory, the remaining being put in temporary files.
+ reader, err := request.Request.MultipartReader()
+ if err != nil {
+ log.Errorf("failed to get reader from post request, err: %v", err)
+ WriteErrorResponse(response, request, s3error.ErrMalformedPOSTRequest)
+ return
+ }
+
+ fileBody, formValues, err := extractHTTPFormValues(reader)
+ if err != nil {
+ log.Errorf("failed to extract form values, err: %v", err)
+ WriteErrorResponse(response, request, s3error.ErrMalformedPOSTRequest)
+ return
+ }
+ objectKey := formValues[common.REQUEST_FORM_KEY]
+ if !isValidObjectName(objectKey) {
+ log.Errorf("got invalid object key: %s", objectKey)
+ WriteErrorResponse(response, request, s3error.ErrInvalidObjectName)
+ return
+ }
+
+ bucketName := request.PathParameter(common.REQUEST_PATH_BUCKET_NAME)
+ backendName := request.HeaderParameter(common.REQUEST_HEADER_STORAGE_CLASS)
+ formValues[common.REQUEST_FORM_BUCKET] = bucketName
+
+ // check if specific bucket exist
+ ctx := common.InitCtxWithAuthInfo(request)
+ bucketMeta, err := s.getBucketMeta(ctx, bucketName)
+ if err != nil {
+ log.Errorf("failed to get bucket meta. err: %v", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+ location := bucketMeta.DefaultLocation
+ if backendName != "" {
+ // check if backend exist
+ if s.isBackendExist(ctx, backendName) == false {
+ log.Errorf("backend %s for bucket %s doesn't exist", backendName, bucketName)
+ WriteErrorResponse(response, request, s3error.ErrGetBackendFailed)
+ return
+ }
+ location = backendName
+ }
+
+ metadata := extractMetadataFromHeader(request)
+
+ acl, err := getAclFromFormValues(formValues)
+ if err != nil {
+ log.Errorf("failed to get acl from from values, err: %v", err)
+ WriteErrorResponse(response, request, s3error.ErrInvalidCannedAcl)
+ return
+ }
+
+ buf := make([]byte, ChunkSize)
+ eof := false
+ stream, err := s.s3Client.PutObject(ctx)
+ defer stream.Close()
+ obj := pb.PutObjectRequest{
+ BucketName: bucketName,
+ ObjectKey: objectKey,
+ Acl: &pb.Acl{CannedAcl: acl.CannedAcl},
+ Attrs: metadata,
+ Location: location,
+ Size: -1,
+ }
+ err = stream.SendMsg(&obj)
+ if err != nil {
+ log.Errorf("failed to call grpc PutObject(%v), err: %v", obj, err)
+ WriteErrorResponse(response, request, s3error.ErrInternalError)
+ return
+ }
+ for !eof {
+ n, err := fileBody.Read(buf)
+ if err != nil && err != io.EOF {
+ log.Errorf("read error:%v", err)
+ break
+ }
+ if err == io.EOF {
+ log.Debugln("finished read")
+ eof = true
+ }
+ err = stream.Send(&s3.PutDataStream{Data: buf[:n]})
+ if err != nil {
+ log.Infof("stream send error: %v", err)
+ break
+ }
+ // make sure that the grpc server receives the EOF.
+ if eof {
+ stream.Send(&s3.PutDataStream{Data: buf[0:0]})
+ }
+ }
+
+ // if read or send data failed, then close stream and return error
+ if !eof {
+ log.Errorf("failed to send data to put object.")
+ WriteErrorResponse(response, request, s3error.ErrInternalError)
+ return
+ }
+
+ result := &s3.PutObjectResponse{}
+ err = stream.RecvMsg(result)
+ if err != nil {
+ log.Errorf("stream receive message failed:%v\n", err)
+ WriteErrorResponse(response, request, s3error.ErrInternalError)
+ return
+ }
+
+ log.Info("succeed to put object data for post object request.")
+ if result.Md5 != "" {
+ response.Header().Set("ETag", "\""+result.Md5+"\"")
+ }
+
+ var redirect string
+ redirect, _ = formValues["Success_action_redirect"]
+ if redirect == "" {
+ redirect, _ = formValues["redirect"]
+ }
+ if redirect != "" {
+ redirectUrl, err := url.Parse(redirect)
+ if err == nil {
+ redirectUrl.Query().Set("bucket", bucketName)
+ redirectUrl.Query().Set("key", objectKey)
+ redirectUrl.Query().Set("etag", result.Md5)
+ http.Redirect(response, request.Request, redirectUrl.String(), http.StatusSeeOther)
+ return
+ }
+ // If URL is Invalid, ignore the redirect field
+ }
+
+ var status string
+ status, _ = formValues["Success_action_status"]
+ if !helper.StringInSlice(status, ValidSuccessActionStatus) {
+ status = "204"
+ }
+
+ statusCode, _ := strconv.Atoi(status)
+ switch statusCode {
+ case 200, 204:
+ response.WriteHeader(statusCode)
+ case 201:
+ encodedSuccessResponse := EncodeResponse(datatype.PostResponse{
+ // TODO the full accessable url is needed.
+ Location: "/" + bucketName + "/" + objectKey,
+ Bucket: bucketName,
+ Key: objectKey,
+ ETag: result.Md5,
+ })
+ response.WriteHeader(201)
+ response.Write(encodedSuccessResponse)
+ }
+
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "strconv"
+ "strings"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/proto"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+var ChunkSize int = 2048
+
+//ObjectPut -
+func (s *APIService) ObjectPut(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter(common.REQUEST_PATH_BUCKET_NAME)
+ objectKey := request.PathParameter(common.REQUEST_PATH_OBJECT_KEY)
+ backendName := request.HeaderParameter(common.REQUEST_HEADER_STORAGE_CLASS)
+ url := request.Request.URL
+ if strings.HasSuffix(url.String(), "/") {
+ objectKey = objectKey + "/"
+ }
+ log.Infof("received request: PUT object, objectkey=%s, bucketName=%s\n:",
+ objectKey, bucketName)
+
+ //var authType = signature.GetRequestAuthType(r)
+ var err error
+ if !isValidObjectName(objectKey) {
+ WriteErrorResponse(response, request, s3error.ErrInvalidObjectName)
+ return
+ }
+
+ // get size
+ size, err := getSize(request, response)
+ if err != nil {
+ return
+ }
+ if size == -1 {
+ WriteErrorResponse(response, request, s3error.ErrMissingContentLength)
+ return
+ }
+
+ // maximum Upload size for objects in a single operation
+ if isMaxObjectSize(size) {
+ WriteErrorResponse(response, request, s3error.ErrEntityTooLarge)
+ return
+ }
+
+ // Save metadata.
+ metadata := extractMetadataFromHeader(request)
+ // Get Content-Md5 sent by client and verify if valid
+ if _, ok := request.Request.Header["Content-Md5"]; !ok {
+ metadata["md5Sum"] = ""
+ } else {
+ if len(request.Request.Header.Get("Content-Md5")) == 0 {
+ log.Infoln("Content Md5 is null!")
+ WriteErrorResponse(response, request, s3error.ErrInvalidDigest)
+ return
+ }
+ md5Bytes, err := checkValidMD5(request.Request.Header.Get("Content-Md5"))
+ if err != nil {
+ log.Infoln("Content Md5 is invalid!")
+ WriteErrorResponse(response, request, s3error.ErrInvalidDigest)
+ return
+ } else {
+ metadata["md5Sum"] = hex.EncodeToString(md5Bytes)
+ }
+ }
+
+ acl, err := getAclFromHeader(request)
+ if err != nil {
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ // check if specific bucket exist
+ ctx := common.InitCtxWithAuthInfo(request)
+ bucketMeta, err := s.getBucketMeta(ctx, bucketName)
+ if err != nil {
+ log.Errorln("failed to get bucket meta. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+ //log.Logf("bucket, acl:%f", bucketMeta.Acl.CannedAcl)
+ location := bucketMeta.DefaultLocation
+ if backendName != "" {
+ // check if backend exist
+ if s.isBackendExist(ctx, backendName) == false {
+ WriteErrorResponse(response, request, s3error.ErrGetBackendFailed)
+ return
+ }
+ location = backendName
+ }
+
+ var limitedDataReader io.Reader
+ if size > 0 { // request.ContentLength is -1 if length is unknown
+ limitedDataReader = io.LimitReader(request.Request.Body, size)
+ } else {
+ limitedDataReader = request.Request.Body
+ }
+
+ buf := make([]byte, ChunkSize)
+ eof := false
+ stream, err := s.s3Client.PutObject(ctx)
+ defer stream.Close()
+ obj := pb.PutObjectRequest{
+ BucketName: bucketName,
+ ObjectKey: objectKey,
+ Acl: &pb.Acl{CannedAcl: acl.CannedAcl},
+ Attrs: metadata,
+ Location: location,
+ Size: size,
+ }
+ err = stream.SendMsg(&obj)
+ if err != nil {
+ WriteErrorResponse(response, request, s3error.ErrInternalError)
+ return
+ }
+ for !eof {
+ n, err := limitedDataReader.Read(buf)
+ if err != nil && err != io.EOF {
+ log.Errorf("read error:%v\n", err)
+ break
+ }
+ if err == io.EOF {
+ log.Debugln("finished read")
+ eof = true
+ }
+ err = stream.Send(&s3.PutDataStream{Data: buf[:n]})
+
+ if err != nil {
+ log.Infof("stream send error: %v\n", err)
+ break
+ }
+ }
+
+ // if read or send data failed, then close stream and return error
+ if !eof {
+ WriteErrorResponse(response, request, s3error.ErrInternalError)
+ return
+ }
+
+ // TODO: is this the right way to get response?
+ rsp := &s3.PutObjectResponse{}
+ err = stream.RecvMsg(rsp)
+ if err != nil {
+ log.Infof("stream receive message failed:%v\n", err)
+ WriteErrorResponse(response, request, s3error.ErrInternalError)
+ }
+
+ log.Info("PUT object successfully.")
+ WriteSuccessResponse(response, nil)
+}
+
+func getSize(request *restful.Request, response *restful.Response) (int64, error) {
+ // get content-length
+ contentLenght := request.HeaderParameter(common.REQUEST_HEADER_CONTENT_LENGTH)
+ size, err := strconv.ParseInt(contentLenght, 10, 64)
+ if err != nil {
+ log.Infof("parse contentLenght[%s] failed, err:%v\n", contentLenght, err)
+ WriteErrorResponse(response, request, s3error.ErrMissingContentLength)
+ return 0, err
+ }
+
+ log.Infof("object size is %v\n", size)
+
+ if size > common.MaxObjectSize {
+ log.Infof("invalid contentLenght:%s\n", contentLenght)
+ errMsg := fmt.Sprintf("invalid contentLenght[%s], it should be less than %d and more than 0",
+ contentLenght, common.MaxObjectSize)
+ err := errors.New(errMsg)
+ WriteErrorResponse(response, request, err)
+ return size, err
+ }
+
+ return size, nil
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+)
+
+func (s *APIService) RouteBucketDelete(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "bucket:delete") {
+ return
+ }
+ if IsQuery(request, "acl") {
+ //TODO
+ } else if IsQuery(request, "versioning") {
+ //TODO
+ } else if IsQuery(request, "website") {
+ //TODO
+ } else if IsQuery(request, "cors") {
+ //TODO
+
+ } else if IsQuery(request, "replication") {
+ //TODO
+
+ } else if IsQuery(request, "lifecycle") {
+ s.BucketLifecycleDelete(request, response)
+
+ } else {
+ s.BucketDelete(request, response)
+ }
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+)
+
+func (s *APIService) RouteBucketGet(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "bucket:get") {
+ return
+ }
+ if IsQuery(request, "acl") {
+ s.BucketAclGet(request, response)
+ } else if IsQuery(request, "uploads") {
+ s.ListBucketUploadRecords(request, response)
+ } else if IsQuery(request, "versioning") {
+ //TODO
+ } else if IsQuery(request, "website") {
+ //TODO
+ } else if IsQuery(request, "cors") {
+ //TODO
+
+ } else if IsQuery(request, "replication") {
+ //TODO
+
+ } else if IsQuery(request, "lifecycle") {
+ s.BucketLifecycleGet(request, response)
+
+ } else {
+ s.BucketGet(request, response)
+ }
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+)
+
+func (s *APIService) RouteBucketHead(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "bucket:get") {
+ return
+ }
+
+ s.HeadBucket(request, response)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "errors"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+ "net/http"
+)
+
+func (s *APIService) RouteBucketPut(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "bucket:put") {
+ response.WriteError(http.StatusMethodNotAllowed, errors.New("authorize failed"))
+ return
+ }
+
+ if IsQuery(request, "acl") {
+ s.BucketAclPut(request, response)
+ } else if IsQuery(request, "versioning") {
+ s.BucketVersioningPut(request, response)
+ } else if IsQuery(request, "website") {
+ //TODO
+ } else if IsQuery(request, "cors") {
+ //TODO
+
+ } else if IsQuery(request, "replication") {
+ //TODO
+
+ } else if IsQuery(request, "lifecycle") {
+ s.BucketLifecyclePut(request, response)
+ } else if IsQuery(request, "DefaultEncryption") {
+ s.BucketSSEPut(request, response)
+ } else {
+ s.BucketPut(request, response)
+ }
+}
+
+
+
package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+)
+
+func (s *APIService) RouteObjectDelete(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "object:delete") {
+ return
+ }
+ if IsQuery(request, "uploadId") {
+ s.AbortMultipartUpload(request, response)
+ } else {
+ s.ObjectDelete(request, response)
+ }
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+)
+
+func (s *APIService) RouteObjectGet(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "object:get") {
+ return
+ }
+
+ if IsQuery(request, "acl") {
+ s.ObjectAclGet(request, response)
+ } else if IsQuery(request, "uploadId") {
+ s.ListObjectParts(request, response)
+ } else {
+ s.ObjectGet(request, response)
+ }
+}
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+)
+
+func (s *APIService) RouteObjectHead(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "object:delete") {
+ return
+ }
+
+ s.HeadObject(request, response)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "regexp"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+)
+
+func (s *APIService) RouteObjectPost(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "object:post") {
+ return
+ }
+ if IsQuery(request, "uploads") {
+ s.MultiPartUploadInit(request, response)
+ } else if IsQuery(request, "uploadId") {
+ s.CompleteMultipartUpload(request, response)
+ } else {
+ // check whether it is the post object operation.
+ contentType := request.HeaderParameter(common.REQUEST_HEADER_CONTENT_TYPE)
+ objectPostValidate := regexp.MustCompile("multipart/form-data*")
+ if contentType != "" && objectPostValidate.MatchString(contentType) {
+ s.ObjectPost(request, response)
+ return
+ }
+ }
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/policy"
+)
+
+func (s *APIService) RouteObjectPut(request *restful.Request, response *restful.Response) {
+ if !policy.Authorize(request, response, "object:put") {
+ return
+ }
+
+ if IsQuery(request, "acl") {
+ s.ObjectAclPut(request, response)
+ } else if IsQuery(request, "tagging") {
+ //TODO
+ } else if IsQuery(request, "uploads") {
+ s.MultiPartUploadInit(request, response)
+ } else if IsQuery(request, "partNumber") && IsQuery(request, "uploadId") && HasHeader(request, "x-amz-copy-source") {
+ s.ObjectPartCopy(request, response)
+ } else if IsQuery(request, "partNumber") && IsQuery(request, "uploadId") {
+ s.UploadPart(request, response)
+ } else if IsQuery(request, "uploadId") {
+ s.CompleteMultipartUpload(request, response)
+ } else if HasHeader(request, "x-amz-copy-source") {
+ s.ObjectCopy(request, response)
+ } else {
+ s.ObjectPut(request, response)
+ }
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "github.com/emicklei/go-restful"
+ "github.com/micro/go-micro/client"
+)
+
+//RegisterRouter - route request to appropriate method
+func RegisterRouter(ws *restful.WebService) {
+ handler := NewAPIService(client.DefaultClient)
+ ws.Route(ws.GET("/").To(handler.ListBuckets)).Doc("Return list of buckets for the user")
+ ws.Route(ws.GET("/storageClasses").To(handler.GetStorageClasses)).Doc("Return supported storage classes.")
+ ws.Route(ws.PUT("/{bucketName}").To(handler.RouteBucketPut)).Doc("Create bucket for the user")
+ //ws.Route(ws.HEAD("/s3/{bucketName}").To(handler.BucketHead)).Doc("Determine if bucket exists and if user has permission to access it")
+ ws.Route(ws.GET("/{bucketName}").To(handler.RouteBucketGet)).Doc("Return list of objects in bucket")
+ ws.Route(ws.DELETE("/{bucketName}").To(handler.RouteBucketDelete)).Doc("Delete bucket")
+ ws.Route(ws.HEAD("/{bucketName}").To(handler.RouteBucketHead)).Doc("Head bucket")
+
+ ws.Route(ws.PUT("/{bucketName}/{objectKey:*}").To(handler.RouteObjectPut)).Doc("Put object")
+ ws.Route(ws.DELETE("/{bucketName}/{objectKey:*}").To(handler.RouteObjectDelete)).Doc("Delete object")
+ ws.Route(ws.GET("/{bucketName}/{objectKey:*}").To(handler.RouteObjectGet)).Doc("Download object")
+ ws.Route(ws.DELETE("/{bucketName}/{objectKey:*}").To(handler.RouteObjectDelete)).Doc("AbortMultipartUpload")
+ ws.Route(ws.HEAD("/{bucketName}/{objectKey:*}").To(handler.RouteObjectHead)).Doc("Head object")
+ ws.Route(ws.POST("/{bucketName}/{objectKey:*}").To(handler.RouteObjectPost)).Doc("Post object")
+ ws.Route(ws.POST("/{bucketName}").To(handler.RouteObjectPost)).Doc("Post object")
+
+ //Router for PUT and GET bucket lifecycle
+ ws.Route(ws.PUT("/{bucketName}/?lifecycle").To(handler.RouteBucketPut)).Doc("Create lifecycle configuration for the bucket")
+ ws.Route(ws.GET("/{bucketName}/?lifecycle").To(handler.RouteBucketGet)).Doc("Get lifecycle configuration from the bucket")
+ ws.Route(ws.DELETE("/{bucketName}/?lifecycle").To(handler.RouteBucketDelete)).Doc("Delete lifecycle configuration from the bucket")
+
+ ws.Route(ws.PUT("/{bucketName}/?versioning").To(handler.RouteBucketPut)).Doc("Create Versioning configuration for the bucket")
+
+ // router for SSE
+ ws.Route(ws.PUT("/{bucketName}/?DefaultEncryption").To(handler.RouteBucketPut)).Doc("Set default encryption on bucket")
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "context"
+ "io"
+ "io/ioutil"
+ "math"
+
+ "github.com/emicklei/go-restful"
+ "github.com/micro/go-micro/client"
+ "github.com/opensds/multi-cloud/backend/proto"
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ s3Service = "s3"
+ backendService = "backend"
+)
+
+type APIService struct {
+ s3Client s3.S3Service
+ backendClient backend.BackendService
+}
+
+func NewAPIService(c client.Client) *APIService {
+ return &APIService{
+ s3Client: s3.NewS3Service(s3Service, c),
+ backendClient: backend.NewBackendService(backendService, c),
+ }
+}
+
+func IsQuery(request *restful.Request, name string) bool {
+ params := request.Request.URL.Query()
+ if params == nil {
+ return false
+ }
+ if _, ok := params[name]; ok {
+ return true
+ }
+ return false
+}
+func HasHeader(request *restful.Request, name string) bool {
+ param := request.HeaderParameter(name)
+ if param == "" {
+ return false
+ }
+ return true
+}
+
+func ReadBody(r *restful.Request) []byte {
+ var reader io.Reader = r.Request.Body
+ b, e := ioutil.ReadAll(reader)
+ if e != nil {
+ return nil
+ }
+ return b
+}
+
+func (s *APIService) getBucketMeta(ctx context.Context, bucketName string) (*s3.Bucket, error) {
+ rsp, err := s.s3Client.GetBucket(ctx, &s3.Bucket{Name: bucketName})
+ // according to gRPC framework work mechanism, if gRPC return error, then no response package can be received by
+ // gRPC client, so in our codes, gRPC server will return nil and set error code to reponse package while business
+ // error happens, and if gRPC client received error, that means some exception happened for gRPC itself.
+ if err == nil {
+ if rsp.GetErrorCode() != int32(ErrNoErr) {
+ err = S3ErrorCode(rsp.GetErrorCode())
+ }
+ }
+ if err != nil {
+ log.Infof("get bucket meta data[bucket=%s] failed, err=%v\n", bucketName, err)
+ return nil, err
+ }
+
+ return rsp.BucketMeta, nil
+}
+
+func (s *APIService) getObjectMeta(ctx context.Context, bucketName, objectName, versiongId string) (*s3.Object, error) {
+ rsp, err := s.s3Client.GetObjectMeta(ctx, &s3.Object{BucketName: bucketName, ObjectKey: objectName, VersionId: versiongId})
+ // according to gRPC framework work mechanism, if gRPC return error, then no response package can be received by
+ // gRPC client, so in our codes, gRPC server will return nil and set error code to reponse package while business
+ // error happens, and if gRPC client received error, that means some exception happened for gRPC itself.
+ if err == nil {
+ if rsp.GetErrorCode() != int32(ErrNoErr) {
+ err = S3ErrorCode(rsp.GetErrorCode())
+ }
+ }
+ if err != nil {
+ log.Infof("get object meta data[bucket=%s,key=%s] failed, err=%v\n", bucketName, objectName, err)
+ return nil, err
+ }
+
+ return rsp.Object, nil
+}
+
+func (s *APIService) isBackendExist(ctx context.Context, backendName string) bool {
+ flag := false
+
+ backendRep, backendErr := s.backendClient.ListBackend(ctx, &backendpb.ListBackendRequest{
+ Offset: 0,
+ Limit: math.MaxInt32,
+ Filter: map[string]string{"name": backendName}})
+ log.Infof("backendErr is %v:", backendErr)
+ if backendErr != nil {
+ log.Infof("Get backend[name=%s] failed.", backendName)
+ } else {
+ log.Infof("backendRep=%+v\n", backendRep)
+ if len(backendRep.Backends) > 0 {
+ log.Infof("backend[name=%s] exist.", backendName)
+ flag = true
+ }
+ }
+
+ return flag
+}
+
+func HandleS3Error(response *restful.Response, request *restful.Request, err error, errCode int32) error {
+ if err != nil {
+ WriteErrorResponse(response, request, err)
+ return err
+ }
+ if errCode != int32(ErrNoErr) {
+ err := S3ErrorCode(errCode)
+ WriteErrorResponse(response, request, err)
+ return err
+ }
+
+ return nil
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+ "encoding/hex"
+ "io"
+ "strconv"
+
+ "github.com/emicklei/go-restful"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/s3/error"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *APIService) UploadPart(request *restful.Request, response *restful.Response) {
+ bucketName := request.PathParameter("bucketName")
+ objectKey := request.PathParameter("objectKey")
+
+ var incomingMd5 string
+ // get Content-Md5 sent by client and verify if valid
+ md5Bytes, err := checkValidMD5(request.HeaderParameter(common.REQUEST_HEADER_CONTENT_MD5))
+ if err != nil {
+ incomingMd5 = ""
+ } else {
+ incomingMd5 = hex.EncodeToString(md5Bytes)
+ }
+
+ size := request.Request.ContentLength
+ /// maximum Upload size for multipart objects in a single operation
+ if isMaxObjectSize(size) {
+ log.Errorf("the size of object to upload is too large.")
+ WriteErrorResponse(response, request, ErrEntityTooLarge)
+ return
+ }
+ log.Infoln("uploadpart size:", size)
+
+ uploadID := request.QueryParameter("uploadId")
+ partIDString := request.QueryParameter("partNumber")
+ partID, err := strconv.Atoi(partIDString)
+ if err != nil {
+ log.Errorf("failed to convert part id string to integer")
+ WriteErrorResponse(response, request, ErrInvalidPart)
+ return
+ }
+ // check partID with maximum part ID for multipart objects
+ if isMaxPartID(partID) {
+ log.Errorf("the part id is invalid.")
+ WriteErrorResponse(response, request, ErrInvalidMaxParts)
+ return
+ }
+
+ ctx := common.InitCtxWithAuthInfo(request)
+ stream, err := s.s3Client.UploadPart(ctx)
+ defer stream.Close()
+
+ uploadRequest := pb.UploadPartRequest{
+ BucketName: bucketName,
+ ObjectKey: objectKey,
+ UploadId: uploadID,
+ PartId: int32(partID),
+ Size: size,
+ Md5Hex: incomingMd5,
+ }
+ err = stream.SendMsg(&uploadRequest)
+ if err != nil {
+ log.Errorln("failed send upload request msg. err:", err)
+ WriteErrorResponse(response, request, err)
+ return
+ }
+
+ var limitedDataReader io.Reader
+ if size > 0 { // request.ContentLength is -1 if length is unknown
+ limitedDataReader = io.LimitReader(request.Request.Body, size)
+ } else {
+ limitedDataReader = request.Request.Body
+ }
+ buf := make([]byte, ChunkSize)
+ eof := false
+ for !eof {
+ n, err := limitedDataReader.Read(buf)
+ if err != nil && err != io.EOF {
+ log.Errorf("read error:%v\n", err)
+ break
+ }
+ if err == io.EOF {
+ log.Debugln("finished read")
+ eof = true
+ }
+
+ err = stream.Send(&pb.PutDataStream{Data: buf[:n]})
+ if err != nil {
+ log.Infof("stream send error: %v\n", err)
+ break
+ }
+ }
+
+ result := pb.UploadPartResponse{}
+ err = stream.RecvMsg(&result)
+ if HandleS3Error(response, request, err, result.GetErrorCode()) != nil {
+ log.Errorln("unable to recv message. err:%v, errcode:%v", err, result.ErrorCode)
+ return
+ }
+
+ if result.ETag != "" {
+ response.Header()["ETag"] = []string{"\"" + result.ETag + "\""}
+ }
+
+ WriteSuccessResponse(response, nil)
+ log.Info("Uploadpart successfully.")
+}
+
+
+
/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+ "encoding/base64"
+ "encoding/xml"
+ "errors"
+ "io"
+ "net/http"
+ "regexp"
+ "strings"
+)
+
+// xmlDecoder provide decoded value in xml.
+func xmlDecoder(body io.Reader, v interface{}) error {
+ d := xml.NewDecoder(body)
+ return d.Decode(v)
+}
+
+// checkValidMD5 - verify if valid md5, returns md5 in bytes.
+func checkValidMD5(md5 string) ([]byte, error) {
+ return base64.StdEncoding.DecodeString(strings.TrimSpace(md5))
+}
+
+/// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
+const (
+ // maximum object size per PUT request is 5GiB
+ maxObjectSize = 1024 * 1024 * 1024 * 5
+ // minimum Part size for multipart upload is 5MB
+ minPartSize = 1024 * 1024 * 5
+ // maximum Part ID for multipart upload is 10000 (Acceptable values range from 1 to 10000 inclusive)
+ maxPartID = 10000
+)
+
+// isMaxObjectSize - verify if max object size
+func isMaxObjectSize(size int64) bool {
+ return size > maxObjectSize
+}
+
+// Check if part size is more than or equal to minimum allowed size.
+func isMinAllowedPartSize(size int64) bool {
+ return size >= minPartSize
+}
+
+// isMaxPartNumber - Check if part ID is greater than the maximum allowed ID.
+func isMaxPartID(partID int) bool {
+ return partID > maxPartID
+}
+
+func contains(stringList []string, element string) bool {
+ for _, e := range stringList {
+ if e == element {
+ return true
+ }
+ }
+ return false
+}
+
+/*
+func requestIdFromContext(ctx context.Context) string {
+ if result, ok := ctx.Value(RequestContextKey).(RequestContext); ok {
+ return result.RequestId
+ }
+ return ""
+}*/
+
+// We support '.' with bucket names but we fallback to using path
+// style requests instead for such buckets.
+var (
+ validBucketName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9\.\-\_\:]{1,61}[A-Za-z0-9]$`)
+ validBucketNameStrict = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
+ ipAddress = regexp.MustCompile(`^(\d+\.){3}\d+$`)
+)
+
+// Common checker for both stricter and basic validation.
+func checkBucketNameCommon(bucketName string, strict bool) (err error) {
+ if strings.TrimSpace(bucketName) == "" {
+ return errors.New("Bucket name cannot be empty")
+ }
+ if len(bucketName) < 3 {
+ return errors.New("Bucket name cannot be smaller than 3 characters")
+ }
+ if len(bucketName) > 63 {
+ return errors.New("Bucket name cannot be greater than 63 characters")
+ }
+ if ipAddress.MatchString(bucketName) {
+ return errors.New("Bucket name cannot be an ip address")
+ }
+ if strings.Contains(bucketName, "..") {
+ return errors.New("Bucket name contains invalid characters")
+ }
+ if strict {
+ if !validBucketNameStrict.MatchString(bucketName) {
+ err = errors.New("Bucket name contains invalid characters")
+ }
+ return err
+ }
+ if !validBucketName.MatchString(bucketName) {
+ err = errors.New("Bucket name contains invalid characters")
+ }
+ return err
+}
+
+// CheckValidBucketName - checks if we have a valid input bucket name.
+func CheckValidBucketName(bucketName string) (err error) {
+ return checkBucketNameCommon(bucketName, false)
+}
+
+func xmlFormat(data interface{}) ([]byte, error) {
+ buffer, err := xml.Marshal(data)
+ if err != nil {
+ return nil, err
+ }
+ // add XML header
+ headerBytes := []byte(xml.Header)
+ output := append(headerBytes, buffer...)
+ return output, nil
+}
+
+func setXmlHeader(w http.ResponseWriter, body []byte) {
+ w.Header().Set("Content-Type", "application/xml")
+}
+
+// hasServerSideEncryptionHeader returns true if the given HTTP header
+// contains server-side-encryption.
+/*func hasServerSideEncryptionHeader(header http.Header) bool {
+ return crypto.S3.IsRequested(header) || crypto.SSEC.IsRequested(header)
+}*/
+
+
+
package obs
+
+import (
+ "fmt"
+ "net/url"
+ "sort"
+ "strings"
+ "time"
+
+ log "github.com/sirupsen/logrus"
+)
+
+func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, params map[string]string,
+ headers map[string][]string, expires int64) (requestUrl string, err error) {
+
+ requestUrl, canonicalizedUrl := obsClient.conf.formatUrls(bucketName, objectKey, params, true)
+ parsedRequestUrl, err := url.Parse(requestUrl)
+ if err != nil {
+ return "", err
+ }
+ encodeHeaders(headers)
+ hostName := parsedRequestUrl.Host
+
+ isV4 := obsClient.conf.signature == SignatureV4
+ prepareHostAndDate(headers, hostName, isV4)
+
+ if obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" {
+ log.Warn("No ak/sk provided, skip to construct authorization")
+ } else {
+ if obsClient.conf.securityProvider.securityToken != "" {
+ params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken
+ }
+
+ if isV4 {
+ date, _ := time.Parse(RFC1123_FORMAT, headers[HEADER_DATE_CAMEL][0])
+ delete(headers, HEADER_DATE_CAMEL)
+ shortDate := date.Format(SHORT_DATE_FORMAT)
+ longDate := date.Format(LONG_DATE_FORMAT)
+
+ signedHeaders, _headers := getSignedHeaders(headers)
+
+ credential, scope := getCredential(obsClient.conf.securityProvider.ak, obsClient.conf.region, shortDate)
+ params[PARAM_ALGORITHM_AMZ_CAMEL] = V4_HASH_PREFIX
+ params[PARAM_CREDENTIAL_AMZ_CAMEL] = credential
+ params[PARAM_DATE_AMZ_CAMEL] = longDate
+ params[PARAM_EXPIRES_AMZ_CAMEL] = Int64ToString(expires)
+ params[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = strings.Join(signedHeaders, ";")
+
+ requestUrl, canonicalizedUrl = obsClient.conf.formatUrls(bucketName, objectKey, params, true)
+ parsedRequestUrl, _ = url.Parse(requestUrl)
+ stringToSign := getV4StringToSign(method, canonicalizedUrl, parsedRequestUrl.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, signedHeaders, _headers)
+ signature := getSignature(stringToSign, obsClient.conf.securityProvider.sk, obsClient.conf.region, shortDate)
+
+ requestUrl += fmt.Sprintf("&%s=%s", PARAM_SIGNATURE_AMZ_CAMEL, UrlEncode(signature, false))
+
+ } else {
+ originDate := headers[HEADER_DATE_CAMEL][0]
+ date, _ := time.Parse(RFC1123_FORMAT, originDate)
+ expires += date.Unix()
+ headers[HEADER_DATE_CAMEL] = []string{Int64ToString(expires)}
+
+ stringToSign := getV2StringToSign(method, canonicalizedUrl, headers)
+ signature := UrlEncode(Base64Encode(HmacSha1([]byte(obsClient.conf.securityProvider.sk), []byte(stringToSign))), false)
+ if strings.Index(requestUrl, "?") < 0 {
+ requestUrl += "?"
+ } else {
+ requestUrl += "&"
+ }
+ delete(headers, HEADER_DATE_CAMEL)
+ requestUrl += fmt.Sprintf("AWSAccessKeyId=%s&Expires=%d&Signature=%s", UrlEncode(obsClient.conf.securityProvider.ak, false),
+ expires, signature)
+ }
+ }
+
+ return
+}
+
+func (obsClient ObsClient) doAuth(method, bucketName, objectKey string, params map[string]string,
+ headers map[string][]string, hostName string) (requestUrl string, err error) {
+
+ requestUrl, canonicalizedUrl := obsClient.conf.formatUrls(bucketName, objectKey, params, true)
+ parsedRequestUrl, err := url.Parse(requestUrl)
+ if err != nil {
+ return "", err
+ }
+ encodeHeaders(headers)
+
+ if hostName == "" {
+ hostName = parsedRequestUrl.Host
+ }
+
+ isV4 := obsClient.conf.signature == SignatureV4
+ prepareHostAndDate(headers, hostName, isV4)
+
+ if obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" {
+ log.Warn("No ak/sk provided, skip to construct authorization")
+ } else {
+ if obsClient.conf.securityProvider.securityToken != "" {
+ headers[HEADER_STS_TOKEN_AMZ] = []string{obsClient.conf.securityProvider.securityToken}
+ }
+ ak := obsClient.conf.securityProvider.ak
+ sk := obsClient.conf.securityProvider.sk
+ var authorization string
+ if isV4 {
+ headers[HEADER_CONTENT_SHA256_AMZ] = []string{EMPTY_CONTENT_SHA256}
+ ret := v4Auth(ak, sk, obsClient.conf.region, method, canonicalizedUrl, parsedRequestUrl.RawQuery, headers)
+ authorization = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"])
+ } else {
+ ret := v2Auth(ak, sk, method, canonicalizedUrl, headers)
+ authorization = fmt.Sprintf("%s %s:%s", V2_HASH_PREFIX, ak, ret["Signature"])
+ }
+ headers[HEADER_AUTH_CAMEL] = []string{authorization}
+ }
+
+ return
+}
+
+func prepareHostAndDate(headers map[string][]string, hostName string, isV4 bool) {
+ headers[HEADER_HOST_CAMEL] = []string{hostName}
+ if date, ok := headers[HEADER_DATE_AMZ]; ok {
+ flag := false
+ if len(date) == 1 {
+ if isV4 {
+ if t, err := time.Parse(LONG_DATE_FORMAT, date[0]); err == nil {
+ headers[HEADER_DATE_CAMEL] = []string{FormatUtcToRfc1123(t)}
+ flag = true
+ }
+ } else {
+ if strings.HasSuffix(date[0], "GMT") {
+ headers[HEADER_DATE_CAMEL] = []string{date[0]}
+ flag = true
+ }
+ }
+ }
+ if !flag {
+ delete(headers, HEADER_DATE_AMZ)
+ }
+ }
+ if _, ok := headers[HEADER_DATE_CAMEL]; !ok {
+ headers[HEADER_DATE_CAMEL] = []string{FormatUtcToRfc1123(time.Now().UTC())}
+ }
+}
+
+func encodeHeaders(headers map[string][]string) {
+ for key, values := range headers {
+ for index, value := range values {
+ values[index] = UrlEncode(value, true)
+ }
+ headers[key] = values
+ }
+}
+
+func attachHeaders(headers map[string][]string) string {
+ length := len(headers)
+ _headers := make(map[string][]string, length)
+ keys := make([]string, 0, length)
+
+ for key, value := range headers {
+ _key := strings.ToLower(strings.TrimSpace(key))
+ if _key != "" {
+ if _key == "content-md5" || _key == "content-type" || _key == "date" || strings.HasPrefix(_key, HEADER_PREFIX) {
+ keys = append(keys, _key)
+ _headers[_key] = value
+ }
+ } else {
+ delete(headers, key)
+ }
+ }
+
+ for _, interestedHeader := range interested_headers {
+ if _, ok := _headers[interestedHeader]; !ok {
+ _headers[interestedHeader] = []string{""}
+ keys = append(keys, interestedHeader)
+ }
+ }
+
+ if _, ok := _headers[HEADER_DATE_CAMEL]; ok {
+ if _, ok := _headers[HEADER_DATE_AMZ]; ok {
+ _headers[HEADER_DATE_CAMEL] = []string{""}
+ } else if _, ok := headers[PARAM_DATE_AMZ_CAMEL]; ok {
+ _headers[HEADER_DATE_CAMEL] = []string{""}
+ }
+ } else if _, ok := _headers[strings.ToLower(HEADER_DATE_CAMEL)]; ok {
+ if _, ok := _headers[HEADER_DATE_AMZ]; ok {
+ _headers[HEADER_DATE_CAMEL] = []string{""}
+ } else if _, ok := headers[PARAM_DATE_AMZ_CAMEL]; ok {
+ _headers[HEADER_DATE_CAMEL] = []string{""}
+ }
+ }
+
+ sort.Strings(keys)
+
+ stringToSign := make([]string, 0, len(keys))
+ for _, key := range keys {
+ var value string
+ if strings.HasPrefix(key, HEADER_PREFIX) {
+ if strings.HasPrefix(key, HEADER_PREFIX_META) {
+ for index, v := range _headers[key] {
+ value += strings.TrimSpace(v)
+ if index != len(_headers[key])-1 {
+ value += ","
+ }
+ }
+ } else {
+ value = strings.Join(_headers[key], ",")
+ }
+ value = fmt.Sprintf("%s:%s", key, value)
+ } else {
+ value = strings.Join(_headers[key], ",")
+ }
+ stringToSign = append(stringToSign, value)
+ }
+ return strings.Join(stringToSign, "\n")
+}
+
+func getV2StringToSign(method, canonicalizedUrl string, headers map[string][]string) string {
+ stringToSign := strings.Join([]string{method, "\n", attachHeaders(headers), "\n", canonicalizedUrl}, "")
+ log.Debug("The v2 auth stringToSign:\n%s", stringToSign)
+ return stringToSign
+}
+
+func v2Auth(ak, sk, method, canonicalizedUrl string, headers map[string][]string) map[string]string {
+ stringToSign := getV2StringToSign(method, canonicalizedUrl, headers)
+ return map[string]string{"Signature": Base64Encode(HmacSha1([]byte(sk), []byte(stringToSign)))}
+}
+
+func getScope(region, shortDate string) string {
+ return fmt.Sprintf("%s/%s/%s/%s", shortDate, region, V4_SERVICE_NAME, V4_SERVICE_SUFFIX)
+}
+
+func getCredential(ak, region, shortDate string) (string, string) {
+ scope := getScope(region, shortDate)
+ return fmt.Sprintf("%s/%s", ak, scope), scope
+}
+
+func getV4StringToSign(method, canonicalizedUrl, queryUrl, scope, longDate, payload string, signedHeaders []string, headers map[string][]string) string {
+ canonicalRequest := make([]string, 0, 10+len(signedHeaders)*4)
+ canonicalRequest = append(canonicalRequest, method)
+ canonicalRequest = append(canonicalRequest, "\n")
+ canonicalRequest = append(canonicalRequest, canonicalizedUrl)
+ canonicalRequest = append(canonicalRequest, "\n")
+ canonicalRequest = append(canonicalRequest, queryUrl)
+ canonicalRequest = append(canonicalRequest, "\n")
+
+ for _, signedHeader := range signedHeaders {
+ values, _ := headers[signedHeader]
+ for _, value := range values {
+ canonicalRequest = append(canonicalRequest, signedHeader)
+ canonicalRequest = append(canonicalRequest, ":")
+ canonicalRequest = append(canonicalRequest, value)
+ canonicalRequest = append(canonicalRequest, "\n")
+ }
+ }
+ canonicalRequest = append(canonicalRequest, "\n")
+ canonicalRequest = append(canonicalRequest, strings.Join(signedHeaders, ";"))
+ canonicalRequest = append(canonicalRequest, "\n")
+ canonicalRequest = append(canonicalRequest, payload)
+
+ _canonicalRequest := strings.Join(canonicalRequest, "")
+ log.Debug("The v4 auth canonicalRequest:\n%s", _canonicalRequest)
+
+ stringToSign := make([]string, 0, 7)
+ stringToSign = append(stringToSign, V4_HASH_PREFIX)
+ stringToSign = append(stringToSign, "\n")
+ stringToSign = append(stringToSign, longDate)
+ stringToSign = append(stringToSign, "\n")
+ stringToSign = append(stringToSign, scope)
+ stringToSign = append(stringToSign, "\n")
+ stringToSign = append(stringToSign, HexSha256([]byte(_canonicalRequest)))
+
+ _stringToSign := strings.Join(stringToSign, "")
+
+ log.Debug("The v4 auth stringToSign:\n%s", _stringToSign)
+ return _stringToSign
+}
+
+func getSignedHeaders(headers map[string][]string) ([]string, map[string][]string) {
+ length := len(headers)
+ _headers := make(map[string][]string, length)
+ signedHeaders := make([]string, 0, length)
+ for key, value := range headers {
+ _key := strings.ToLower(strings.TrimSpace(key))
+ if _key != "" {
+ signedHeaders = append(signedHeaders, _key)
+ _headers[_key] = value
+ } else {
+ delete(headers, key)
+ }
+ }
+ sort.Strings(signedHeaders)
+ return signedHeaders, _headers
+}
+
+func getSignature(stringToSign, sk, region, shortDate string) string {
+ key := HmacSha256([]byte(V4_HASH_PRE+sk), []byte(shortDate))
+ key = HmacSha256(key, []byte(region))
+ key = HmacSha256(key, []byte(V4_SERVICE_NAME))
+ key = HmacSha256(key, []byte(V4_SERVICE_SUFFIX))
+ return Hex(HmacSha256(key, []byte(stringToSign)))
+}
+
+func V4Auth(ak, sk, region, method, canonicalizedUrl, queryUrl string, headers map[string][]string) map[string]string {
+ return v4Auth(ak, sk, region, method, canonicalizedUrl, queryUrl, headers)
+}
+
+func v4Auth(ak, sk, region, method, canonicalizedUrl, queryUrl string, headers map[string][]string) map[string]string {
+ var t time.Time
+
+ if val, ok := headers[HEADER_DATE_AMZ]; ok {
+ var err error
+ t, err = time.Parse(LONG_DATE_FORMAT, val[0])
+ if err != nil {
+ t = time.Now().UTC()
+ }
+ } else if val, ok := headers[PARAM_DATE_AMZ_CAMEL]; ok {
+ var err error
+ t, err = time.Parse(LONG_DATE_FORMAT, val[0])
+ if err != nil {
+ t = time.Now().UTC()
+ }
+ } else if val, ok := headers[HEADER_DATE_CAMEL]; ok {
+ var err error
+ t, err = time.Parse(RFC1123_FORMAT, val[0])
+ if err != nil {
+ t = time.Now().UTC()
+ }
+ } else if val, ok := headers[strings.ToLower(HEADER_DATE_CAMEL)]; ok {
+ var err error
+ t, err = time.Parse(RFC1123_FORMAT, val[0])
+ if err != nil {
+ t = time.Now().UTC()
+ }
+ } else {
+ t = time.Now().UTC()
+ }
+ shortDate := t.Format(SHORT_DATE_FORMAT)
+ longDate := t.Format(LONG_DATE_FORMAT)
+
+ signedHeaders, _headers := getSignedHeaders(headers)
+
+ credential, scope := getCredential(ak, region, shortDate)
+
+ stringToSign := getV4StringToSign(method, canonicalizedUrl, queryUrl, scope, longDate, EMPTY_CONTENT_SHA256, signedHeaders, _headers)
+
+ signature := getSignature(stringToSign, sk, region, shortDate)
+
+ ret := make(map[string]string, 3)
+ ret["Credential"] = credential
+ ret["SignedHeaders"] = strings.Join(signedHeaders, ";")
+ ret["Signature"] = signature
+ return ret
+}
+
+
+
package obs
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "sort"
+ "strings"
+
+ log "github.com/sirupsen/logrus"
+)
+
+type ObsClient struct {
+ conf *config
+ httpClient *http.Client
+ transport *http.Transport
+}
+
+func New(ak, sk, endpoint string, configurers ...configurer) (*ObsClient, error) {
+ conf := &config{securityProvider: &securityProvider{ak: ak, sk: sk}, endpoint: endpoint}
+ conf.maxRetryCount = -1
+ for _, configurer := range configurers {
+ configurer(conf)
+ }
+
+ if err := conf.initConfigWithDefault(); err != nil {
+ return nil, err
+ }
+
+ transport, err := conf.getTransport()
+ if err != nil {
+ return nil, err
+ }
+
+ info := make([]string, 3)
+ info[0] = fmt.Sprintf("[OBS SDK Version=%s", obs_sdk_version)
+ info[1] = fmt.Sprintf("Endpoint=%s", conf.endpoint)
+ accessMode := "Virtual Hosting"
+ if conf.pathStyle {
+ accessMode = "Path"
+ }
+ info[2] = fmt.Sprintf("Access Mode=%s]", accessMode)
+ log.Warn(strings.Join(info, "];["))
+
+ log.Debug("Create obsclient with config:\n%s\n", conf)
+ obsClient := &ObsClient{conf: conf, httpClient: &http.Client{Transport: transport, CheckRedirect: checkRedirectFunc}, transport: transport}
+ return obsClient, nil
+}
+
+func (obsClient ObsClient) Refresh(ak, sk, securityToken string) {
+ sp := &securityProvider{ak: strings.TrimSpace(ak), sk: strings.TrimSpace(sk), securityToken: strings.TrimSpace(securityToken)}
+ obsClient.conf.securityProvider = sp
+}
+
+func (obsClient ObsClient) Close() {
+ obsClient.transport.CloseIdleConnections()
+ obsClient.transport = nil
+ obsClient.httpClient = nil
+ obsClient.conf = nil
+}
+
+func (obsClient ObsClient) ListBuckets(input *ListBucketsInput) (output *ListBucketsOutput, err error) {
+ if input == nil {
+ input = &ListBucketsInput{}
+ }
+ output = &ListBucketsOutput{}
+ err = obsClient.doActionWithoutBucket("ListBuckets", HTTP_GET, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) CreateBucket(input *CreateBucketInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("CreateBucketInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("CreateBucket", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucket(bucketName string) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("DeleteBucket", HTTP_DELETE, bucketName, defaultSerializable, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketStoragePolicy(input *SetBucketStoragePolicyInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketStoragePolicyInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketStoragePolicy", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketStoragePolicy(bucketName string) (output *GetBucketStoragePolicyOutput, err error) {
+ output = &GetBucketStoragePolicyOutput{}
+ err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStoragePolicy), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) ListObjects(input *ListObjectsInput) (output *ListObjectsOutput, err error) {
+ if input == nil {
+ return nil, errors.New("ListObjectsInput is nil")
+ }
+ output = &ListObjectsOutput{}
+ err = obsClient.doActionWithBucket("ListObjects", HTTP_GET, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok {
+ output.Location = location[0]
+ }
+ }
+ return
+}
+
+func (obsClient ObsClient) ListVersions(input *ListVersionsInput) (output *ListVersionsOutput, err error) {
+ if input == nil {
+ return nil, errors.New("ListVersionsInput is nil")
+ }
+ output = &ListVersionsOutput{}
+ err = obsClient.doActionWithBucket("ListVersions", HTTP_GET, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok {
+ output.Location = location[0]
+ }
+ }
+ return
+}
+
+func (obsClient ObsClient) ListMultipartUploads(input *ListMultipartUploadsInput) (output *ListMultipartUploadsOutput, err error) {
+ if input == nil {
+ return nil, errors.New("ListMultipartUploadsInput is nil")
+ }
+ output = &ListMultipartUploadsOutput{}
+ err = obsClient.doActionWithBucket("ListMultipartUploads", HTTP_GET, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketQuota(input *SetBucketQuotaInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketQuotaInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketQuota", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketQuota(bucketName string) (output *GetBucketQuotaOutput, err error) {
+ output = &GetBucketQuotaOutput{}
+ err = obsClient.doActionWithBucket("GetBucketQuota", HTTP_GET, bucketName, newSubResourceSerial(SubResourceQuota), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) HeadBucket(bucketName string) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("HeadBucket", HTTP_HEAD, bucketName, defaultSerializable, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketMetadata(input *GetBucketMetadataInput) (output *GetBucketMetadataOutput, err error) {
+ output = &GetBucketMetadataOutput{}
+ err = obsClient.doActionWithBucket("GetBucketMetadata", HTTP_HEAD, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ ParseGetBucketMetadataOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketStorageInfo(bucketName string) (output *GetBucketStorageInfoOutput, err error) {
+ output = &GetBucketStorageInfoOutput{}
+ err = obsClient.doActionWithBucket("GetBucketStorageInfo", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageInfo), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketLocation(bucketName string) (output *GetBucketLocationOutput, err error) {
+ output = &GetBucketLocationOutput{}
+ err = obsClient.doActionWithBucket("GetBucketLocation", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLocation), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketAcl(input *SetBucketAclInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketAclInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketAcl", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketAcl(bucketName string) (output *GetBucketAclOutput, err error) {
+ output = &GetBucketAclOutput{}
+ err = obsClient.doActionWithBucket("GetBucketAcl", HTTP_GET, bucketName, newSubResourceSerial(SubResourceAcl), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketPolicy(input *SetBucketPolicyInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketPolicy is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketPolicy", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketPolicy(bucketName string) (output *GetBucketPolicyOutput, err error) {
+ output = &GetBucketPolicyOutput{}
+ err = obsClient.doActionWithBucketV2("GetBucketPolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourcePolicy), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketPolicy(bucketName string) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("DeleteBucketPolicy", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourcePolicy), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketCors(input *SetBucketCorsInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketCorsInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketCors", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketCors(bucketName string) (output *GetBucketCorsOutput, err error) {
+ output = &GetBucketCorsOutput{}
+ err = obsClient.doActionWithBucket("GetBucketCors", HTTP_GET, bucketName, newSubResourceSerial(SubResourceCors), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketCors(bucketName string) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("DeleteBucketCors", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceCors), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketVersioning(input *SetBucketVersioningInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketVersioningInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketVersioning", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketVersioning(bucketName string) (output *GetBucketVersioningOutput, err error) {
+ output = &GetBucketVersioningOutput{}
+ err = obsClient.doActionWithBucket("GetBucketVersioning", HTTP_GET, bucketName, newSubResourceSerial(SubResourceVersioning), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketWebsiteConfiguration(input *SetBucketWebsiteConfigurationInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketWebsiteConfigurationInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketWebsiteConfiguration", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketWebsiteConfiguration(bucketName string) (output *GetBucketWebsiteConfigurationOutput, err error) {
+ output = &GetBucketWebsiteConfigurationOutput{}
+ err = obsClient.doActionWithBucket("GetBucketWebsiteConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceWebsite), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketWebsiteConfiguration(bucketName string) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("DeleteBucketWebsiteConfiguration", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceWebsite), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketLoggingConfiguration(input *SetBucketLoggingConfigurationInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketLoggingConfigurationInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketLoggingConfiguration", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketLoggingConfiguration(bucketName string) (output *GetBucketLoggingConfigurationOutput, err error) {
+ output = &GetBucketLoggingConfigurationOutput{}
+ err = obsClient.doActionWithBucket("GetBucketLoggingConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLogging), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketLifecycleConfiguration(input *SetBucketLifecycleConfigurationInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketLifecycleConfigurationInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketLifecycleConfiguration", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketLifecycleConfiguration(bucketName string) (output *GetBucketLifecycleConfigurationOutput, err error) {
+ output = &GetBucketLifecycleConfigurationOutput{}
+ err = obsClient.doActionWithBucket("GetBucketLifecycleConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLifecycle), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketLifecycleConfiguration(bucketName string) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("DeleteBucketLifecycleConfiguration", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceLifecycle), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketTagging(input *SetBucketTaggingInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketTaggingInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketTagging", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketTagging(bucketName string) (output *GetBucketTaggingOutput, err error) {
+ output = &GetBucketTaggingOutput{}
+ err = obsClient.doActionWithBucket("GetBucketTagging", HTTP_GET, bucketName, newSubResourceSerial(SubResourceTagging), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketTagging(bucketName string) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("DeleteBucketTagging", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceTagging), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketNotification(input *SetBucketNotificationInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetBucketNotificationInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucket("SetBucketNotification", HTTP_PUT, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketNotification(bucketName string) (output *GetBucketNotificationOutput, err error) {
+ output = &GetBucketNotificationOutput{}
+ err = obsClient.doActionWithBucket("GetBucketNotification", HTTP_GET, bucketName, newSubResourceSerial(SubResourceNotification), output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteObject(input *DeleteObjectInput) (output *DeleteObjectOutput, err error) {
+ if input == nil {
+ return nil, errors.New("DeleteObjectInput is nil")
+ }
+ output = &DeleteObjectOutput{}
+ err = obsClient.doActionWithBucketAndKey("DeleteObject", HTTP_DELETE, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ ParseDeleteObjectOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteObjects(input *DeleteObjectsInput) (output *DeleteObjectsOutput, err error) {
+ if input == nil {
+ return nil, errors.New("DeleteObjectsInput is nil")
+ }
+ output = &DeleteObjectsOutput{}
+ err = obsClient.doActionWithBucket("DeleteObjects", HTTP_POST, input.Bucket, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetObjectAcl(input *SetObjectAclInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("SetObjectAclInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucketAndKey("SetObjectAcl", HTTP_PUT, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetObjectAcl(input *GetObjectAclInput) (output *GetObjectAclOutput, err error) {
+ if input == nil {
+ return nil, errors.New("GetObjectAclInput is nil")
+ }
+ output = &GetObjectAclOutput{}
+ err = obsClient.doActionWithBucketAndKey("GetObjectAcl", HTTP_GET, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ if versionId, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok {
+ output.VersionId = versionId[0]
+ }
+ }
+ return
+}
+
+func (obsClient ObsClient) RestoreObject(input *RestoreObjectInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("RestoreObjectInput is nil")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucketAndKey("RestoreObject", HTTP_POST, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetObjectMetadata(input *GetObjectMetadataInput) (output *GetObjectMetadataOutput, err error) {
+ if input == nil {
+ return nil, errors.New("GetObjectMetadataInput is nil")
+ }
+ output = &GetObjectMetadataOutput{}
+ err = obsClient.doActionWithBucketAndKey("GetObjectMetadata", HTTP_HEAD, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ ParseGetObjectMetadataOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) GetObject(input *GetObjectInput) (output *GetObjectOutput, err error) {
+ if input == nil {
+ return nil, errors.New("GetObjectInput is nil")
+ }
+ output = &GetObjectOutput{}
+ err = obsClient.doActionWithBucketAndKey("GetObject", HTTP_GET, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ ParseGetObjectOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) PutObject(input *PutObjectInput) (output *PutObjectOutput, err error) {
+ if input == nil {
+ return nil, errors.New("PutObjectInput is nil")
+ }
+
+ if input.ContentType == "" && input.Key != "" {
+ if contentType, ok := mime_types[input.Key[strings.LastIndex(input.Key, ".")+1:]]; ok {
+ input.ContentType = contentType
+ }
+ }
+
+ output = &PutObjectOutput{}
+ var repeatable bool
+ if input.Body != nil {
+ _, repeatable = input.Body.(*strings.Reader)
+ if input.ContentLength > 0 {
+ input.Body = &readerWrapper{reader: input.Body, totalCount: input.ContentLength}
+ }
+ }
+ if repeatable {
+ err = obsClient.doActionWithBucketAndKey("PutObject", HTTP_PUT, input.Bucket, input.Key, input, output)
+ } else {
+ err = obsClient.doActionWithBucketAndKeyUnRepeatable("PutObject", HTTP_PUT, input.Bucket, input.Key, input, output)
+ }
+ if err != nil {
+ output = nil
+ } else {
+ ParsePutObjectOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) PutFile(input *PutFileInput) (output *PutObjectOutput, err error) {
+ if input == nil {
+ return nil, errors.New("PutFileInput is nil")
+ }
+
+ var body io.Reader
+ sourceFile := strings.TrimSpace(input.SourceFile)
+ if sourceFile != "" {
+ fd, err := os.Open(sourceFile)
+ if err != nil {
+ return nil, err
+ }
+ defer fd.Close()
+
+ stat, err := fd.Stat()
+ if err != nil {
+ return nil, err
+ }
+ fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile}
+ fileReaderWrapper.reader = fd
+ if input.ContentLength > 0 {
+ if input.ContentLength > stat.Size() {
+ input.ContentLength = stat.Size()
+ }
+ fileReaderWrapper.totalCount = input.ContentLength
+ } else {
+ fileReaderWrapper.totalCount = stat.Size()
+ }
+ body = fileReaderWrapper
+ }
+
+ _input := &PutObjectInput{}
+ _input.PutObjectBasicInput = input.PutObjectBasicInput
+ _input.Body = body
+
+ if _input.ContentType == "" && _input.Key != "" {
+ if contentType, ok := mime_types[_input.Key[strings.LastIndex(_input.Key, ".")+1:]]; ok {
+ _input.ContentType = contentType
+ } else if contentType, ok := mime_types[sourceFile[strings.LastIndex(sourceFile, ".")+1:]]; ok {
+ _input.ContentType = contentType
+ }
+ }
+
+ output = &PutObjectOutput{}
+ err = obsClient.doActionWithBucketAndKey("PutFile", HTTP_PUT, _input.Bucket, _input.Key, _input, output)
+ if err != nil {
+ output = nil
+ } else {
+ ParsePutObjectOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) CopyObject(input *CopyObjectInput) (output *CopyObjectOutput, err error) {
+ if input == nil {
+ return nil, errors.New("CopyObjectInput is nil")
+ }
+
+ if strings.TrimSpace(input.CopySourceBucket) == "" {
+ return nil, errors.New("Source bucket is empty")
+ }
+ if strings.TrimSpace(input.CopySourceKey) == "" {
+ return nil, errors.New("Source key is empty")
+ }
+
+ output = &CopyObjectOutput{}
+ err = obsClient.doActionWithBucketAndKey("CopyObject", HTTP_PUT, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ ParseCopyObjectOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) AbortMultipartUpload(input *AbortMultipartUploadInput) (output *BaseModel, err error) {
+ if input == nil {
+ return nil, errors.New("AbortMultipartUploadInput is nil")
+ }
+ if input.UploadId == "" {
+ return nil, errors.New("UploadId is empty")
+ }
+ output = &BaseModel{}
+ err = obsClient.doActionWithBucketAndKey("AbortMultipartUpload", HTTP_DELETE, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) InitiateMultipartUpload(input *InitiateMultipartUploadInput) (output *InitiateMultipartUploadOutput, err error) {
+ if input == nil {
+ return nil, errors.New("InitiateMultipartUploadInput is nil")
+ }
+
+ if input.ContentType == "" && input.Key != "" {
+ if contentType, ok := mime_types[input.Key[strings.LastIndex(input.Key, ".")+1:]]; ok {
+ input.ContentType = contentType
+ }
+ }
+
+ output = &InitiateMultipartUploadOutput{}
+ err = obsClient.doActionWithBucketAndKey("InitiateMultipartUpload", HTTP_POST, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ ParseInitiateMultipartUploadOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) UploadPart(input *UploadPartInput) (output *UploadPartOutput, err error) {
+ if input == nil {
+ return nil, errors.New("UploadPartInput is nil")
+ }
+
+ if input.UploadId == "" {
+ return nil, errors.New("UploadId is empty")
+ }
+
+ output = &UploadPartOutput{}
+ var repeatable bool
+ if input.Body != nil {
+ _, repeatable = input.Body.(*strings.Reader)
+ if input.PartSize > 0 {
+ input.Body = &readerWrapper{reader: input.Body, totalCount: input.PartSize}
+ }
+ } else if sourceFile := strings.TrimSpace(input.SourceFile); sourceFile != "" {
+ fd, err := os.Open(sourceFile)
+ if err != nil {
+ return nil, err
+ }
+ defer fd.Close()
+
+ stat, err := fd.Stat()
+ if err != nil {
+ return nil, err
+ }
+ fileSize := stat.Size()
+ fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile}
+ fileReaderWrapper.reader = fd
+
+ if input.Offset < 0 || input.Offset > fileSize {
+ input.Offset = 0
+ }
+
+ if input.PartSize <= 0 || input.PartSize > (fileSize-input.Offset) {
+ input.PartSize = fileSize - input.Offset
+ }
+ fileReaderWrapper.totalCount = input.PartSize
+ fd.Seek(input.Offset, 0)
+ input.Body = fileReaderWrapper
+ repeatable = true
+ }
+ if repeatable {
+ err = obsClient.doActionWithBucketAndKey("UploadPart", HTTP_PUT, input.Bucket, input.Key, input, output)
+ } else {
+ err = obsClient.doActionWithBucketAndKeyUnRepeatable("UploadPart", HTTP_PUT, input.Bucket, input.Key, input, output)
+ }
+ if err != nil {
+ output = nil
+ } else {
+ ParseUploadPartOutput(output)
+ output.PartNumber = input.PartNumber
+ }
+ return
+}
+
+func (obsClient ObsClient) CompleteMultipartUpload(input *CompleteMultipartUploadInput) (output *CompleteMultipartUploadOutput, err error) {
+ if input == nil {
+ return nil, errors.New("CompleteMultipartUploadInput is nil")
+ }
+
+ if input.UploadId == "" {
+ return nil, errors.New("UploadId is empty")
+ }
+
+ var parts partSlice = input.Parts
+ sort.Sort(parts)
+
+ output = &CompleteMultipartUploadOutput{}
+ err = obsClient.doActionWithBucketAndKey("CompleteMultipartUpload", HTTP_POST, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ ParseCompleteMultipartUploadOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) ListParts(input *ListPartsInput) (output *ListPartsOutput, err error) {
+ if input == nil {
+ return nil, errors.New("ListPartsInput is nil")
+ }
+ if input.UploadId == "" {
+ return nil, errors.New("UploadId is empty")
+ }
+ output = &ListPartsOutput{}
+ err = obsClient.doActionWithBucketAndKey("ListParts", HTTP_GET, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) CopyPart(input *CopyPartInput) (output *CopyPartOutput, err error) {
+ if input == nil {
+ return nil, errors.New("CopyPartInput is nil")
+ }
+ if input.UploadId == "" {
+ return nil, errors.New("UploadId is empty")
+ }
+ if strings.TrimSpace(input.CopySourceBucket) == "" {
+ return nil, errors.New("Source bucket is empty")
+ }
+ if strings.TrimSpace(input.CopySourceKey) == "" {
+ return nil, errors.New("Source key is empty")
+ }
+
+ output = &CopyPartOutput{}
+ err = obsClient.doActionWithBucketAndKey("CopyPart", HTTP_PUT, input.Bucket, input.Key, input, output)
+ if err != nil {
+ output = nil
+ } else {
+ ParseCopyPartOutput(output)
+ output.PartNumber = input.PartNumber
+ }
+ return
+}
+
+
+
package obs
+
+import (
+ "crypto/tls"
+ "crypto/x509"
+ "errors"
+ "fmt"
+ "net"
+ "net/http"
+ "net/url"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+)
+
+type securityProvider struct {
+ ak string
+ sk string
+ securityToken string
+}
+
+type urlHolder struct {
+ scheme string
+ host string
+ port int
+}
+
+type config struct {
+ securityProvider *securityProvider
+ urlHolder *urlHolder
+ endpoint string
+ signature SignatureType
+ pathStyle bool
+ region string
+ connectTimeout int
+ socketTimeout int
+ headerTimeout int
+ idleConnTimeout int
+ finalTimeout int
+ maxRetryCount int
+ proxyUrl string
+ maxConnsPerHost int
+ sslVerify bool
+ pemCerts []byte
+}
+
+func (conf config) String() string {
+ return fmt.Sprintf("[endpoint:%s, signature:%s, pathStyle:%v, region:%s"+
+ "\nconnectTimeout:%d, socketTimeout:%dheaderTimeout:%d, idleConnTimeout:%d"+
+ "\nmaxRetryCount:%d, maxConnsPerHost:%d, sslVerify:%v, proxyUrl:%s]",
+ conf.endpoint, conf.signature, conf.pathStyle, conf.region,
+ conf.connectTimeout, conf.socketTimeout, conf.headerTimeout, conf.idleConnTimeout,
+ conf.maxRetryCount, conf.maxConnsPerHost, conf.sslVerify, conf.proxyUrl,
+ )
+}
+
+type configurer func(conf *config)
+
+func WithSslVerify(sslVerify bool) configurer {
+ return WithSslVerifyAndPemCerts(sslVerify, nil)
+}
+
+func WithSslVerifyAndPemCerts(sslVerify bool, pemCerts []byte) configurer {
+ return func(conf *config) {
+ conf.sslVerify = sslVerify
+ conf.pemCerts = pemCerts
+ }
+}
+
+func WithHeaderTimeout(headerTimeout int) configurer {
+ return func(conf *config) {
+ conf.headerTimeout = headerTimeout
+ }
+}
+
+func WithProxyUrl(proxyUrl string) configurer {
+ return func(conf *config) {
+ conf.proxyUrl = proxyUrl
+ }
+}
+
+func WithMaxConnections(maxConnsPerHost int) configurer {
+ return func(conf *config) {
+ conf.maxConnsPerHost = maxConnsPerHost
+ }
+}
+
+func WithPathStyle(pathStyle bool) configurer {
+ return func(conf *config) {
+ conf.pathStyle = pathStyle
+ }
+}
+
+func WithSignature(signature SignatureType) configurer {
+ return func(conf *config) {
+ conf.signature = signature
+ }
+}
+
+func WithRegion(region string) configurer {
+ return func(conf *config) {
+ conf.region = region
+ }
+}
+
+func WithConnectTimeout(connectTimeout int) configurer {
+ return func(conf *config) {
+ conf.connectTimeout = connectTimeout
+ }
+}
+
+func WithSocketTimeout(socketTimeout int) configurer {
+ return func(conf *config) {
+ conf.socketTimeout = socketTimeout
+ }
+}
+
+func WithIdleConnTimeout(idleConnTimeout int) configurer {
+ return func(conf *config) {
+ conf.idleConnTimeout = idleConnTimeout
+ }
+}
+
+func WithMaxRetryCount(maxRetryCount int) configurer {
+ return func(conf *config) {
+ conf.maxRetryCount = maxRetryCount
+ }
+}
+
+func WithSecurityToken(securityToken string) configurer {
+ return func(conf *config) {
+ conf.securityProvider.securityToken = securityToken
+ }
+}
+
+func (conf *config) initConfigWithDefault() error {
+ conf.securityProvider.ak = strings.TrimSpace(conf.securityProvider.ak)
+ conf.securityProvider.sk = strings.TrimSpace(conf.securityProvider.sk)
+ conf.securityProvider.securityToken = strings.TrimSpace(conf.securityProvider.securityToken)
+ conf.endpoint = strings.TrimSpace(conf.endpoint)
+ if conf.endpoint == "" {
+ return errors.New("endpoint is not set")
+ }
+
+ if index := strings.Index(conf.endpoint, "?"); index > 0 {
+ conf.endpoint = conf.endpoint[:index]
+ }
+
+ for strings.LastIndex(conf.endpoint, "/") == len(conf.endpoint)-1 {
+ conf.endpoint = conf.endpoint[:len(conf.endpoint)-1]
+ }
+
+ if conf.signature == "" {
+ conf.signature = DEFAULT_SIGNATURE
+ }
+
+ urlHolder := &urlHolder{}
+ var address string
+ if strings.HasPrefix(conf.endpoint, "https://") {
+ urlHolder.scheme = "https"
+ address = conf.endpoint[len("https://"):]
+ } else if strings.HasPrefix(conf.endpoint, "http://") {
+ urlHolder.scheme = "http"
+ address = conf.endpoint[len("http://"):]
+ } else {
+ urlHolder.scheme = "http"
+ address = conf.endpoint
+ }
+
+ addr := strings.Split(address, ":")
+ if len(addr) == 2 {
+ if port, err := strconv.Atoi(addr[1]); err == nil {
+ urlHolder.port = port
+ }
+ }
+ urlHolder.host = addr[0]
+ if urlHolder.port == 0 {
+ if urlHolder.scheme == "https" {
+ urlHolder.port = 443
+ } else {
+ urlHolder.port = 80
+ }
+ }
+
+ if IsIP(urlHolder.host) {
+ conf.pathStyle = true
+ }
+
+ conf.urlHolder = urlHolder
+
+ conf.region = strings.TrimSpace(conf.region)
+ if conf.region == "" {
+ conf.region = DEFAULT_REGION
+ }
+
+ if conf.connectTimeout <= 0 {
+ conf.connectTimeout = DEFAULT_CONNECT_TIMEOUT
+ }
+
+ if conf.socketTimeout <= 0 {
+ conf.socketTimeout = DEFAULT_SOCKET_TIMEOUT
+ }
+
+ conf.finalTimeout = conf.socketTimeout * 10
+
+ if conf.headerTimeout <= 0 {
+ conf.headerTimeout = DEFAULT_HEADER_TIMEOUT
+ }
+
+ if conf.idleConnTimeout < 0 {
+ conf.idleConnTimeout = DEFAULT_IDLE_CONN_TIMEOUT
+ }
+
+ if conf.maxRetryCount < 0 {
+ conf.maxRetryCount = DEFAULT_MAX_RETRY_COUNT
+ }
+
+ if conf.maxConnsPerHost <= 0 {
+ conf.maxConnsPerHost = DEFAULT_MAX_CONN_PER_HOST
+ }
+
+ conf.proxyUrl = strings.TrimSpace(conf.proxyUrl)
+ return nil
+}
+
+func (conf *config) getTransport() (*http.Transport, error) {
+ transport := &http.Transport{
+ Dial: func(network, addr string) (net.Conn, error) {
+ conn, err := net.DialTimeout(network, addr, time.Second*time.Duration(conf.connectTimeout))
+ if err != nil {
+ return nil, err
+ }
+ return getConnDelegate(conn, conf.socketTimeout, conf.finalTimeout), nil
+ },
+ MaxIdleConns: conf.maxConnsPerHost,
+ MaxIdleConnsPerHost: conf.maxConnsPerHost,
+ ResponseHeaderTimeout: time.Second * time.Duration(conf.headerTimeout),
+ IdleConnTimeout: time.Second * time.Duration(conf.idleConnTimeout),
+ }
+
+ if conf.proxyUrl != "" {
+ proxyUrl, err := url.Parse(conf.proxyUrl)
+ if err != nil {
+ return nil, err
+ }
+ transport.Proxy = http.ProxyURL(proxyUrl)
+ }
+
+ tlsConfig := &tls.Config{InsecureSkipVerify: !conf.sslVerify}
+ if conf.sslVerify && conf.pemCerts != nil {
+ pool := x509.NewCertPool()
+ pool.AppendCertsFromPEM(conf.pemCerts)
+ tlsConfig.RootCAs = pool
+ }
+ transport.TLSClientConfig = tlsConfig
+
+ return transport, nil
+}
+
+func checkRedirectFunc(req *http.Request, via []*http.Request) error {
+ return http.ErrUseLastResponse
+}
+
+func DummyQueryEscape(s string) string {
+ return s
+}
+
+func (conf *config) formatUrls(bucketName, objectKey string, params map[string]string, escape bool) (requestUrl string, canonicalizedUrl string) {
+
+ urlHolder := conf.urlHolder
+
+ if bucketName == "" {
+ requestUrl = fmt.Sprintf("%s://%s:%d", urlHolder.scheme, urlHolder.host, urlHolder.port)
+ canonicalizedUrl = "/"
+ } else {
+ if conf.pathStyle {
+ requestUrl = fmt.Sprintf("%s://%s:%d/%s", urlHolder.scheme, urlHolder.host, urlHolder.port, bucketName)
+ canonicalizedUrl = "/" + bucketName
+ } else {
+ requestUrl = fmt.Sprintf("%s://%s.%s:%d", urlHolder.scheme, bucketName, urlHolder.host, urlHolder.port)
+ if conf.signature == "v2" {
+ canonicalizedUrl = "/" + bucketName + "/"
+ } else {
+ canonicalizedUrl = "/"
+ }
+ }
+ }
+ var escapeFunc func(s string) string
+ if escape {
+ escapeFunc = url.QueryEscape
+ } else {
+ escapeFunc = DummyQueryEscape
+ }
+
+ if objectKey != "" {
+ encodeObjectKey := escapeFunc(objectKey)
+ requestUrl += "/" + encodeObjectKey
+ if !strings.HasSuffix(canonicalizedUrl, "/") {
+ canonicalizedUrl += "/"
+ }
+ canonicalizedUrl += encodeObjectKey
+ }
+
+ keys := make([]string, 0, len(params))
+ for key, _ := range params {
+ keys = append(keys, strings.TrimSpace(key))
+ }
+ sort.Strings(keys)
+ i := 0
+
+ for index, key := range keys {
+ if index == 0 {
+ requestUrl += "?"
+ } else {
+ requestUrl += "&"
+ }
+ _key := url.QueryEscape(key)
+ requestUrl += _key
+
+ _value := params[key]
+
+ if conf.signature == "v4" {
+ requestUrl += "=" + url.QueryEscape(_value)
+ } else {
+ if _value != "" {
+ requestUrl += "=" + url.QueryEscape(_value)
+ _value = "=" + _value
+ } else {
+ _value = ""
+ }
+ lowerKey := strings.ToLower(key)
+ _, ok := allowed_resource_parameter_names[lowerKey]
+ ok = ok || strings.HasPrefix(lowerKey, HEADER_PREFIX)
+ if ok {
+ if i == 0 {
+ canonicalizedUrl += "?"
+ } else {
+ canonicalizedUrl += "&"
+ }
+ canonicalizedUrl += getQueryUrl(_key, _value)
+ i++
+ }
+ }
+ }
+ return
+}
+
+func getQueryUrl(key, value string) string {
+ queryUrl := ""
+ queryUrl += key
+ queryUrl += value
+ return queryUrl
+}
+
+
+
package obs
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "reflect"
+ "strings"
+ "time"
+
+ log "github.com/sirupsen/logrus"
+)
+
+func cleanHeaderPrefix(header http.Header) map[string][]string {
+ responseHeaders := make(map[string][]string)
+ for key, value := range header {
+ if len(value) > 0 {
+ key = strings.ToLower(key)
+ if strings.HasPrefix(key, HEADER_PREFIX) {
+ key = key[len(HEADER_PREFIX):]
+ }
+ responseHeaders[key] = value
+ }
+ }
+ return responseHeaders
+}
+
+func ParseStringToStorageClassType(value string) (ret StorageClassType) {
+ switch value {
+ case "STANDARD":
+ ret = StorageClassStandard
+ case "STANDARD_IA":
+ ret = StorageClassWarm
+ case "GLACIER":
+ ret = StorageClassCold
+ default:
+ ret = ""
+ }
+ return
+}
+
+func convertGrantToXml(grant Grant) string {
+ xml := make([]string, 0, 4)
+ xml = append(xml, fmt.Sprintf("<Grant><Grantee xsi:type=\"%s\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">", grant.Grantee.Type))
+ if grant.Grantee.Type == GranteeUser {
+ xml = append(xml, fmt.Sprintf("<ID>%s</ID>", grant.Grantee.ID))
+ if grant.Grantee.DisplayName != "" {
+ xml = append(xml, fmt.Sprintf("<DisplayName>%s</DisplayName>", grant.Grantee.DisplayName))
+ }
+ } else {
+ xml = append(xml, fmt.Sprintf("<URI>%s</URI>", grant.Grantee.URI))
+ }
+ xml = append(xml, fmt.Sprintf("</Grantee><Permission>%s</Permission></Grant>", grant.Permission))
+ return strings.Join(xml, "")
+}
+
+func ConvertLoggingStatusToXml(input BucketLoggingStatus, returnMd5 bool) (data string, md5 string) {
+ grantsLength := len(input.TargetGrants)
+ xml := make([]string, 0, 8+grantsLength)
+
+ xml = append(xml, "<BucketLoggingStatus>")
+ if input.TargetBucket != "" || input.TargetPrefix != "" {
+ xml = append(xml, "<LoggingEnabled>")
+ xml = append(xml, fmt.Sprintf("<TargetBucket>%s</TargetBucket>", input.TargetBucket))
+ xml = append(xml, fmt.Sprintf("<TargetPrefix>%s</TargetPrefix>", input.TargetPrefix))
+
+ if grantsLength > 0 {
+ xml = append(xml, "<TargetGrants>")
+ for _, grant := range input.TargetGrants {
+ xml = append(xml, convertGrantToXml(grant))
+ }
+ xml = append(xml, "</TargetGrants>")
+ }
+
+ xml = append(xml, "</LoggingEnabled>")
+ }
+ xml = append(xml, "</BucketLoggingStatus>")
+ data = strings.Join(xml, "")
+ if returnMd5 {
+ md5 = Base64Md5([]byte(data))
+ }
+ return
+}
+
+func ConvertAclToXml(input AccessControlPolicy, returnMd5 bool) (data string, md5 string) {
+ xml := make([]string, 0, 4+len(input.Grants))
+ xml = append(xml, fmt.Sprintf("<AccessControlPolicy><Owner><ID>%s</ID>", input.Owner.ID))
+ if input.Owner.DisplayName != "" {
+ xml = append(xml, fmt.Sprintf("<DisplayName>%s</DisplayName>", input.Owner.DisplayName))
+ }
+ xml = append(xml, "</Owner><AccessControlList>")
+ for _, grant := range input.Grants {
+ xml = append(xml, convertGrantToXml(grant))
+ }
+ xml = append(xml, "</AccessControlList></AccessControlPolicy>")
+ data = strings.Join(xml, "")
+ if returnMd5 {
+ md5 = Base64Md5([]byte(data))
+ }
+ return
+}
+
+func convertConditionToXml(condition Condition) string {
+ xml := make([]string, 0, 2)
+ if condition.KeyPrefixEquals != "" {
+ xml = append(xml, fmt.Sprintf("<KeyPrefixEquals>%s</KeyPrefixEquals>", condition.KeyPrefixEquals))
+ }
+ if condition.HttpErrorCodeReturnedEquals != "" {
+ xml = append(xml, fmt.Sprintf("<HttpErrorCodeReturnedEquals>%s</HttpErrorCodeReturnedEquals>", condition.HttpErrorCodeReturnedEquals))
+ }
+ if len(xml) > 0 {
+ return fmt.Sprintf("<Condition>%s</Condition>", strings.Join(xml, ""))
+ }
+ return ""
+}
+
+func ConvertWebsiteConfigurationToXml(input BucketWebsiteConfiguration, returnMd5 bool) (data string, md5 string) {
+ routingRuleLength := len(input.RoutingRules)
+ xml := make([]string, 0, 6+routingRuleLength*10)
+ xml = append(xml, "<WebsiteConfiguration>")
+
+ if input.RedirectAllRequestsTo.HostName != "" {
+ xml = append(xml, fmt.Sprintf("<RedirectAllRequestsTo><HostName>%s</HostName>", input.RedirectAllRequestsTo.HostName))
+ if input.RedirectAllRequestsTo.Protocol != "" {
+ xml = append(xml, fmt.Sprintf("<Protocol>%s</Protocol>", input.RedirectAllRequestsTo.Protocol))
+ }
+ xml = append(xml, "</RedirectAllRequestsTo>")
+ } else {
+ xml = append(xml, fmt.Sprintf("<IndexDocument><Suffix>%s</Suffix></IndexDocument>", input.IndexDocument.Suffix))
+ if input.ErrorDocument.Key != "" {
+ xml = append(xml, fmt.Sprintf("<ErrorDocument><Key>%s</Key></ErrorDocument>", input.ErrorDocument.Key))
+ }
+ if routingRuleLength > 0 {
+ xml = append(xml, "<RoutingRules>")
+ for _, routingRule := range input.RoutingRules {
+ xml = append(xml, "<RoutingRule>")
+ xml = append(xml, "<Redirect>")
+ if routingRule.Redirect.Protocol != "" {
+ xml = append(xml, fmt.Sprintf("<Protocol>%s</Protocol>", routingRule.Redirect.Protocol))
+ }
+ if routingRule.Redirect.HostName != "" {
+ xml = append(xml, fmt.Sprintf("<HostName>%s</HostName>", routingRule.Redirect.HostName))
+ }
+ if routingRule.Redirect.ReplaceKeyPrefixWith != "" {
+ xml = append(xml, fmt.Sprintf("<ReplaceKeyPrefixWith>%s</ReplaceKeyPrefixWith>", routingRule.Redirect.ReplaceKeyPrefixWith))
+ }
+
+ if routingRule.Redirect.ReplaceKeyWith != "" {
+ xml = append(xml, fmt.Sprintf("<ReplaceKeyWith>%s</ReplaceKeyWith>", routingRule.Redirect.ReplaceKeyWith))
+ }
+ if routingRule.Redirect.HttpRedirectCode != "" {
+ xml = append(xml, fmt.Sprintf("<HttpRedirectCode>%s</HttpRedirectCode>", routingRule.Redirect.HttpRedirectCode))
+ }
+ xml = append(xml, "</Redirect>")
+
+ if ret := convertConditionToXml(routingRule.Condition); ret != "" {
+ xml = append(xml, ret)
+ }
+ xml = append(xml, "</RoutingRule>")
+ }
+ xml = append(xml, "</RoutingRules>")
+ }
+ }
+
+ xml = append(xml, "</WebsiteConfiguration>")
+ data = strings.Join(xml, "")
+ if returnMd5 {
+ md5 = Base64Md5([]byte(data))
+ }
+ return
+}
+
+func convertTransitionsToXml(transitions []Transition) string {
+ if length := len(transitions); length > 0 {
+ xml := make([]string, 0, length)
+ for _, transition := range transitions {
+ var temp string
+ if transition.Days > 0 {
+ temp = fmt.Sprintf("<Days>%d</Days>", transition.Days)
+ } else if !transition.Date.IsZero() {
+ temp = fmt.Sprintf("<Date>%s</Date>", transition.Date.UTC().Format(ISO8601_MIDNIGHT_DATE_FORMAT))
+ }
+ if temp != "" {
+ xml = append(xml, fmt.Sprintf("<Transition>%s<StorageClass>%s</StorageClass></Transition>", temp, transition.StorageClass))
+ }
+ }
+ return strings.Join(xml, "")
+ }
+ return ""
+}
+
+func convertExpirationToXml(expiration Expiration) string {
+ if expiration.Days > 0 {
+ return fmt.Sprintf("<Expiration><Days>%d</Days></Expiration>", expiration.Days)
+ } else if !expiration.Date.IsZero() {
+ return fmt.Sprintf("<Expiration><Date>%s</Date></Expiration>", expiration.Date.UTC().Format(ISO8601_MIDNIGHT_DATE_FORMAT))
+ }
+ return ""
+}
+func convertNoncurrentVersionTransitionsToXml(noncurrentVersionTransitions []NoncurrentVersionTransition) string {
+ if length := len(noncurrentVersionTransitions); length > 0 {
+ xml := make([]string, 0, length)
+ for _, noncurrentVersionTransition := range noncurrentVersionTransitions {
+ if noncurrentVersionTransition.NoncurrentDays > 0 {
+ xml = append(xml, fmt.Sprintf("<NoncurrentVersionTransition><NoncurrentDays>%d</NoncurrentDays>"+
+ "<StorageClass>%s</StorageClass></NoncurrentVersionTransition>",
+ noncurrentVersionTransition.NoncurrentDays, noncurrentVersionTransition.StorageClass))
+ }
+ }
+ return strings.Join(xml, "")
+ }
+ return ""
+}
+func convertNoncurrentVersionExpirationToXml(noncurrentVersionExpiration NoncurrentVersionExpiration) string {
+ if noncurrentVersionExpiration.NoncurrentDays > 0 {
+ return fmt.Sprintf("<NoncurrentVersionExpiration><NoncurrentDays>%d</NoncurrentDays></NoncurrentVersionExpiration>", noncurrentVersionExpiration.NoncurrentDays)
+ }
+ return ""
+}
+
+func ConvertLifecyleConfigurationToXml(input BucketLifecyleConfiguration, returnMd5 bool) (data string, md5 string) {
+ xml := make([]string, 0, 2+len(input.LifecycleRules)*9)
+ xml = append(xml, "<LifecycleConfiguration>")
+ for _, lifecyleRule := range input.LifecycleRules {
+ xml = append(xml, "<Rule>")
+ if lifecyleRule.ID != "" {
+ xml = append(xml, fmt.Sprintf("<ID>%s</ID>", lifecyleRule.ID))
+ }
+ xml = append(xml, fmt.Sprintf("<Prefix>%s</Prefix>", lifecyleRule.Prefix))
+ xml = append(xml, fmt.Sprintf("<Status>%s</Status>", lifecyleRule.Status))
+ if ret := convertTransitionsToXml(lifecyleRule.Transitions); ret != "" {
+ xml = append(xml, ret)
+ }
+ if ret := convertExpirationToXml(lifecyleRule.Expiration); ret != "" {
+ xml = append(xml, ret)
+ }
+ if ret := convertNoncurrentVersionTransitionsToXml(lifecyleRule.NoncurrentVersionTransitions); ret != "" {
+ xml = append(xml, ret)
+ }
+ if ret := convertNoncurrentVersionExpirationToXml(lifecyleRule.NoncurrentVersionExpiration); ret != "" {
+ xml = append(xml, ret)
+ }
+ xml = append(xml, "</Rule>")
+ }
+ xml = append(xml, "</LifecycleConfiguration>")
+ data = strings.Join(xml, "")
+ if returnMd5 {
+ md5 = Base64Md5([]byte(data))
+ }
+ return
+}
+
+func converntFilterRulesToXml(filterRules []FilterRule) string {
+ if length := len(filterRules); length > 0 {
+ xml := make([]string, 0, length*4)
+ for _, filterRule := range filterRules {
+ xml = append(xml, "<FilterRule>")
+ if filterRule.Name != "" {
+ xml = append(xml, fmt.Sprintf("<Name>%s</Name>", filterRule.Name))
+ }
+ if filterRule.Value != "" {
+ xml = append(xml, fmt.Sprintf("<Value>%s</Value>", filterRule.Value))
+ }
+ xml = append(xml, "</FilterRule>")
+ }
+ return fmt.Sprintf("<Filter><S3Key>%s</S3Key></Filter>", strings.Join(xml, ""))
+ }
+ return ""
+}
+
+func converntEventsToXml(events []string) string {
+ if length := len(events); length > 0 {
+ xml := make([]string, 0, length)
+ for _, event := range events {
+ xml = append(xml, fmt.Sprintf("<Event>%s</Event>", event))
+ }
+ return strings.Join(xml, "")
+ }
+ return ""
+}
+
+func ConvertNotificationToXml(input BucketNotification, returnMd5 bool) (data string, md5 string) {
+ xml := make([]string, 0, 2+len(input.TopicConfigurations)*6)
+ xml = append(xml, "<NotificationConfiguration>")
+ for _, topicConfiguration := range input.TopicConfigurations {
+ xml = append(xml, "<TopicConfiguration>")
+ if topicConfiguration.ID != "" {
+ xml = append(xml, fmt.Sprintf("<Id>%s</Id>", topicConfiguration.ID))
+ }
+ xml = append(xml, fmt.Sprintf("<Topic>%s</Topic>", topicConfiguration.Topic))
+
+ if ret := converntEventsToXml(topicConfiguration.Events); ret != "" {
+ xml = append(xml, ret)
+ }
+ if ret := converntFilterRulesToXml(topicConfiguration.FilterRules); ret != "" {
+ xml = append(xml, ret)
+ }
+ xml = append(xml, "</TopicConfiguration>")
+ }
+ xml = append(xml, "</NotificationConfiguration>")
+ data = strings.Join(xml, "")
+ if returnMd5 {
+ md5 = Base64Md5([]byte(data))
+ }
+ return
+}
+
+func ConvertCompleteMultipartUploadInputToXml(input CompleteMultipartUploadInput, returnMd5 bool) (data string, md5 string) {
+ xml := make([]string, 0, 2+len(input.Parts)*4)
+ xml = append(xml, "<CompleteMultipartUpload>")
+ for _, part := range input.Parts {
+ xml = append(xml, "<Part>")
+ xml = append(xml, fmt.Sprintf("<PartNumber>%d</PartNumber>", part.PartNumber))
+ xml = append(xml, fmt.Sprintf("<ETag>%s</ETag>", part.ETag))
+ xml = append(xml, "</Part>")
+ }
+ xml = append(xml, "</CompleteMultipartUpload>")
+ data = strings.Join(xml, "")
+ if returnMd5 {
+ md5 = Base64Md5([]byte(data))
+ }
+ return
+}
+
+func parseSseHeader(responseHeaders map[string][]string) (sseHeader ISseHeader) {
+ if ret, ok := responseHeaders[HEADER_SSEC_ENCRYPTION]; ok {
+ sseCHeader := SseCHeader{Encryption: ret[0]}
+ if ret, ok = responseHeaders[HEADER_SSEC_KEY_MD5]; ok {
+ sseCHeader.KeyMD5 = ret[0]
+ }
+ sseHeader = sseCHeader
+ } else if ret, ok := responseHeaders[HEADER_SSEKMS_ENCRYPTION]; ok {
+ sseKmsHeader := SseKmsHeader{Encryption: ret[0]}
+ if ret, ok = responseHeaders[HEADER_SSEKMS_KEY]; ok {
+ sseKmsHeader.Key = ret[0]
+ }
+ sseHeader = sseKmsHeader
+ }
+ return
+}
+
+func ParseGetObjectMetadataOutput(output *GetObjectMetadataOutput) {
+ if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok {
+ output.VersionId = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_WEBSITE_REDIRECT_LOCATION]; ok {
+ output.WebsiteRedirectLocation = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_EXPIRATION]; ok {
+ output.Expiration = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_RESTORE]; ok {
+ output.Restore = ret[0]
+ }
+
+ if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok {
+ output.StorageClass = ParseStringToStorageClassType(ret[0])
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok {
+ output.ETag = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_CONTENT_TYPE]; ok {
+ output.ContentType = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN]; ok {
+ output.AllowOrigin = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_HEADERS]; ok {
+ output.AllowHeader = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_MAX_AGE]; ok {
+ output.MaxAgeSeconds = StringToInt(ret[0], 0)
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_METHODS]; ok {
+ output.AllowMethod = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS]; ok {
+ output.ExposeHeader = ret[0]
+ }
+
+ output.SseHeader = parseSseHeader(output.ResponseHeaders)
+ if ret, ok := output.ResponseHeaders[HEADER_LASTMODIFIED]; ok {
+ ret, err := time.Parse(time.RFC1123, ret[0])
+ if err == nil {
+ output.LastModified = ret
+ }
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LENGTH]; ok {
+ output.ContentLength = StringToInt64(ret[0], 0)
+ }
+
+ output.Metadata = make(map[string]string)
+
+ for key, value := range output.ResponseHeaders {
+ if strings.HasPrefix(key, PREFIX_META) {
+ _key := key[len(PREFIX_META):]
+ output.ResponseHeaders[_key] = value
+ output.Metadata[_key] = value[0]
+ delete(output.ResponseHeaders, key)
+ }
+ }
+
+}
+
+func ParseCopyObjectOutput(output *CopyObjectOutput) {
+ if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok {
+ output.VersionId = ret[0]
+ }
+ output.SseHeader = parseSseHeader(output.ResponseHeaders)
+ if ret, ok := output.ResponseHeaders[HEADER_COPY_SOURCE_VERSION_ID]; ok {
+ output.CopySourceVersionId = ret[0]
+ }
+}
+
+func ParsePutObjectOutput(output *PutObjectOutput) {
+ if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok {
+ output.VersionId = ret[0]
+ }
+ output.SseHeader = parseSseHeader(output.ResponseHeaders)
+ if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok {
+ output.StorageClass = ParseStringToStorageClassType(ret[0])
+ }
+
+ if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok {
+ output.ETag = ret[0]
+ }
+}
+
+func ParseInitiateMultipartUploadOutput(output *InitiateMultipartUploadOutput) {
+ output.SseHeader = parseSseHeader(output.ResponseHeaders)
+}
+
+func ParseUploadPartOutput(output *UploadPartOutput) {
+ output.SseHeader = parseSseHeader(output.ResponseHeaders)
+ if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok {
+ output.ETag = ret[0]
+ }
+}
+
+func ParseCompleteMultipartUploadOutput(output *CompleteMultipartUploadOutput) {
+ output.SseHeader = parseSseHeader(output.ResponseHeaders)
+ if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok {
+ output.VersionId = ret[0]
+ }
+}
+
+func ParseCopyPartOutput(output *CopyPartOutput) {
+ output.SseHeader = parseSseHeader(output.ResponseHeaders)
+}
+
+func ParseGetBucketMetadataOutput(output *GetBucketMetadataOutput) {
+ if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS]; ok {
+ output.StorageClass = ParseStringToStorageClassType(ret[0])
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok {
+ output.Location = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN]; ok {
+ output.AllowOrigin = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_HEADERS]; ok {
+ output.AllowHeader = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_MAX_AGE]; ok {
+ output.MaxAgeSeconds = StringToInt(ret[0], 0)
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_METHODS]; ok {
+ output.AllowMethod = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS]; ok {
+ output.ExposeHeader = ret[0]
+ }
+}
+
+func ParseDeleteObjectOutput(output *DeleteObjectOutput) {
+ if versionId, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok {
+ output.VersionId = versionId[0]
+ }
+
+ if deleteMarker, ok := output.ResponseHeaders[HEADER_DELETE_MARKER]; ok {
+ output.DeleteMarker = deleteMarker[0] == "true"
+ }
+}
+
+func ParseGetObjectOutput(output *GetObjectOutput) {
+ ParseGetObjectMetadataOutput(&output.GetObjectMetadataOutput)
+ if ret, ok := output.ResponseHeaders[HEADER_DELETE_MARKER]; ok {
+ output.DeleteMarker = ret[0] == "true"
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_CACHE_CONTROL]; ok {
+ output.CacheControl = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_CONTENT_DISPOSITION]; ok {
+ output.ContentDisposition = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_CONTENT_ENCODING]; ok {
+ output.ContentEncoding = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LANGUAGE]; ok {
+ output.ContentLanguage = ret[0]
+ }
+ if ret, ok := output.ResponseHeaders[HEADER_EXPIRES]; ok {
+ output.Expires = ret[0]
+ }
+}
+
+func ConvertRequestToIoReaderV2(req interface{}) (io.Reader, string, error) {
+ data, err := TransToXml(req)
+ if err == nil {
+ log.Debug("Do http request with data: %s", string(data))
+ return bytes.NewReader(data), Base64Md5(data), nil
+ }
+ return nil, "", err
+}
+
+func ConvertRequestToIoReader(req interface{}) (io.Reader, error) {
+ body, err := TransToXml(req)
+ if err == nil {
+ log.Debug("Do http request with data: %s", string(body))
+ return bytes.NewReader(body), nil
+ }
+ return nil, err
+}
+
+func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResult bool) (err error) {
+ readCloser, ok := baseModel.(IReadCloser)
+ if !ok {
+ defer resp.Body.Close()
+ body, err := ioutil.ReadAll(resp.Body)
+ if err == nil && len(body) > 0 {
+ if xmlResult {
+ err = ParseXml(body, baseModel)
+ if err != nil {
+ log.Error("Unmarshal error: %v", err)
+ }
+ } else {
+ s := reflect.TypeOf(baseModel).Elem()
+ for i := 0; i < s.NumField(); i++ {
+ if s.Field(i).Tag == "body" {
+ reflect.ValueOf(baseModel).Elem().FieldByName(s.Field(i).Name).SetString(string(body))
+ break
+ }
+ }
+ }
+ }
+ } else {
+ readCloser.setReadCloser(resp.Body)
+ }
+
+ baseModel.setStatusCode(resp.StatusCode)
+ responseHeaders := cleanHeaderPrefix(resp.Header)
+ baseModel.setResponseHeaders(responseHeaders)
+ if values, ok := responseHeaders[HEADER_REQUEST_ID]; ok {
+ baseModel.setRequestId(values[0])
+ }
+ return
+}
+
+func ParseResponseToObsError(resp *http.Response) error {
+ obsError := ObsError{}
+ ParseResponseToBaseModel(resp, &obsError, true)
+ obsError.Status = resp.Status
+ return obsError
+}
+
+
+
package obs
+
+import (
+ "encoding/xml"
+ "fmt"
+)
+
+type ObsError struct {
+ BaseModel
+ Status string
+ XMLName xml.Name `xml:"Error"`
+ Code string `xml:"Code"`
+ Message string `xml:"Message"`
+ Resource string `xml:"Resource"`
+ HostId string `xml:"HostId"`
+}
+
+func (err ObsError) Error() string {
+ return fmt.Sprintf("obs: service returned error: Status=%s, Code=%s, Message=%s, RequestId=%s",
+ err.Status, err.Code, err.Message, err.RequestId)
+}
+
+
+
package obs
+
+import (
+ "bytes"
+ "errors"
+ "io"
+ "math/rand"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "strings"
+ "time"
+
+ log "github.com/sirupsen/logrus"
+)
+
+func prepareHeaders(headers map[string][]string, meta bool) map[string][]string {
+ _headers := make(map[string][]string, len(headers))
+ if headers != nil {
+ for key, value := range headers {
+ key = strings.TrimSpace(key)
+ if key == "" {
+ continue
+ }
+ _key := strings.ToLower(key)
+ if _, ok := allowed_request_http_header_metadata_names[_key]; !ok && !strings.HasPrefix(key, HEADER_PREFIX) {
+ if !meta {
+ continue
+ }
+ _key = HEADER_PREFIX_META + _key
+ } else {
+ _key = key
+ }
+ _headers[_key] = value
+ }
+ }
+ return _headers
+}
+
+func (obsClient ObsClient) doActionWithoutBucket(action, method string, input ISerializable, output IBaseModel) error {
+ return obsClient.doAction(action, method, "", "", input, output, true, true)
+}
+
+func (obsClient ObsClient) doActionWithBucketV2(action, method, bucketName string, input ISerializable, output IBaseModel) error {
+ if strings.TrimSpace(bucketName) == "" {
+ return errors.New("Bucket is empty")
+ }
+ return obsClient.doAction(action, method, bucketName, "", input, output, false, true)
+}
+
+func (obsClient ObsClient) doActionWithBucket(action, method, bucketName string, input ISerializable, output IBaseModel) error {
+ if strings.TrimSpace(bucketName) == "" {
+ return errors.New("Bucket is empty")
+ }
+ return obsClient.doAction(action, method, bucketName, "", input, output, true, true)
+}
+
+func (obsClient ObsClient) doActionWithBucketAndKey(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel) error {
+ return obsClient._doActionWithBucketAndKey(action, method, bucketName, objectKey, input, output, true)
+}
+
+func (obsClient ObsClient) doActionWithBucketAndKeyUnRepeatable(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel) error {
+ return obsClient._doActionWithBucketAndKey(action, method, bucketName, objectKey, input, output, false)
+}
+
+func (obsClient ObsClient) _doActionWithBucketAndKey(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, repeatable bool) error {
+ if strings.TrimSpace(bucketName) == "" {
+ return errors.New("Key is empty")
+ }
+ if strings.TrimSpace(objectKey) == "" {
+ return errors.New("Key is empty")
+ }
+ return obsClient.doAction(action, method, bucketName, objectKey, input, output, true, repeatable)
+}
+
+func (obsClient ObsClient) doAction(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, xmlResult bool, repeatable bool) error {
+
+ var resp *http.Response
+ var respError error
+ log.Info("Enter method %s...", action)
+ start := GetCurrentTimestamp()
+
+ params, headers, data := input.trans()
+
+ if params == nil {
+ params = make(map[string]string)
+ }
+
+ if headers == nil {
+ headers = make(map[string][]string)
+ }
+
+ switch method {
+ case HTTP_GET:
+ resp, respError = obsClient.doHttpGet(bucketName, objectKey, params, headers, data, repeatable)
+ case HTTP_POST:
+ resp, respError = obsClient.doHttpPost(bucketName, objectKey, params, headers, data, repeatable)
+ case HTTP_PUT:
+ resp, respError = obsClient.doHttpPut(bucketName, objectKey, params, headers, data, repeatable)
+ case HTTP_DELETE:
+ resp, respError = obsClient.doHttpDelete(bucketName, objectKey, params, headers, data, repeatable)
+ case HTTP_HEAD:
+ resp, respError = obsClient.doHttpHead(bucketName, objectKey, params, headers, data, repeatable)
+ case HTTP_OPTIONS:
+ resp, respError = obsClient.doHttpOptions(bucketName, objectKey, params, headers, data, repeatable)
+ default:
+ respError = errors.New("Unexpect http method error")
+ }
+ if respError == nil && output != nil {
+ respError = ParseResponseToBaseModel(resp, output, xmlResult)
+ if respError != nil {
+ log.Warn("Parse response to BaseModel with error: %v", respError)
+ }
+ } else {
+ log.Warn("Do http request with error: %v", respError)
+ }
+
+ log.Debug("End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start))
+
+ return respError
+}
+
+func (obsClient ObsClient) doHttpGet(bucketName, objectKey string, params map[string]string,
+ headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) {
+ return obsClient.doHttp(HTTP_GET, bucketName, objectKey, params, prepareHeaders(headers, false), data, repeatable)
+}
+
+func (obsClient ObsClient) doHttpHead(bucketName, objectKey string, params map[string]string,
+ headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) {
+ return obsClient.doHttp(HTTP_HEAD, bucketName, objectKey, params, prepareHeaders(headers, false), data, repeatable)
+}
+
+func (obsClient ObsClient) doHttpOptions(bucketName, objectKey string, params map[string]string,
+ headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) {
+ return obsClient.doHttp(HTTP_OPTIONS, bucketName, objectKey, params, prepareHeaders(headers, false), data, repeatable)
+}
+
+func (obsClient ObsClient) doHttpDelete(bucketName, objectKey string, params map[string]string,
+ headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) {
+ return obsClient.doHttp(HTTP_DELETE, bucketName, objectKey, params, prepareHeaders(headers, false), data, repeatable)
+}
+
+func (obsClient ObsClient) doHttpPut(bucketName, objectKey string, params map[string]string,
+ headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) {
+ return obsClient.doHttp(HTTP_PUT, bucketName, objectKey, params, prepareHeaders(headers, true), data, repeatable)
+}
+
+func (obsClient ObsClient) doHttpPost(bucketName, objectKey string, params map[string]string,
+ headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) {
+ return obsClient.doHttp(HTTP_POST, bucketName, objectKey, params, prepareHeaders(headers, true), data, repeatable)
+}
+
+func (obsClient ObsClient) doHttpWithSignedUrl(action, method string, signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader, output IBaseModel, xmlResult bool) (respError error) {
+ req, err := http.NewRequest(method, signedUrl, data)
+ if err != nil {
+ return err
+ }
+
+ var resp *http.Response
+
+ log.Info("Do %s with signedUrl %s...", action, signedUrl)
+
+ req.Header = actualSignedRequestHeaders
+ if value, ok := req.Header[HEADER_HOST_CAMEL]; ok {
+ req.Host = value[0]
+ delete(req.Header, HEADER_HOST_CAMEL)
+ } else if value, ok := req.Header[HEADER_HOST]; ok {
+ req.Host = value[0]
+ delete(req.Header, HEADER_HOST)
+ }
+
+ if value, ok := req.Header[HEADER_CONTENT_LENGTH_CAMEL]; ok {
+ req.ContentLength = StringToInt64(value[0], -1)
+ delete(req.Header, HEADER_CONTENT_LENGTH_CAMEL)
+ } else if value, ok := req.Header[HEADER_CONTENT_LENGTH]; ok {
+ req.ContentLength = StringToInt64(value[0], -1)
+ delete(req.Header, HEADER_CONTENT_LENGTH)
+ }
+
+ req.Header[HEADER_USER_AGENT_CAMEL] = []string{USER_AGENT}
+ start := GetCurrentTimestamp()
+ resp, err = obsClient.httpClient.Do(req)
+ log.Info("Do http request cost %d ms", (GetCurrentTimestamp() - start))
+
+
+ var msg interface{}
+ if err != nil {
+ respError = err
+ resp = nil
+ } else {
+ log.Debug("Response headers: %v", resp.Header)
+ if resp.StatusCode >= 300 {
+ respError = ParseResponseToObsError(resp)
+ msg = resp.Status
+ resp = nil
+ } else {
+ if output != nil {
+ respError = ParseResponseToBaseModel(resp, output, xmlResult)
+ }
+ if respError != nil {
+ log.Warn("Parse response to BaseModel with error: %v", respError)
+ }
+ }
+ }
+
+ if msg != nil {
+ log.Error("Failed to send request with reason:%v", msg)
+ }
+
+ log.Debug("End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start))
+
+ return
+}
+
+func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params map[string]string,
+ headers map[string][]string, data interface{}, repeatable bool) (resp *http.Response, respError error) {
+
+ bucketName = strings.TrimSpace(bucketName)
+
+ objectKey = strings.TrimSpace(objectKey)
+
+ method = strings.ToUpper(method)
+
+ var redirectUrl string
+ var requestUrl string
+ maxRetryCount := obsClient.conf.maxRetryCount
+
+ var _data io.Reader
+ if data != nil {
+ if dataStr, ok := data.(string); ok {
+ log.Debug("Do http request with string: %s", dataStr)
+ headers["Content-Length"] = []string{IntToString(len(dataStr))}
+ _data = strings.NewReader(dataStr)
+ } else if dataByte, ok := data.([]byte); ok {
+ log.Debug("Do http request with byte array")
+ headers["Content-Length"] = []string{IntToString(len(dataByte))}
+ _data = bytes.NewReader(dataByte)
+ } else if dataReader, ok := data.(io.Reader); ok {
+ _data = dataReader
+ } else {
+ log.Warn("Data is not a valid io.Reader")
+ return nil, errors.New("Data is not a valid io.Reader")
+ }
+ }
+
+ for i := 0; i <= maxRetryCount; i++ {
+ if redirectUrl != "" {
+ parsedRedirectUrl, err := url.Parse(redirectUrl)
+ if err != nil {
+ return nil, err
+ }
+ requestUrl, _ = obsClient.doAuth(method, bucketName, objectKey, params, headers, parsedRedirectUrl.Host)
+ if parsedRequestUrl, _ := url.Parse(requestUrl); parsedRequestUrl.RawQuery != "" && parsedRedirectUrl.RawQuery == "" {
+ redirectUrl += "?" + parsedRequestUrl.RawQuery
+ }
+ requestUrl = redirectUrl
+ } else {
+ var err error
+ requestUrl, err = obsClient.doAuth(method, bucketName, objectKey, params, headers, "")
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ req, err := http.NewRequest(method, requestUrl, _data)
+ if err != nil {
+ return nil, err
+ }
+ log.Debug("Do request with url [%s] and method [%s]", requestUrl, method)
+
+ auth := headers[HEADER_AUTH_CAMEL]
+ delete(headers, HEADER_AUTH_CAMEL)
+ log.Debug("Request headers: %v", headers)
+ headers[HEADER_AUTH_CAMEL] = auth
+
+ for key, value := range headers {
+ if key == HEADER_HOST_CAMEL {
+ req.Host = value[0]
+ delete(headers, key)
+ } else if key == HEADER_CONTENT_LENGTH_CAMEL {
+ req.ContentLength = StringToInt64(value[0], -1)
+ delete(headers, key)
+ } else {
+ req.Header[key] = value
+ }
+ }
+
+ req.Header[HEADER_USER_AGENT_CAMEL] = []string{USER_AGENT}
+
+ start := GetCurrentTimestamp()
+ resp, err = obsClient.httpClient.Do(req)
+ log.Info("Do http request cost %d ms", (GetCurrentTimestamp() - start))
+
+ var msg interface{}
+ if err != nil {
+ msg = err
+ respError = err
+ resp = nil
+ } else {
+ log.Debug("Response headers: %v", resp.Header)
+ if resp.StatusCode < 300 {
+ break
+ } else if !repeatable || (resp.StatusCode >= 400 && resp.StatusCode < 500) || resp.StatusCode == 304 {
+ respError = ParseResponseToObsError(resp)
+ resp = nil
+ break
+ } else if resp.StatusCode >= 300 && resp.StatusCode < 400 {
+ if location := resp.Header.Get(HEADER_LOCATION_CAMEL); location != "" {
+ redirectUrl = location
+ log.Warn("Redirect request to %s", redirectUrl)
+ msg = resp.Status
+ maxRetryCount++
+ } else {
+ respError = ParseResponseToObsError(resp)
+ resp = nil
+ break
+ }
+ } else {
+ msg = resp.Status
+ }
+ }
+ if i != maxRetryCount {
+ if resp != nil {
+ resp.Body.Close()
+ resp = nil
+ }
+ if _, ok := headers[HEADER_AUTH_CAMEL]; ok {
+ delete(headers, HEADER_AUTH_CAMEL)
+ }
+ log.Warn("Failed to send request with reason:%v, will try again", msg)
+ if r, ok := _data.(*strings.Reader); ok {
+ r.Seek(0, 0)
+ } else if r, ok := _data.(*bytes.Reader); ok {
+ r.Seek(0, 0)
+ } else if r, ok := _data.(*fileReaderWrapper); ok {
+ fd, err := os.Open(r.filePath)
+ if err != nil {
+ return nil, err
+ }
+ defer fd.Close()
+ fileReaderWrapper := &fileReaderWrapper{filePath: r.filePath}
+ fileReaderWrapper.mark = r.mark
+ fileReaderWrapper.reader = fd
+ fileReaderWrapper.totalCount = r.totalCount
+ _data = fileReaderWrapper
+ fd.Seek(r.mark, 0)
+ } else if r, ok := _data.(*readerWrapper); ok {
+ r.seek(0, 0)
+ }
+ time.Sleep(time.Duration(float64(i+2) * rand.Float64() * float64(time.Second)))
+ } else {
+ log.Error("Failed to send request with reason:%v", msg)
+ if resp != nil {
+ respError = ParseResponseToObsError(resp)
+ resp = nil
+ }
+ }
+ }
+ return
+}
+
+type connDelegate struct {
+ conn net.Conn
+ socketTimeout time.Duration
+ finalTimeout time.Duration
+}
+
+func getConnDelegate(conn net.Conn, socketTimeout int, finalTimeout int) *connDelegate {
+ return &connDelegate{
+ conn: conn,
+ socketTimeout: time.Second * time.Duration(socketTimeout),
+ finalTimeout: time.Second * time.Duration(finalTimeout),
+ }
+}
+
+func (delegate *connDelegate) Read(b []byte) (n int, err error) {
+ delegate.SetReadDeadline(time.Now().Add(delegate.socketTimeout))
+ n, err = delegate.conn.Read(b)
+ delegate.SetReadDeadline(time.Now().Add(delegate.finalTimeout))
+ return n, err
+}
+
+func (delegate *connDelegate) Write(b []byte) (n int, err error) {
+ delegate.SetWriteDeadline(time.Now().Add(delegate.socketTimeout))
+ n, err = delegate.conn.Write(b)
+ finalTimeout := time.Now().Add(delegate.finalTimeout)
+ delegate.SetWriteDeadline(finalTimeout)
+ delegate.SetReadDeadline(finalTimeout)
+ return n, err
+}
+
+func (delegate *connDelegate) Close() error {
+ return delegate.conn.Close()
+}
+
+func (delegate *connDelegate) LocalAddr() net.Addr {
+ return delegate.conn.LocalAddr()
+}
+
+func (delegate *connDelegate) RemoteAddr() net.Addr {
+ return delegate.conn.RemoteAddr()
+}
+
+func (delegate *connDelegate) SetDeadline(t time.Time) error {
+ return delegate.conn.SetDeadline(t)
+}
+
+func (delegate *connDelegate) SetReadDeadline(t time.Time) error {
+ return delegate.conn.SetReadDeadline(t)
+}
+
+func (delegate *connDelegate) SetWriteDeadline(t time.Time) error {
+ return delegate.conn.SetWriteDeadline(t)
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package obs
+
+import (
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "os/user"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/go-ini/ini"
+ "github.com/sirupsen/logrus"
+ "gopkg.in/natefinch/lumberjack.v2"
+)
+
+type LogFormatter struct {
+ TimestampFormat string
+ LogFormat string
+}
+
+const (
+ debugLevel = "debug"
+ infoLevel = "info"
+ warnLevel = "warn"
+ errorLevel = "error"
+ path = "path"
+ level = "level"
+ format = "format"
+ defaultLogPath = "/var/log/multi-cloud"
+ defaultLogLevel = "info"
+ unknownHost = "unknownhost"
+ unknownUser = "unknownuser"
+ configFileName = "/etc/multi-cloud/multi-cloud.conf"
+ defaultLogFormat = "[%time%] [%level%] [%filename%] [%funcName%():%lineNo%] [PID:%process%] %message%"
+ defaultTimestampFormat = time.RFC3339
+ logSection = "log"
+ tenMb = 10
+ threeMonth = 100
+)
+
+func InitLogs() {
+ path, level, format := readConfigurationFile()
+ configureLogModule(path, level, format)
+}
+
+func configureLogModule(path, level, format string) {
+ configureWriter(path, format)
+ configureLevel(level)
+}
+
+func configureWriter(path, format string) {
+ logrus.SetFormatter(&LogFormatter{
+ TimestampFormat: defaultTimestampFormat,
+ LogFormat: format + "\n",
+ })
+ fileWriter := &lumberjack.Logger{
+ Filename: filepath.Join(path, logName()),
+ MaxSize: tenMb,
+ MaxAge: threeMonth,
+ Compress: true,
+ }
+ multiWriter := io.MultiWriter(os.Stdout, fileWriter)
+ logrus.SetOutput(multiWriter)
+}
+
+func configureLevel(level string) {
+ switch level {
+ case debugLevel:
+ logrus.SetLevel(logrus.DebugLevel)
+ case infoLevel:
+ logrus.SetLevel(logrus.InfoLevel)
+ case warnLevel:
+ logrus.SetLevel(logrus.WarnLevel)
+ case errorLevel:
+ logrus.SetLevel(logrus.ErrorLevel)
+ }
+ logrus.SetReportCaller(true)
+}
+
+func logName() (name string) {
+ name = fmt.Sprintf("%s.%s.%s.log",
+ filepath.Base(os.Args[0]),
+ hostName(),
+ userName())
+ return name
+}
+
+func shortHostname(hostname string) string {
+ if i := strings.Index(hostname, "."); i >= 0 {
+ return hostname[:i]
+ }
+ return hostname
+}
+
+func hostName() string {
+ host := unknownHost
+ h, err := os.Hostname()
+ if err == nil {
+ host = shortHostname(h)
+ }
+ return host
+}
+
+func userName() string {
+ userName := unknownUser
+ current, err := user.Current()
+ if err == nil {
+ userName = current.Username
+ }
+ // Sanitize userName since it may contain filepath separators on Windows.
+ userName = strings.Replace(userName, `\`, "_", -1)
+ return userName
+}
+
+func readConfigurationFile() (cfgPath, cfgLevel, cfgFormat string) {
+ cfgPath = defaultLogPath
+ cfgLevel = defaultLogLevel
+ cfgFormat = defaultLogFormat
+ cfg, err := ini.Load(configFileName)
+ if err != nil {
+ log.Println("Failed to open config file")
+ return cfgPath, cfgLevel, cfgFormat
+ }
+ if cfg.Section(logSection).HasKey(path) {
+ cfgPath = cfg.Section(logSection).Key(path).String()
+ }
+ if cfg.Section(logSection).HasKey(level) {
+ cfgLevel = strings.ToLower(cfg.Section(logSection).Key(level).String())
+ }
+ if cfg.Section(logSection).HasKey(format) {
+ cfgFormat = cfg.Section(logSection).Key(format).String()
+ }
+
+ return cfgPath, cfgLevel, cfgFormat
+}
+
+func (f *LogFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+ output := f.LogFormat
+ if output == "" {
+ output = defaultLogFormat
+ }
+
+ timestampFormat := f.TimestampFormat
+ if timestampFormat == "" {
+ timestampFormat = defaultTimestampFormat
+ }
+
+ output = strings.Replace(output, "%time%", entry.Time.Format(timestampFormat), 1)
+
+ output = strings.Replace(output, "%message%", entry.Message, 1)
+
+ level := strings.ToUpper(entry.Level.String())
+ output = strings.Replace(output, "%level%", level, 1)
+
+ output = strings.Replace(output, "%process%", strconv.Itoa(os.Getpid()), 1)
+
+ output = strings.Replace(output, "%filename%", entry.Caller.File, 1)
+ output = strings.Replace(output, "%lineNo%", strconv.Itoa(entry.Caller.Line), 1)
+ funcName := entry.Caller.Function
+ if strings.Contains(funcName, "/") {
+ funcName = string([]byte(funcName)[strings.LastIndex(funcName, "/")+1:])
+ }
+ output = strings.Replace(output, "%funcName%", funcName, 1)
+
+ return []byte(output), nil
+}
+
+
+
package obs
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "strings"
+ "time"
+)
+
+func (obsClient ObsClient) CreateSignedUrl(input *CreateSignedUrlInput) (output *CreateSignedUrlOutput, err error) {
+ if input == nil {
+ return nil, errors.New("CreateSignedUrlInput is nil")
+ }
+
+ params := make(map[string]string, len(input.QueryParams))
+ for key, value := range input.QueryParams {
+ params[key] = value
+ }
+
+ if input.SubResource != "" {
+ params[string(input.SubResource)] = ""
+ }
+
+ headers := make(map[string][]string, len(input.Headers))
+ for key, value := range input.Headers {
+ headers[key] = []string{value}
+ }
+
+ if input.Expires <= 0 {
+ input.Expires = 300
+ }
+
+ requestUrl, err := obsClient.doAuthTemporary(string(input.Method), input.Bucket, input.Key, params, headers, int64(input.Expires))
+ if err != nil {
+ return nil, err
+ }
+
+ output = &CreateSignedUrlOutput{
+ SignedUrl: requestUrl,
+ ActualSignedRequestHeaders: headers,
+ }
+ return
+}
+
+func (obsClient ObsClient) CreateBrowserBasedSignature(input *CreateBrowserBasedSignatureInput) (output *CreateBrowserBasedSignatureOutput, err error) {
+ if input == nil {
+ return nil, errors.New("CreateBrowserBasedSignatureInput is nil")
+ }
+
+ params := make(map[string]string, len(input.FormParams))
+ for key, value := range input.FormParams {
+ params[key] = value
+ }
+
+ date := time.Now().UTC()
+ shortDate := date.Format(SHORT_DATE_FORMAT)
+ longDate := date.Format(LONG_DATE_FORMAT)
+
+ credential, _ := getCredential(obsClient.conf.securityProvider.ak, obsClient.conf.region, shortDate)
+
+ if input.Expires <= 0 {
+ input.Expires = 300
+ }
+
+ expiration := date.Add(time.Second * time.Duration(input.Expires)).Format(ISO8601_DATE_FORMAT)
+ params[PARAM_ALGORITHM_AMZ_CAMEL] = V4_HASH_PREFIX
+ params[PARAM_CREDENTIAL_AMZ_CAMEL] = credential
+ params[PARAM_DATE_AMZ_CAMEL] = longDate
+
+ if obsClient.conf.securityProvider.securityToken != "" {
+ params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken
+ }
+
+ matchAnyBucket := true
+ matchAnyKey := true
+ count := 5
+ if bucket := strings.TrimSpace(input.Bucket); bucket != "" {
+ params["bucket"] = bucket
+ matchAnyBucket = false
+ count--
+ }
+
+ if key := strings.TrimSpace(input.Key); key != "" {
+ params["key"] = key
+ matchAnyKey = false
+ count--
+ }
+
+ originPolicySlice := make([]string, 0, len(params)+count)
+ originPolicySlice = append(originPolicySlice, fmt.Sprintf("{\"expiration\":\"%s\",", expiration))
+ originPolicySlice = append(originPolicySlice, "\"conditions\":[")
+ for key, value := range params {
+ if _key := strings.TrimSpace(strings.ToLower(key)); _key != "" {
+ originPolicySlice = append(originPolicySlice, fmt.Sprintf("{\"%s\":\"%s\"},", _key, value))
+ }
+ }
+
+ if matchAnyBucket {
+ originPolicySlice = append(originPolicySlice, "[\"starts-with\", \"$bucket\", \"\"],")
+ }
+
+ if matchAnyKey {
+ originPolicySlice = append(originPolicySlice, "[\"starts-with\", \"$key\", \"\"],")
+ }
+
+ originPolicySlice = append(originPolicySlice, "]}")
+
+ originPolicy := strings.Join(originPolicySlice, "")
+ policy := Base64Encode([]byte(originPolicy))
+ signature := getSignature(policy, obsClient.conf.securityProvider.sk, obsClient.conf.region, shortDate)
+
+ output = &CreateBrowserBasedSignatureOutput{
+ OriginPolicy: originPolicy,
+ Policy: policy,
+ Algorithm: params[PARAM_ALGORITHM_AMZ_CAMEL],
+ Credential: params[PARAM_CREDENTIAL_AMZ_CAMEL],
+ Date: params[PARAM_DATE_AMZ_CAMEL],
+ Signature: signature,
+ }
+ return
+}
+
+func (obsClient ObsClient) ListBucketsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListBucketsOutput, err error) {
+ output = &ListBucketsOutput{}
+ err = obsClient.doHttpWithSignedUrl("ListBuckets", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) CreateBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("CreateBucket", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("DeleteBucket", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketStoragePolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketStoragePolicy", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketStoragePolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketStoragePolicyOutput, err error) {
+ output = &GetBucketStoragePolicyOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketStoragePolicy", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) ListObjectsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListObjectsOutput, err error) {
+ output = &ListObjectsOutput{}
+ err = obsClient.doHttpWithSignedUrl("ListObjects", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok {
+ output.Location = location[0]
+ }
+ }
+ return
+}
+
+func (obsClient ObsClient) ListVersionsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListVersionsOutput, err error) {
+ output = &ListVersionsOutput{}
+ err = obsClient.doHttpWithSignedUrl("ListVersions", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok {
+ output.Location = location[0]
+ }
+ }
+ return
+}
+
+func (obsClient ObsClient) ListMultipartUploadsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListMultipartUploadsOutput, err error) {
+ output = &ListMultipartUploadsOutput{}
+ err = obsClient.doHttpWithSignedUrl("ListMultipartUploads", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketQuotaWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketQuota", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketQuotaWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketQuotaOutput, err error) {
+ output = &GetBucketQuotaOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketQuota", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) HeadBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("HeadBucket", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketMetadataWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketMetadataOutput, err error) {
+ output = &GetBucketMetadataOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketMetadata", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParseGetBucketMetadataOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketStorageInfoWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketStorageInfoOutput, err error) {
+ output = &GetBucketStorageInfoOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketStorageInfo", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketLocationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLocationOutput, err error) {
+ output = &GetBucketLocationOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketLocation", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketAcl", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketAclOutput, err error) {
+ output = &GetBucketAclOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketAcl", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketPolicy", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketPolicyOutput, err error) {
+ output = &GetBucketPolicyOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketPolicy", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, false)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("DeleteBucketPolicy", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketCors", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketCorsOutput, err error) {
+ output = &GetBucketCorsOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketCors", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("DeleteBucketCors", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketVersioningWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketVersioning", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketVersioningWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketVersioningOutput, err error) {
+ output = &GetBucketVersioningOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketVersioning", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketWebsiteConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketWebsiteConfigurationOutput, err error) {
+ output = &GetBucketWebsiteConfigurationOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketWebsiteConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("DeleteBucketWebsiteConfiguration", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketLoggingConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketLoggingConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketLoggingConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLoggingConfigurationOutput, err error) {
+ output = &GetBucketLoggingConfigurationOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketLoggingConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketLifecycleConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLifecycleConfigurationOutput, err error) {
+ output = &GetBucketLifecycleConfigurationOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketLifecycleConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("DeleteBucketLifecycleConfiguration", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketTagging", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketTaggingOutput, err error) {
+ output = &GetBucketTaggingOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketTagging", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("DeleteBucketTagging", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetBucketNotificationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetBucketNotification", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetBucketNotificationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketNotificationOutput, err error) {
+ output = &GetBucketNotificationOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetBucketNotification", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *DeleteObjectOutput, err error) {
+ output = &DeleteObjectOutput{}
+ err = obsClient.doHttpWithSignedUrl("DeleteObject", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParseDeleteObjectOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) DeleteObjectsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *DeleteObjectsOutput, err error) {
+ output = &DeleteObjectsOutput{}
+ err = obsClient.doHttpWithSignedUrl("DeleteObjects", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) SetObjectAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("SetObjectAcl", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetObjectAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectAclOutput, err error) {
+ output = &GetObjectAclOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetObjectAcl", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ if versionId, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok {
+ output.VersionId = versionId[0]
+ }
+ }
+ return
+}
+
+func (obsClient ObsClient) RestoreObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("RestoreObject", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) GetObjectMetadataWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectMetadataOutput, err error) {
+ output = &GetObjectMetadataOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetObjectMetadata", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParseGetObjectMetadataOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) GetObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectOutput, err error) {
+ output = &GetObjectOutput{}
+ err = obsClient.doHttpWithSignedUrl("GetObject", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParseGetObjectOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) PutObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *PutObjectOutput, err error) {
+ output = &PutObjectOutput{}
+ err = obsClient.doHttpWithSignedUrl("PutObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParsePutObjectOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) PutFileWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, sourceFile string) (output *PutObjectOutput, err error) {
+ var data io.Reader
+ sourceFile = strings.TrimSpace(sourceFile)
+ if sourceFile != "" {
+ fd, err := os.Open(sourceFile)
+ if err != nil {
+ return nil, err
+ }
+ defer fd.Close()
+
+ stat, err := fd.Stat()
+ if err != nil {
+ return nil, err
+ }
+ fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile}
+ fileReaderWrapper.reader = fd
+
+ var contentLength int64
+ if value, ok := actualSignedRequestHeaders[HEADER_CONTENT_LENGTH_CAMEL]; ok {
+ contentLength = StringToInt64(value[0], -1)
+ } else if value, ok := actualSignedRequestHeaders[HEADER_CONTENT_LENGTH]; ok {
+ contentLength = StringToInt64(value[0], -1)
+ } else {
+ contentLength = stat.Size()
+ }
+ if contentLength > stat.Size() {
+ return nil, errors.New("ContentLength is larger than fileSize")
+ }
+ fileReaderWrapper.totalCount = contentLength
+ data = fileReaderWrapper
+ }
+
+ output = &PutObjectOutput{}
+ err = obsClient.doHttpWithSignedUrl("PutObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParsePutObjectOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) CopyObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *CopyObjectOutput, err error) {
+ output = &CopyObjectOutput{}
+ err = obsClient.doHttpWithSignedUrl("CopyObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParseCopyObjectOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) AbortMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) {
+ output = &BaseModel{}
+ err = obsClient.doHttpWithSignedUrl("AbortMultipartUpload", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) InitiateMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *InitiateMultipartUploadOutput, err error) {
+ output = &InitiateMultipartUploadOutput{}
+ err = obsClient.doHttpWithSignedUrl("InitiateMultipartUpload", HTTP_POST, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParseInitiateMultipartUploadOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) UploadPartWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *UploadPartOutput, err error) {
+ output = &UploadPartOutput{}
+ err = obsClient.doHttpWithSignedUrl("UploadPart", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParseUploadPartOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) CompleteMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *CompleteMultipartUploadOutput, err error) {
+ output = &CompleteMultipartUploadOutput{}
+ err = obsClient.doHttpWithSignedUrl("CompleteMultipartUpload", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParseCompleteMultipartUploadOutput(output)
+ }
+ return
+}
+
+func (obsClient ObsClient) ListPartsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListPartsOutput, err error) {
+ output = &ListPartsOutput{}
+ err = obsClient.doHttpWithSignedUrl("ListParts", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ }
+ return
+}
+
+func (obsClient ObsClient) CopyPartWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *CopyPartOutput, err error) {
+ output = &CopyPartOutput{}
+ err = obsClient.doHttpWithSignedUrl("CopyPart", HTTP_PUT, signedUrl, actualSignedRequestHeaders, nil, output, true)
+ if err != nil {
+ output = nil
+ } else {
+ ParseCopyPartOutput(output)
+ }
+ return
+}
+
+
+
package obs
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "os"
+ "strings"
+)
+
+type IReadCloser interface {
+ setReadCloser(body io.ReadCloser)
+}
+
+func (output *GetObjectOutput) setReadCloser(body io.ReadCloser) {
+ output.Body = body
+}
+
+type IBaseModel interface {
+ setStatusCode(statusCode int)
+
+ setRequestId(requestId string)
+
+ setResponseHeaders(responseHeaders map[string][]string)
+}
+
+type ISerializable interface {
+ trans() (map[string]string, map[string][]string, interface{})
+}
+
+type DefaultSerializable struct {
+ params map[string]string
+ headers map[string][]string
+ data interface{}
+}
+
+func (input DefaultSerializable) trans() (map[string]string, map[string][]string, interface{}) {
+ return input.params, input.headers, input.data
+}
+
+var defaultSerializable = &DefaultSerializable{}
+
+func newSubResourceSerial(subResource SubResourceType) *DefaultSerializable {
+ return &DefaultSerializable{map[string]string{string(subResource): ""}, nil, nil}
+}
+
+func trans(subResource SubResourceType, input interface{}) (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(subResource): ""}
+ data, _ = ConvertRequestToIoReader(input)
+ return
+}
+
+func (baseModel *BaseModel) setStatusCode(statusCode int) {
+ baseModel.StatusCode = statusCode
+}
+
+func (baseModel *BaseModel) setRequestId(requestId string) {
+ baseModel.RequestId = requestId
+}
+
+func (baseModel *BaseModel) setResponseHeaders(responseHeaders map[string][]string) {
+ baseModel.ResponseHeaders = responseHeaders
+}
+
+func (input ListBucketsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ headers = make(map[string][]string)
+ if input.QueryLocation {
+ headers[HEADER_LOCATION_AMZ] = []string{"true"}
+ }
+ return
+}
+
+func (input CreateBucketInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ headers = make(map[string][]string)
+ if acl := string(input.ACL); acl != "" {
+ headers[HEADER_ACL_AMZ] = []string{acl}
+ }
+
+ if storageClass := string(input.StorageClass); storageClass != "" {
+ headers[HEADER_STORAGE_CLASS] = []string{storageClass}
+ }
+
+ if location := strings.TrimSpace(input.Location); location != "" {
+ input.Location = location
+ data, _ = ConvertRequestToIoReader(input)
+ }
+ return
+}
+
+func (input SetBucketStoragePolicyInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ return trans(SubResourceStoragePolicy, input)
+}
+
+func (input ListObjsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = make(map[string]string)
+ if input.Prefix != "" {
+ params["prefix"] = input.Prefix
+ }
+ if input.Delimiter != "" {
+ params["delimiter"] = input.Delimiter
+ }
+ if input.MaxKeys > 0 {
+ params["max-keys"] = IntToString(input.MaxKeys)
+ }
+ headers = make(map[string][]string)
+ if origin := strings.TrimSpace(input.Origin); origin != "" {
+ headers[HEADER_ORIGIN_CAMEL] = []string{origin}
+ }
+ if requestHeader := strings.TrimSpace(input.RequestHeader); requestHeader != "" {
+ headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{requestHeader}
+ }
+ return
+}
+
+func (input ListObjectsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params, headers, data = input.ListObjsInput.trans()
+ if input.Marker != "" {
+ params["marker"] = input.Marker
+ }
+ return
+}
+
+func (input ListVersionsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params, headers, data = input.ListObjsInput.trans()
+ params[string(SubResourceVersions)] = ""
+ if input.KeyMarker != "" {
+ params["key-marker"] = input.KeyMarker
+ }
+ if input.VersionIdMarker != "" {
+ params["version-id-marker"] = input.VersionIdMarker
+ }
+ return
+}
+
+func (input ListMultipartUploadsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceUploads): ""}
+ if input.Prefix != "" {
+ params["prefix"] = input.Prefix
+ }
+ if input.Delimiter != "" {
+ params["delimiter"] = input.Delimiter
+ }
+ if input.MaxUploads > 0 {
+ params["max-uploads"] = IntToString(input.MaxUploads)
+ }
+ if input.KeyMarker != "" {
+ params["key-marker"] = input.KeyMarker
+ }
+ if input.UploadIdMarker != "" {
+ params["upload-id-marker"] = input.UploadIdMarker
+ }
+ return
+}
+
+func (input SetBucketQuotaInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ return trans(SubResourceQuota, input)
+}
+
+func (input SetBucketAclInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceAcl): ""}
+ headers = make(map[string][]string)
+
+ if acl := string(input.ACL); acl != "" {
+ headers[HEADER_ACL_AMZ] = []string{acl}
+ } else {
+ data, _ = ConvertAclToXml(input.AccessControlPolicy, false)
+ }
+ return
+}
+
+func (input SetBucketPolicyInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourcePolicy): ""}
+ data = strings.NewReader(input.Policy)
+ return
+}
+
+func (input SetBucketCorsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceCors): ""}
+ data, md5, _ := ConvertRequestToIoReaderV2(input)
+ headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}}
+ return
+}
+
+func (input SetBucketVersioningInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ return trans(SubResourceVersioning, input)
+}
+
+func (input SetBucketWebsiteConfigurationInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceWebsite): ""}
+ data, _ = ConvertWebsiteConfigurationToXml(input.BucketWebsiteConfiguration, false)
+ return
+}
+
+func (input GetBucketMetadataInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ headers = make(map[string][]string)
+ if origin := strings.TrimSpace(input.Origin); origin != "" {
+ headers[HEADER_ORIGIN_CAMEL] = []string{origin}
+ }
+ if requestHeader := strings.TrimSpace(input.RequestHeader); requestHeader != "" {
+ headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{requestHeader}
+ }
+ return
+}
+
+func (input SetBucketLoggingConfigurationInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceLogging): ""}
+ data, _ = ConvertLoggingStatusToXml(input.BucketLoggingStatus, false)
+ return
+}
+
+func (input SetBucketLifecycleConfigurationInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceLifecycle): ""}
+ data, md5 := ConvertLifecyleConfigurationToXml(input.BucketLifecyleConfiguration, true)
+ headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}}
+ return
+}
+
+func (input SetBucketTaggingInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceTagging): ""}
+ data, md5, _ := ConvertRequestToIoReaderV2(input)
+ headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}}
+ return
+}
+
+func (input SetBucketNotificationInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceNotification): ""}
+ data, _ = ConvertNotificationToXml(input.BucketNotification, false)
+ return
+}
+
+func (input DeleteObjectInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = make(map[string]string)
+ if input.VersionId != "" {
+ params[PARAM_VERSION_ID] = input.VersionId
+ }
+ return
+}
+
+func (input DeleteObjectsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceDelete): ""}
+ data, md5, _ := ConvertRequestToIoReaderV2(input)
+ headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}}
+ return
+}
+
+func (input SetObjectAclInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceAcl): ""}
+ if input.VersionId != "" {
+ params[PARAM_VERSION_ID] = input.VersionId
+ }
+ headers = make(map[string][]string)
+ if acl := string(input.ACL); acl != "" {
+ headers[HEADER_ACL_AMZ] = []string{acl}
+ } else {
+ data, _ = ConvertAclToXml(input.AccessControlPolicy, false)
+ }
+ return
+}
+
+func (input GetObjectAclInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceAcl): ""}
+ if input.VersionId != "" {
+ params[PARAM_VERSION_ID] = input.VersionId
+ }
+ return
+}
+
+func (input RestoreObjectInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{string(SubResourceRestore): ""}
+ if input.VersionId != "" {
+ params[PARAM_VERSION_ID] = input.VersionId
+ }
+ data, _ = ConvertRequestToIoReader(input)
+ return
+}
+
+func (header SseKmsHeader) GetEncryption() string {
+ if header.Encryption != "" {
+ return header.Encryption
+ }
+ return DEFAULT_SSE_KMS_ENCRYPTION
+}
+
+func (header SseKmsHeader) GetKey() string {
+ return header.Key
+}
+
+func (header SseCHeader) GetEncryption() string {
+ if header.Encryption != "" {
+ return header.Encryption
+ }
+ return DEFAULT_SSE_C_ENCRYPTION
+}
+
+func (header SseCHeader) GetKey() string {
+ return header.Key
+}
+
+func (header SseCHeader) GetKeyMD5() string {
+ if header.KeyMD5 != "" {
+ return header.KeyMD5
+ }
+
+ if ret, err := Base64Decode(header.GetKey()); err == nil {
+ return Base64Md5(ret)
+ }
+ return ""
+}
+
+func setSseHeader(headers map[string][]string, sseHeader ISseHeader, sseCOnly bool) {
+ if sseHeader != nil {
+ if sseCHeader, ok := sseHeader.(SseCHeader); ok {
+ headers[HEADER_SSEC_ENCRYPTION_AMZ] = []string{sseCHeader.GetEncryption()}
+ headers[HEADER_SSEC_KEY_AMZ] = []string{sseCHeader.GetKey()}
+ headers[HEADER_SSEC_KEY_MD5_AMZ] = []string{sseCHeader.GetKeyMD5()}
+ } else if sseKmsHeader, ok := sseHeader.(SseKmsHeader); !sseCOnly && ok {
+ headers[HEADER_SSEKMS_ENCRYPTION_AMZ] = []string{sseKmsHeader.GetEncryption()}
+ headers[HEADER_SSEKMS_KEY_AMZ] = []string{sseKmsHeader.GetKey()}
+ }
+ }
+}
+
+func (input GetObjectMetadataInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = make(map[string]string)
+ if input.VersionId != "" {
+ params[PARAM_VERSION_ID] = input.VersionId
+ }
+ headers = make(map[string][]string)
+
+ if input.Origin != "" {
+ headers[HEADER_ORIGIN_CAMEL] = []string{input.Origin}
+ }
+
+ if input.RequestHeader != "" {
+ headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{input.RequestHeader}
+ }
+ setSseHeader(headers, input.SseHeader, true)
+ return
+}
+
+func (input GetObjectInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params, headers, data = input.GetObjectMetadataInput.trans()
+ if input.ResponseCacheControl != "" {
+ params[PARAM_RESPONSE_CACHE_CONTROL] = input.ResponseCacheControl
+ }
+ if input.ResponseContentDisposition != "" {
+ params[PARAM_RESPONSE_CONTENT_DISPOSITION] = input.ResponseContentDisposition
+ }
+ if input.ResponseContentEncoding != "" {
+ params[PARAM_RESPONSE_CONTENT_ENCODING] = input.ResponseContentEncoding
+ }
+ if input.ResponseContentLanguage != "" {
+ params[PARAM_RESPONSE_CONTENT_LANGUAGE] = input.ResponseContentLanguage
+ }
+ if input.ResponseContentType != "" {
+ params[PARAM_RESPONSE_CONTENT_TYPE] = input.ResponseContentType
+ }
+ if input.ResponseExpires != "" {
+ params[PARAM_RESPONSE_EXPIRES] = input.ResponseExpires
+ }
+ if input.ImageProcess != "" {
+ params[PARAM_IMAGE_PROCESS] = input.ImageProcess
+ }
+ if input.RangeStart >= 0 && input.RangeEnd > input.RangeStart {
+ headers[HEADER_RANGE] = []string{fmt.Sprintf("bytes=%d-%d", input.RangeStart, input.RangeEnd)}
+ }
+
+ if input.IfMatch != "" {
+ headers[HEADER_IF_MATCH] = []string{input.IfMatch}
+ }
+ if input.IfNoneMatch != "" {
+ headers[HEADER_IF_NONE_MATCH] = []string{input.IfNoneMatch}
+ }
+ if !input.IfModifiedSince.IsZero() {
+ headers[HEADER_IF_MODIFIED_SINCE] = []string{FormatUtcToRfc1123(input.IfModifiedSince)}
+ }
+ if !input.IfUnmodifiedSince.IsZero() {
+ headers[HEADER_IF_UNMODIFIED_SINCE] = []string{FormatUtcToRfc1123(input.IfUnmodifiedSince)}
+ }
+ return
+}
+
+func (input ObjectOperationInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ headers = make(map[string][]string)
+ params = make(map[string]string)
+ if acl := string(input.ACL); acl != "" {
+ headers[HEADER_ACL_AMZ] = []string{acl}
+ }
+ if storageClass := string(input.StorageClass); storageClass != "" {
+ headers[HEADER_STORAGE_CLASS2_AMZ] = []string{storageClass}
+ }
+ if input.WebsiteRedirectLocation != "" {
+ headers[HEADER_WEBSITE_REDIRECT_LOCATION_AMZ] = []string{input.WebsiteRedirectLocation}
+ }
+ setSseHeader(headers, input.SseHeader, false)
+ if input.Metadata != nil {
+ for key, value := range input.Metadata {
+ key = strings.TrimSpace(key)
+ if !strings.HasPrefix(key, HEADER_PREFIX_META) {
+ key = HEADER_PREFIX_META + key
+ }
+ headers[key] = []string{value}
+ }
+ }
+ return
+}
+
+func (input PutObjectBasicInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params, headers, data = input.ObjectOperationInput.trans()
+
+ if input.ContentMD5 != "" {
+ headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5}
+ }
+
+ if input.ContentLength > 0 {
+ headers[HEADER_CONTENT_LENGTH_CAMEL] = []string{Int64ToString(input.ContentLength)}
+ }
+ if input.ContentType != "" {
+ headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType}
+ }
+
+ return
+}
+
+func (input PutObjectInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params, headers, data = input.PutObjectBasicInput.trans()
+ if input.Body != nil {
+ data = input.Body
+ }
+ return
+}
+
+func (input CopyObjectInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params, headers, data = input.ObjectOperationInput.trans()
+
+ var copySource string
+ if input.CopySourceVersionId != "" {
+ copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false), input.CopySourceVersionId)
+ } else {
+ copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false))
+ }
+ headers[HEADER_COPY_SOURCE_AMZ] = []string{copySource}
+
+ if directive := string(input.MetadataDirective); directive != "" {
+ headers[HEADER_METADATA_DIRECTIVE_AMZ] = []string{directive}
+ }
+
+ if input.MetadataDirective == ReplaceMetadata {
+ if input.CacheControl != "" {
+ headers[HEADER_CACHE_CONTROL] = []string{input.CacheControl}
+ }
+ if input.ContentDisposition != "" {
+ headers[HEADER_CONTENT_DISPOSITION] = []string{input.ContentDisposition}
+ }
+ if input.ContentEncoding != "" {
+ headers[HEADER_CONTENT_ENCODING] = []string{input.ContentEncoding}
+ }
+ if input.ContentLanguage != "" {
+ headers[HEADER_CONTENT_LANGUAGE] = []string{input.ContentLanguage}
+ }
+ if input.ContentType != "" {
+ headers[HEADER_CONTENT_TYPE] = []string{input.ContentType}
+ }
+ if input.Expires != "" {
+ headers[HEADER_EXPIRES] = []string{input.Expires}
+ }
+ }
+
+ if input.CopySourceIfMatch != "" {
+ headers[HEADER_COPY_SOURCE_IF_MATCH_AMZ] = []string{input.CopySourceIfMatch}
+ }
+ if input.CopySourceIfNoneMatch != "" {
+ headers[HEADER_COPY_SOURCE_IF_NONE_MATCH_AMZ] = []string{input.CopySourceIfNoneMatch}
+ }
+ if !input.CopySourceIfModifiedSince.IsZero() {
+ headers[HEADER_COPY_SOURCE_IF_MODIFIED_SINCE_AMZ] = []string{FormatUtcToRfc1123(input.CopySourceIfModifiedSince)}
+ }
+ if !input.CopySourceIfUnmodifiedSince.IsZero() {
+ headers[HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE_AMZ] = []string{FormatUtcToRfc1123(input.CopySourceIfUnmodifiedSince)}
+ }
+ if input.SourceSseHeader != nil {
+ if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok {
+ headers[HEADER_SSEC_COPY_SOURCE_ENCRYPTION_AMZ] = []string{sseCHeader.GetEncryption()}
+ headers[HEADER_SSEC_COPY_SOURCE_KEY_AMZ] = []string{sseCHeader.GetKey()}
+ headers[HEADER_SSEC_COPY_SOURCE_KEY_MD5_AMZ] = []string{sseCHeader.GetKeyMD5()}
+ }
+ }
+ return
+}
+
+func (input AbortMultipartUploadInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{"uploadId": input.UploadId}
+ return
+}
+
+func (input InitiateMultipartUploadInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params, headers, data = input.ObjectOperationInput.trans()
+ params[string(SubResourceUploads)] = ""
+ return
+}
+
+func (input UploadPartInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)}
+ headers = make(map[string][]string)
+ setSseHeader(headers, input.SseHeader, true)
+ if input.Body != nil {
+ data = input.Body
+ }
+ return
+}
+
+func (input CompleteMultipartUploadInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{"uploadId": input.UploadId}
+ data, _ = ConvertCompleteMultipartUploadInputToXml(input, false)
+ return
+}
+
+func (input ListPartsInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{"uploadId": input.UploadId}
+ if input.MaxParts > 0 {
+ params["max-parts"] = IntToString(input.MaxParts)
+ }
+ if input.PartNumberMarker > 0 {
+ params["part-number-marker"] = IntToString(input.PartNumberMarker)
+ }
+ return
+}
+
+func (input CopyPartInput) trans() (params map[string]string, headers map[string][]string, data interface{}) {
+ params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)}
+ headers = make(map[string][]string, 1)
+ var copySource string
+ if input.CopySourceVersionId != "" {
+ copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false), input.CopySourceVersionId)
+ } else {
+ copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false))
+ }
+ headers[HEADER_COPY_SOURCE_AMZ] = []string{copySource}
+
+ if input.CopySourceRangeStart >= 0 && input.CopySourceRangeEnd > input.CopySourceRangeStart {
+ headers[HEADER_COPY_SOURCE_RANGE_AMZ] = []string{fmt.Sprintf("bytes=%d-%d", input.CopySourceRangeStart, input.CopySourceRangeEnd)}
+ }
+
+ setSseHeader(headers, input.SseHeader, true)
+ if input.SourceSseHeader != nil {
+ if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok {
+ headers[HEADER_SSEC_COPY_SOURCE_ENCRYPTION_AMZ] = []string{sseCHeader.GetEncryption()}
+ headers[HEADER_SSEC_COPY_SOURCE_KEY_AMZ] = []string{sseCHeader.GetKey()}
+ headers[HEADER_SSEC_COPY_SOURCE_KEY_MD5_AMZ] = []string{sseCHeader.GetKeyMD5()}
+ }
+ }
+ return
+}
+
+type partSlice []Part
+
+func (parts partSlice) Len() int {
+ return len(parts)
+}
+
+func (parts partSlice) Less(i, j int) bool {
+ return parts[i].PartNumber < parts[j].PartNumber
+}
+
+func (parts partSlice) Swap(i, j int) {
+ parts[i], parts[j] = parts[j], parts[i]
+}
+
+type readerWrapper struct {
+ reader io.Reader
+ mark int64
+ totalCount int64
+ readedCount int64
+}
+
+func (rw *readerWrapper) seek(offset int64, whence int) (int64, error) {
+ if r, ok := rw.reader.(*strings.Reader); ok {
+ return r.Seek(offset, whence)
+ } else if r, ok := rw.reader.(*bytes.Reader); ok {
+ return r.Seek(offset, whence)
+ } else if r, ok := rw.reader.(*os.File); ok {
+ return r.Seek(offset, whence)
+ }
+ return offset, nil
+}
+
+func (rw *readerWrapper) Read(p []byte) (n int, err error) {
+ if rw.totalCount == 0 {
+ return 0, io.EOF
+ }
+ if rw.totalCount > 0 {
+ n, err = rw.reader.Read(p)
+ readedOnce := int64(n)
+ if remainCount := rw.totalCount - rw.readedCount; remainCount > readedOnce {
+ rw.readedCount += readedOnce
+ return n, err
+ } else {
+ rw.readedCount += remainCount
+ return int(remainCount), io.EOF
+ }
+ }
+ return rw.reader.Read(p)
+}
+
+type fileReaderWrapper struct {
+ readerWrapper
+ filePath string
+}
+
+
+
package obs
+
+import (
+ "crypto/hmac"
+ "crypto/md5"
+ "crypto/sha1"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/xml"
+ "fmt"
+ "net/url"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+)
+
+var regex = regexp.MustCompile("^[\u4e00-\u9fa5]$")
+var ipRegex = regexp.MustCompile("^((2[0-4]\\d|25[0-5]|[01]?\\d\\d?)\\.){3}(2[0-4]\\d|25[0-5]|[01]?\\d\\d?)$")
+var v4AuthRegex = regexp.MustCompile("Credential=(.+?),SignedHeaders=(.+?),Signature=.+")
+var regionRegex = regexp.MustCompile(".+/\\d+/(.+?)/.+")
+
+func StringToInt(value string, def int) int {
+ ret, err := strconv.Atoi(value)
+ if err != nil {
+ ret = def
+ }
+ return ret
+}
+
+func StringToInt64(value string, def int64) int64 {
+ ret, err := strconv.ParseInt(value, 10, 64)
+ if err != nil {
+ ret = def
+ }
+ return ret
+}
+
+func IntToString(value int) string {
+ return strconv.Itoa(value)
+}
+
+func Int64ToString(value int64) string {
+ return strconv.FormatInt(value, 10)
+}
+
+func GetCurrentTimestamp() int64 {
+ return time.Now().UnixNano() / 1000000
+}
+
+func FormatUtcNow(format string) string {
+ return time.Now().UTC().Format(format)
+}
+
+func FormatUtcToRfc1123(t time.Time) string {
+ ret := t.UTC().Format(time.RFC1123)
+ return ret[:strings.LastIndex(ret, "UTC")] + "GMT"
+}
+
+func Md5(value []byte) []byte {
+ m := md5.New()
+ m.Write(value)
+ return m.Sum(nil)
+}
+
+func HmacSha1(key, value []byte) []byte {
+ mac := hmac.New(sha1.New, key)
+ mac.Write(value)
+ return mac.Sum(nil)
+}
+
+func HmacSha256(key, value []byte) []byte {
+ mac := hmac.New(sha256.New, key)
+ mac.Write(value)
+ return mac.Sum(nil)
+}
+
+func Base64Encode(value []byte) string {
+ return base64.StdEncoding.EncodeToString(value)
+}
+
+func Base64Decode(value string) ([]byte, error) {
+ return base64.StdEncoding.DecodeString(value)
+}
+
+func HexMd5(value []byte) string {
+ return Hex(Md5(value))
+}
+
+func Base64Md5(value []byte) string {
+ return Base64Encode(Md5(value))
+}
+
+func Sha256Hash(value []byte) []byte {
+ hash := sha256.New()
+ hash.Write(value)
+ return hash.Sum(nil)
+}
+
+func ParseXml(value []byte, result interface{}) error {
+ if len(value) == 0 {
+ return nil
+ }
+ return xml.Unmarshal(value, result)
+}
+
+func TransToXml(value interface{}) ([]byte, error) {
+ if value == nil {
+ return []byte{}, nil
+ }
+ return xml.Marshal(value)
+}
+
+func Hex(value []byte) string {
+ return hex.EncodeToString(value)
+}
+
+func HexSha256(value []byte) string {
+ return Hex(Sha256Hash(value))
+}
+
+func UrlDecode(value string) (string, error) {
+ ret, err := url.QueryUnescape(value)
+ if err == nil {
+ return ret, nil
+ }
+ return "", err
+}
+
+func IsIP(value string) bool {
+ return ipRegex.MatchString(value)
+}
+
+func UrlEncode(value string, chineseOnly bool) string {
+ if chineseOnly {
+ values := make([]string, 0, len(value))
+ for _, val := range value {
+ _value := string(val)
+ if regex.MatchString(_value) {
+ _value = url.QueryEscape(_value)
+ }
+ values = append(values, _value)
+ }
+ return strings.Join(values, "")
+ }
+ return url.QueryEscape(value)
+}
+
+func copyHeaders(m map[string][]string) (ret map[string][]string) {
+ if m != nil {
+ ret = make(map[string][]string, len(m))
+ for key, values := range m {
+ _values := make([]string, 0, len(values))
+ for _, value := range values {
+ _values = append(_values, value)
+ }
+ ret[strings.ToLower(key)] = _values
+ }
+ } else {
+ ret = make(map[string][]string)
+ }
+
+ return
+}
+
+func parseHeaders(headers map[string][]string) (signature string, region string, signedHeaders string) {
+ signature = "v2"
+ if receviedAuthorization, ok := headers[strings.ToLower(HEADER_AUTH_CAMEL)]; ok && len(receviedAuthorization) > 0 {
+ if strings.HasPrefix(receviedAuthorization[0], V4_HASH_PREFIX) {
+ signature = "v4"
+ matches := v4AuthRegex.FindStringSubmatch(receviedAuthorization[0])
+ if len(matches) >= 3 {
+ region = matches[1]
+ regions := regionRegex.FindStringSubmatch(region)
+ if len(regions) >= 2 {
+ region = regions[1]
+ }
+ signedHeaders = matches[2]
+ }
+
+ } else if strings.HasPrefix(receviedAuthorization[0], V2_HASH_PREFIX) {
+ signature = "v2"
+ }
+ }
+ return
+}
+
+func getTemporaryKeys() []string {
+ return []string{
+ "Signature",
+ "signature",
+ "X-Amz-Signature",
+ "x-amz-signature",
+ }
+}
+
+func GetAuthorization(ak, sk, method, bucketName, objectKey, queryUrl string, headers map[string][]string) (ret map[string]string) {
+
+ if strings.HasPrefix(queryUrl, "?") {
+ queryUrl = queryUrl[1:]
+ }
+
+ method = strings.ToUpper(method)
+
+ querys := strings.Split(queryUrl, "&")
+
+ params := make(map[string]string)
+
+ for _, value := range querys {
+ kv := strings.Split(value, "=")
+ length := len(kv)
+ if length == 1 {
+ key, _ := UrlDecode(kv[0])
+ params[key] = ""
+ } else if length >= 2 {
+ key, _ := UrlDecode(kv[0])
+ vals := make([]string, 0, length-1)
+ for i := 1; i < length; i++ {
+ val, _ := UrlDecode(kv[i])
+ vals = append(vals, val)
+ }
+ params[key] = strings.Join(vals, "=")
+ }
+ }
+
+ isTemporary := false
+ signature := "v2"
+ temporaryKeys := getTemporaryKeys()
+ for _, key := range temporaryKeys {
+ if _, ok := params[key]; ok {
+ isTemporary = true
+ if strings.ToLower(key) == "signature" {
+ signature = "v2"
+ } else if strings.ToLower(key) == "x-amz-signature" {
+ signature = "v4"
+ }
+ break
+ }
+ }
+ headers = copyHeaders(headers)
+ pathStyle := false
+ if receviedHost, ok := headers[HEADER_HOST]; ok && len(receviedHost) > 0 && !strings.HasPrefix(receviedHost[0], bucketName+".") {
+ pathStyle = true
+ }
+ conf := &config{securityProvider: &securityProvider{ak: ak, sk: sk},
+ urlHolder: &urlHolder{scheme: "https", host: "dummy", port: 443},
+ pathStyle: pathStyle}
+
+ if isTemporary {
+ return getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature, conf, params, headers)
+ } else {
+ signature, region, signedHeaders := parseHeaders(headers)
+ if signature == "v4" {
+ conf.signature = SignatureV4
+ requestUrl, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false)
+ parsedRequestUrl, _ := url.Parse(requestUrl)
+ headerKeys := strings.Split(signedHeaders, ";")
+ _headers := make(map[string][]string, len(headerKeys))
+ for _, headerKey := range headerKeys {
+ _headers[headerKey] = headers[headerKey]
+ }
+ ret = v4Auth(ak, sk, region, method, canonicalizedUrl, parsedRequestUrl.RawQuery, _headers)
+ ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"])
+ } else if signature == "v2" {
+ conf.signature = SignatureV2
+ _, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false)
+ ret = v2Auth(ak, sk, method, canonicalizedUrl, headers)
+ ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", V2_HASH_PREFIX, ak, ret["Signature"])
+ }
+ return
+ }
+
+}
+
+func getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature string, conf *config, params map[string]string,
+ headers map[string][]string) (ret map[string]string) {
+
+ if signature == "v4" {
+ conf.signature = SignatureV4
+
+ longDate, ok := params[PARAM_DATE_AMZ_CAMEL]
+ if !ok {
+ longDate = params[HEADER_DATE_AMZ]
+ }
+ shortDate := longDate[:8]
+
+ credential, ok := params[PARAM_CREDENTIAL_AMZ_CAMEL]
+ if !ok {
+ credential = params[strings.ToLower(PARAM_CREDENTIAL_AMZ_CAMEL)]
+ }
+
+ _credential, _ := UrlDecode(credential)
+
+ regions := regionRegex.FindStringSubmatch(_credential)
+ var region string
+ if len(regions) >= 2 {
+ region = regions[1]
+ }
+
+ _, scope := getCredential(ak, region, shortDate)
+
+ expires, ok := params[PARAM_EXPIRES_AMZ_CAMEL]
+ if !ok {
+ expires = params[strings.ToLower(PARAM_EXPIRES_AMZ_CAMEL)]
+ }
+
+ signedHeaders, ok := params[PARAM_SIGNEDHEADERS_AMZ_CAMEL]
+ if !ok {
+ signedHeaders = params[strings.ToLower(PARAM_SIGNEDHEADERS_AMZ_CAMEL)]
+ }
+
+ algorithm, ok := params[PARAM_ALGORITHM_AMZ_CAMEL]
+ if !ok {
+ algorithm = params[strings.ToLower(PARAM_ALGORITHM_AMZ_CAMEL)]
+ }
+
+ if _, ok := params[PARAM_SIGNATURE_AMZ_CAMEL]; ok {
+ delete(params, PARAM_SIGNATURE_AMZ_CAMEL)
+ } else if _, ok := params[strings.ToLower(PARAM_SIGNATURE_AMZ_CAMEL)]; ok {
+ delete(params, strings.ToLower(PARAM_SIGNATURE_AMZ_CAMEL))
+ }
+
+ ret = make(map[string]string, 6)
+ ret[PARAM_ALGORITHM_AMZ_CAMEL] = algorithm
+ ret[PARAM_CREDENTIAL_AMZ_CAMEL] = credential
+ ret[PARAM_DATE_AMZ_CAMEL] = longDate
+ ret[PARAM_EXPIRES_AMZ_CAMEL] = expires
+ ret[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = signedHeaders
+
+ requestUrl, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false)
+ parsedRequestUrl, _ := url.Parse(requestUrl)
+ stringToSign := getV4StringToSign(method, canonicalizedUrl, parsedRequestUrl.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, strings.Split(signedHeaders, ";"), headers)
+ ret[PARAM_SIGNATURE_AMZ_CAMEL] = UrlEncode(getSignature(stringToSign, sk, region, shortDate), false)
+ } else if signature == "v2" {
+ conf.signature = SignatureV2
+ _, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false)
+ expires, ok := params["Expires"]
+ if !ok {
+ expires = params["expires"]
+ }
+ headers[HEADER_DATE_CAMEL] = []string{expires}
+ stringToSign := getV2StringToSign(method, canonicalizedUrl, headers)
+ ret = make(map[string]string, 3)
+ ret["Signature"] = UrlEncode(Base64Encode(HmacSha1([]byte(sk), []byte(stringToSign))), false)
+ ret["AWSAccessKeyId"] = UrlEncode(ak, false)
+ ret["Expires"] = UrlEncode(expires, false)
+ }
+
+ return
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+ "encoding/json"
+ "errors"
+ "math/rand"
+ "os"
+ "reflect"
+
+ log "github.com/sirupsen/logrus"
+)
+
+//remove redundant elements
+func RvRepElement(arr []string) []string {
+ result := []string{}
+ for i := 0; i < len(arr); i++ {
+ flag := true
+ for j := range result {
+ if arr[i] == result[j] {
+ flag = false
+ break
+ }
+ }
+ if flag == true {
+ result = append(result, arr[i])
+ }
+ }
+ return result
+}
+
+func Contained(obj, target interface{}) bool {
+ targetValue := reflect.ValueOf(target)
+ switch reflect.TypeOf(target).Kind() {
+ case reflect.Slice, reflect.Array:
+ for i := 0; i < targetValue.Len(); i++ {
+ if targetValue.Index(i).Interface() == obj {
+ return true
+ }
+ }
+ case reflect.Map:
+ if targetValue.MapIndex(reflect.ValueOf(obj)).IsValid() {
+ return true
+ }
+ default:
+ return false
+ }
+ return false
+}
+
+func MergeGeneralMaps(maps ...map[string]interface{}) map[string]interface{} {
+ var out = make(map[string]interface{})
+ for _, m := range maps {
+ for k, v := range m {
+ out[k] = v
+ }
+ }
+ return out
+}
+
+func MergeStringMaps(maps ...map[string]string) map[string]string {
+ var out = make(map[string]string)
+ for _, m := range maps {
+ for k, v := range m {
+ out[k] = v
+ }
+ }
+ return out
+}
+
+func PathExists(path string) (bool, error) {
+ _, err := os.Stat(path)
+ if err == nil {
+ return true, nil
+ }
+ if os.IsNotExist(err) {
+ return false, nil
+ }
+ return false, err
+}
+
+func Retry(retryNum int, desc string, silent bool, fn func(retryIdx int, lastErr error) error) error {
+ var err error
+ for i := 0; i < retryNum; i++ {
+ if err = fn(i, err); err != nil {
+ if !silent {
+ log.Errorf("%s:%s, retry %d time(s)", desc, err, i+1)
+ }
+ } else {
+ return nil
+ }
+ }
+ if !silent {
+ log.Errorf("%s retry exceed the max retry times(%d).", desc, retryNum)
+ }
+ return err
+}
+
+// StructToMap ...
+func StructToMap(structObj interface{}) (map[string]interface{}, error) {
+ jsonStr, err := json.Marshal(structObj)
+ if nil != err {
+ return nil, err
+ }
+
+ var result map[string]interface{}
+ err = json.Unmarshal(jsonStr, &result)
+ if err != nil {
+ return nil, err
+ }
+
+ return result, nil
+}
+
+// Epsilon ...
+const Epsilon float64 = 0.00000001
+
+// IsFloatEqual ...
+func IsFloatEqual(a, b float64) bool {
+ if (a-b) < Epsilon && (b-a) < Epsilon {
+ return true
+ }
+
+ return false
+}
+
+// IsEqual ...
+func IsEqual(key string, value interface{}, reqValue interface{}) (bool, error) {
+ switch value.(type) {
+ case bool:
+ v, ok1 := value.(bool)
+ r, ok2 := reqValue.(bool)
+
+ if ok1 && ok2 {
+ if v == r {
+ return true, nil
+ }
+
+ return false, nil
+ }
+
+ return false, errors.New("the type of " + key + " must be bool")
+ case float64:
+ v, ok1 := value.(float64)
+ r, ok2 := reqValue.(float64)
+
+ if ok1 && ok2 {
+ if IsFloatEqual(v, r) {
+ return true, nil
+ }
+
+ return false, nil
+ }
+
+ return false, errors.New("the type of " + key + " must be float64")
+ case string:
+ v, ok1 := value.(string)
+ r, ok2 := reqValue.(string)
+ if ok1 && ok2 {
+ if v == r {
+ return true, nil
+ }
+
+ return false, nil
+ }
+
+ return false, errors.New("the type of " + key + " must be string")
+ default:
+ return false, errors.New("the type of " + key + " must be bool or float64 or string")
+ }
+}
+
+func RandSeqWithAlnum(n int) string {
+ alnum := []rune("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+ return RandSeq(n, alnum)
+}
+
+func RandSeq(n int, chs []rune) string {
+ b := make([]rune, n)
+ for i := range b {
+ b[i] = chs[rand.Intn(len(chs))]
+ }
+ return string(b)
+}
+
+
+
// Code generated by protoc-gen-micro. DO NOT EDIT.
+// source: backend.proto
+
+/*
+Package backend is a generated protocol buffer package.
+
+It is generated from these files:
+ backend.proto
+
+It has these top-level messages:
+ CreateBackendRequest
+ CreateBackendResponse
+ GetBackendRequest
+ GetBackendResponse
+ ListBackendRequest
+ ListBackendResponse
+ UpdateBackendRequest
+ UpdateBackendResponse
+ DeleteBackendRequest
+ DeleteBackendResponse
+ BackendDetail
+ ListTypeRequest
+ ListTypeResponse
+ TypeDetail
+*/
+package backend
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+import (
+ context "context"
+
+ client "github.com/micro/go-micro/client"
+ server "github.com/micro/go-micro/server"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ client.Option
+var _ server.Option
+
+// Client API for Backend service
+
+type BackendService interface {
+ CreateBackend(ctx context.Context, in *CreateBackendRequest, opts ...client.CallOption) (*CreateBackendResponse, error)
+ GetBackend(ctx context.Context, in *GetBackendRequest, opts ...client.CallOption) (*GetBackendResponse, error)
+ ListBackend(ctx context.Context, in *ListBackendRequest, opts ...client.CallOption) (*ListBackendResponse, error)
+ UpdateBackend(ctx context.Context, in *UpdateBackendRequest, opts ...client.CallOption) (*UpdateBackendResponse, error)
+ DeleteBackend(ctx context.Context, in *DeleteBackendRequest, opts ...client.CallOption) (*DeleteBackendResponse, error)
+ ListType(ctx context.Context, in *ListTypeRequest, opts ...client.CallOption) (*ListTypeResponse, error)
+}
+
+type backendService struct {
+ c client.Client
+ name string
+}
+
+func NewBackendService(name string, c client.Client) BackendService {
+ if c == nil {
+ c = client.NewClient()
+ }
+ if len(name) == 0 {
+ name = "backend"
+ }
+ return &backendService{
+ c: c,
+ name: name,
+ }
+}
+
+func (c *backendService) CreateBackend(ctx context.Context, in *CreateBackendRequest, opts ...client.CallOption) (*CreateBackendResponse, error) {
+ req := c.c.NewRequest(c.name, "Backend.CreateBackend", in)
+ out := new(CreateBackendResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *backendService) GetBackend(ctx context.Context, in *GetBackendRequest, opts ...client.CallOption) (*GetBackendResponse, error) {
+ req := c.c.NewRequest(c.name, "Backend.GetBackend", in)
+ out := new(GetBackendResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *backendService) ListBackend(ctx context.Context, in *ListBackendRequest, opts ...client.CallOption) (*ListBackendResponse, error) {
+ req := c.c.NewRequest(c.name, "Backend.ListBackend", in)
+ out := new(ListBackendResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *backendService) UpdateBackend(ctx context.Context, in *UpdateBackendRequest, opts ...client.CallOption) (*UpdateBackendResponse, error) {
+ req := c.c.NewRequest(c.name, "Backend.UpdateBackend", in)
+ out := new(UpdateBackendResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *backendService) DeleteBackend(ctx context.Context, in *DeleteBackendRequest, opts ...client.CallOption) (*DeleteBackendResponse, error) {
+ req := c.c.NewRequest(c.name, "Backend.DeleteBackend", in)
+ out := new(DeleteBackendResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *backendService) ListType(ctx context.Context, in *ListTypeRequest, opts ...client.CallOption) (*ListTypeResponse, error) {
+ req := c.c.NewRequest(c.name, "Backend.ListType", in)
+ out := new(ListTypeResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+// Server API for Backend service
+
+type BackendHandler interface {
+ CreateBackend(context.Context, *CreateBackendRequest, *CreateBackendResponse) error
+ GetBackend(context.Context, *GetBackendRequest, *GetBackendResponse) error
+ ListBackend(context.Context, *ListBackendRequest, *ListBackendResponse) error
+ UpdateBackend(context.Context, *UpdateBackendRequest, *UpdateBackendResponse) error
+ DeleteBackend(context.Context, *DeleteBackendRequest, *DeleteBackendResponse) error
+ ListType(context.Context, *ListTypeRequest, *ListTypeResponse) error
+}
+
+func RegisterBackendHandler(s server.Server, hdlr BackendHandler, opts ...server.HandlerOption) error {
+ type backend interface {
+ CreateBackend(ctx context.Context, in *CreateBackendRequest, out *CreateBackendResponse) error
+ GetBackend(ctx context.Context, in *GetBackendRequest, out *GetBackendResponse) error
+ ListBackend(ctx context.Context, in *ListBackendRequest, out *ListBackendResponse) error
+ UpdateBackend(ctx context.Context, in *UpdateBackendRequest, out *UpdateBackendResponse) error
+ DeleteBackend(ctx context.Context, in *DeleteBackendRequest, out *DeleteBackendResponse) error
+ ListType(ctx context.Context, in *ListTypeRequest, out *ListTypeResponse) error
+ }
+ type Backend struct {
+ backend
+ }
+ h := &backendHandler{hdlr}
+ return s.Handle(s.NewHandler(&Backend{h}, opts...))
+}
+
+type backendHandler struct {
+ BackendHandler
+}
+
+func (h *backendHandler) CreateBackend(ctx context.Context, in *CreateBackendRequest, out *CreateBackendResponse) error {
+ return h.BackendHandler.CreateBackend(ctx, in, out)
+}
+
+func (h *backendHandler) GetBackend(ctx context.Context, in *GetBackendRequest, out *GetBackendResponse) error {
+ return h.BackendHandler.GetBackend(ctx, in, out)
+}
+
+func (h *backendHandler) ListBackend(ctx context.Context, in *ListBackendRequest, out *ListBackendResponse) error {
+ return h.BackendHandler.ListBackend(ctx, in, out)
+}
+
+func (h *backendHandler) UpdateBackend(ctx context.Context, in *UpdateBackendRequest, out *UpdateBackendResponse) error {
+ return h.BackendHandler.UpdateBackend(ctx, in, out)
+}
+
+func (h *backendHandler) DeleteBackend(ctx context.Context, in *DeleteBackendRequest, out *DeleteBackendResponse) error {
+ return h.BackendHandler.DeleteBackend(ctx, in, out)
+}
+
+func (h *backendHandler) ListType(ctx context.Context, in *ListTypeRequest, out *ListTypeResponse) error {
+ return h.BackendHandler.ListType(ctx, in, out)
+}
+
+
+
// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: backend.proto
+
+/*
+Package backend is a generated protocol buffer package.
+
+It is generated from these files:
+ backend.proto
+
+It has these top-level messages:
+ CreateBackendRequest
+ CreateBackendResponse
+ GetBackendRequest
+ GetBackendResponse
+ ListBackendRequest
+ ListBackendResponse
+ UpdateBackendRequest
+ UpdateBackendResponse
+ DeleteBackendRequest
+ DeleteBackendResponse
+ BackendDetail
+ ListTypeRequest
+ ListTypeResponse
+ TypeDetail
+*/
+package backend
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type CreateBackendRequest struct {
+ Backend *BackendDetail `protobuf:"bytes,1,opt,name=backend" json:"backend,omitempty"`
+}
+
+func (m *CreateBackendRequest) Reset() { *m = CreateBackendRequest{} }
+func (m *CreateBackendRequest) String() string { return proto.CompactTextString(m) }
+func (*CreateBackendRequest) ProtoMessage() {}
+func (*CreateBackendRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *CreateBackendRequest) GetBackend() *BackendDetail {
+ if m != nil {
+ return m.Backend
+ }
+ return nil
+}
+
+type CreateBackendResponse struct {
+ Backend *BackendDetail `protobuf:"bytes,1,opt,name=backend" json:"backend,omitempty"`
+}
+
+func (m *CreateBackendResponse) Reset() { *m = CreateBackendResponse{} }
+func (m *CreateBackendResponse) String() string { return proto.CompactTextString(m) }
+func (*CreateBackendResponse) ProtoMessage() {}
+func (*CreateBackendResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+
+func (m *CreateBackendResponse) GetBackend() *BackendDetail {
+ if m != nil {
+ return m.Backend
+ }
+ return nil
+}
+
+type GetBackendRequest struct {
+ Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+}
+
+func (m *GetBackendRequest) Reset() { *m = GetBackendRequest{} }
+func (m *GetBackendRequest) String() string { return proto.CompactTextString(m) }
+func (*GetBackendRequest) ProtoMessage() {}
+func (*GetBackendRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
+
+func (m *GetBackendRequest) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+type GetBackendResponse struct {
+ Backend *BackendDetail `protobuf:"bytes,1,opt,name=backend" json:"backend,omitempty"`
+}
+
+func (m *GetBackendResponse) Reset() { *m = GetBackendResponse{} }
+func (m *GetBackendResponse) String() string { return proto.CompactTextString(m) }
+func (*GetBackendResponse) ProtoMessage() {}
+func (*GetBackendResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
+
+func (m *GetBackendResponse) GetBackend() *BackendDetail {
+ if m != nil {
+ return m.Backend
+ }
+ return nil
+}
+
+type ListBackendRequest struct {
+ Limit int32 `protobuf:"varint,1,opt,name=limit" json:"limit,omitempty"`
+ Offset int32 `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
+ SortKeys []string `protobuf:"bytes,3,rep,name=sortKeys" json:"sortKeys,omitempty"`
+ SortDirs []string `protobuf:"bytes,4,rep,name=sortDirs" json:"sortDirs,omitempty"`
+ Filter map[string]string `protobuf:"bytes,5,rep,name=Filter" json:"Filter,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+}
+
+func (m *ListBackendRequest) Reset() { *m = ListBackendRequest{} }
+func (m *ListBackendRequest) String() string { return proto.CompactTextString(m) }
+func (*ListBackendRequest) ProtoMessage() {}
+func (*ListBackendRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+
+func (m *ListBackendRequest) GetLimit() int32 {
+ if m != nil {
+ return m.Limit
+ }
+ return 0
+}
+
+func (m *ListBackendRequest) GetOffset() int32 {
+ if m != nil {
+ return m.Offset
+ }
+ return 0
+}
+
+func (m *ListBackendRequest) GetSortKeys() []string {
+ if m != nil {
+ return m.SortKeys
+ }
+ return nil
+}
+
+func (m *ListBackendRequest) GetSortDirs() []string {
+ if m != nil {
+ return m.SortDirs
+ }
+ return nil
+}
+
+func (m *ListBackendRequest) GetFilter() map[string]string {
+ if m != nil {
+ return m.Filter
+ }
+ return nil
+}
+
+type ListBackendResponse struct {
+ Backends []*BackendDetail `protobuf:"bytes,1,rep,name=backends" json:"backends,omitempty"`
+ Next int32 `protobuf:"varint,2,opt,name=next" json:"next,omitempty"`
+}
+
+func (m *ListBackendResponse) Reset() { *m = ListBackendResponse{} }
+func (m *ListBackendResponse) String() string { return proto.CompactTextString(m) }
+func (*ListBackendResponse) ProtoMessage() {}
+func (*ListBackendResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+
+func (m *ListBackendResponse) GetBackends() []*BackendDetail {
+ if m != nil {
+ return m.Backends
+ }
+ return nil
+}
+
+func (m *ListBackendResponse) GetNext() int32 {
+ if m != nil {
+ return m.Next
+ }
+ return 0
+}
+
+type UpdateBackendRequest struct {
+ Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+ Access string `protobuf:"bytes,2,opt,name=access" json:"access,omitempty"`
+ Security string `protobuf:"bytes,3,opt,name=security" json:"security,omitempty"`
+}
+
+func (m *UpdateBackendRequest) Reset() { *m = UpdateBackendRequest{} }
+func (m *UpdateBackendRequest) String() string { return proto.CompactTextString(m) }
+func (*UpdateBackendRequest) ProtoMessage() {}
+func (*UpdateBackendRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+
+func (m *UpdateBackendRequest) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *UpdateBackendRequest) GetAccess() string {
+ if m != nil {
+ return m.Access
+ }
+ return ""
+}
+
+func (m *UpdateBackendRequest) GetSecurity() string {
+ if m != nil {
+ return m.Security
+ }
+ return ""
+}
+
+type UpdateBackendResponse struct {
+ Backend *BackendDetail `protobuf:"bytes,1,opt,name=backend" json:"backend,omitempty"`
+}
+
+func (m *UpdateBackendResponse) Reset() { *m = UpdateBackendResponse{} }
+func (m *UpdateBackendResponse) String() string { return proto.CompactTextString(m) }
+func (*UpdateBackendResponse) ProtoMessage() {}
+func (*UpdateBackendResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+
+func (m *UpdateBackendResponse) GetBackend() *BackendDetail {
+ if m != nil {
+ return m.Backend
+ }
+ return nil
+}
+
+type DeleteBackendRequest struct {
+ Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+}
+
+func (m *DeleteBackendRequest) Reset() { *m = DeleteBackendRequest{} }
+func (m *DeleteBackendRequest) String() string { return proto.CompactTextString(m) }
+func (*DeleteBackendRequest) ProtoMessage() {}
+func (*DeleteBackendRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+
+func (m *DeleteBackendRequest) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+type DeleteBackendResponse struct {
+}
+
+func (m *DeleteBackendResponse) Reset() { *m = DeleteBackendResponse{} }
+func (m *DeleteBackendResponse) String() string { return proto.CompactTextString(m) }
+func (*DeleteBackendResponse) ProtoMessage() {}
+func (*DeleteBackendResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+
+type BackendDetail struct {
+ Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+ TenantId string `protobuf:"bytes,2,opt,name=tenantId" json:"tenantId,omitempty"`
+ UserId string `protobuf:"bytes,3,opt,name=userId" json:"userId,omitempty"`
+ Name string `protobuf:"bytes,4,opt,name=name" json:"name,omitempty"`
+ Type string `protobuf:"bytes,5,opt,name=type" json:"type,omitempty"`
+ Region string `protobuf:"bytes,6,opt,name=region" json:"region,omitempty"`
+ Endpoint string `protobuf:"bytes,7,opt,name=endpoint" json:"endpoint,omitempty"`
+ BucketName string `protobuf:"bytes,8,opt,name=bucketName" json:"bucketName,omitempty"`
+ Access string `protobuf:"bytes,9,opt,name=access" json:"access,omitempty"`
+ Security string `protobuf:"bytes,10,opt,name=security" json:"security,omitempty"`
+}
+
+func (m *BackendDetail) Reset() { *m = BackendDetail{} }
+func (m *BackendDetail) String() string { return proto.CompactTextString(m) }
+func (*BackendDetail) ProtoMessage() {}
+func (*BackendDetail) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+
+func (m *BackendDetail) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *BackendDetail) GetTenantId() string {
+ if m != nil {
+ return m.TenantId
+ }
+ return ""
+}
+
+func (m *BackendDetail) GetUserId() string {
+ if m != nil {
+ return m.UserId
+ }
+ return ""
+}
+
+func (m *BackendDetail) GetName() string {
+ if m != nil {
+ return m.Name
+ }
+ return ""
+}
+
+func (m *BackendDetail) GetType() string {
+ if m != nil {
+ return m.Type
+ }
+ return ""
+}
+
+func (m *BackendDetail) GetRegion() string {
+ if m != nil {
+ return m.Region
+ }
+ return ""
+}
+
+func (m *BackendDetail) GetEndpoint() string {
+ if m != nil {
+ return m.Endpoint
+ }
+ return ""
+}
+
+func (m *BackendDetail) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *BackendDetail) GetAccess() string {
+ if m != nil {
+ return m.Access
+ }
+ return ""
+}
+
+func (m *BackendDetail) GetSecurity() string {
+ if m != nil {
+ return m.Security
+ }
+ return ""
+}
+
+type ListTypeRequest struct {
+ Limit int32 `protobuf:"varint,1,opt,name=limit" json:"limit,omitempty"`
+ Offset int32 `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
+ SortKeys []string `protobuf:"bytes,3,rep,name=sortKeys" json:"sortKeys,omitempty"`
+ SortDirs []string `protobuf:"bytes,4,rep,name=sortDirs" json:"sortDirs,omitempty"`
+ Filter map[string]string `protobuf:"bytes,5,rep,name=Filter" json:"Filter,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+}
+
+func (m *ListTypeRequest) Reset() { *m = ListTypeRequest{} }
+func (m *ListTypeRequest) String() string { return proto.CompactTextString(m) }
+func (*ListTypeRequest) ProtoMessage() {}
+func (*ListTypeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
+
+func (m *ListTypeRequest) GetLimit() int32 {
+ if m != nil {
+ return m.Limit
+ }
+ return 0
+}
+
+func (m *ListTypeRequest) GetOffset() int32 {
+ if m != nil {
+ return m.Offset
+ }
+ return 0
+}
+
+func (m *ListTypeRequest) GetSortKeys() []string {
+ if m != nil {
+ return m.SortKeys
+ }
+ return nil
+}
+
+func (m *ListTypeRequest) GetSortDirs() []string {
+ if m != nil {
+ return m.SortDirs
+ }
+ return nil
+}
+
+func (m *ListTypeRequest) GetFilter() map[string]string {
+ if m != nil {
+ return m.Filter
+ }
+ return nil
+}
+
+type ListTypeResponse struct {
+ Types []*TypeDetail `protobuf:"bytes,1,rep,name=types" json:"types,omitempty"`
+ Next int32 `protobuf:"varint,2,opt,name=next" json:"next,omitempty"`
+}
+
+func (m *ListTypeResponse) Reset() { *m = ListTypeResponse{} }
+func (m *ListTypeResponse) String() string { return proto.CompactTextString(m) }
+func (*ListTypeResponse) ProtoMessage() {}
+func (*ListTypeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
+
+func (m *ListTypeResponse) GetTypes() []*TypeDetail {
+ if m != nil {
+ return m.Types
+ }
+ return nil
+}
+
+func (m *ListTypeResponse) GetNext() int32 {
+ if m != nil {
+ return m.Next
+ }
+ return 0
+}
+
+type TypeDetail struct {
+ Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+ Description string `protobuf:"bytes,2,opt,name=description" json:"description,omitempty"`
+}
+
+func (m *TypeDetail) Reset() { *m = TypeDetail{} }
+func (m *TypeDetail) String() string { return proto.CompactTextString(m) }
+func (*TypeDetail) ProtoMessage() {}
+func (*TypeDetail) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
+
+func (m *TypeDetail) GetName() string {
+ if m != nil {
+ return m.Name
+ }
+ return ""
+}
+
+func (m *TypeDetail) GetDescription() string {
+ if m != nil {
+ return m.Description
+ }
+ return ""
+}
+
+func init() {
+ proto.RegisterType((*CreateBackendRequest)(nil), "CreateBackendRequest")
+ proto.RegisterType((*CreateBackendResponse)(nil), "CreateBackendResponse")
+ proto.RegisterType((*GetBackendRequest)(nil), "GetBackendRequest")
+ proto.RegisterType((*GetBackendResponse)(nil), "GetBackendResponse")
+ proto.RegisterType((*ListBackendRequest)(nil), "ListBackendRequest")
+ proto.RegisterType((*ListBackendResponse)(nil), "ListBackendResponse")
+ proto.RegisterType((*UpdateBackendRequest)(nil), "UpdateBackendRequest")
+ proto.RegisterType((*UpdateBackendResponse)(nil), "UpdateBackendResponse")
+ proto.RegisterType((*DeleteBackendRequest)(nil), "DeleteBackendRequest")
+ proto.RegisterType((*DeleteBackendResponse)(nil), "DeleteBackendResponse")
+ proto.RegisterType((*BackendDetail)(nil), "BackendDetail")
+ proto.RegisterType((*ListTypeRequest)(nil), "ListTypeRequest")
+ proto.RegisterType((*ListTypeResponse)(nil), "ListTypeResponse")
+ proto.RegisterType((*TypeDetail)(nil), "TypeDetail")
+}
+
+func init() { proto.RegisterFile("backend.proto", fileDescriptor0) }
+
+var fileDescriptor0 = []byte{
+ // 630 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x55, 0x4d, 0x6f, 0xd3, 0x40,
+ 0x10, 0x8d, 0x93, 0x26, 0x4d, 0xc6, 0x6a, 0x69, 0xb7, 0x49, 0xb0, 0x2c, 0x04, 0xc1, 0x48, 0x28,
+ 0xe2, 0xb0, 0x12, 0x05, 0xa9, 0xd0, 0x03, 0x2a, 0xa5, 0x80, 0x2a, 0x10, 0x07, 0x8b, 0x5e, 0xb8,
+ 0xb9, 0xf6, 0x14, 0xad, 0x9a, 0xda, 0xc6, 0xbb, 0x41, 0xf8, 0xcc, 0x9f, 0xe5, 0xc4, 0x95, 0x2b,
+ 0xda, 0x0f, 0x27, 0xce, 0xc6, 0x08, 0x45, 0x42, 0xdc, 0xfc, 0x66, 0x66, 0x77, 0x66, 0xe7, 0xcd,
+ 0x3c, 0xc3, 0xce, 0x65, 0x14, 0x5f, 0x63, 0x9a, 0xd0, 0xbc, 0xc8, 0x44, 0x16, 0x9c, 0xc0, 0xf0,
+ 0x55, 0x81, 0x91, 0xc0, 0x53, 0x6d, 0x0e, 0xf1, 0xcb, 0x1c, 0xb9, 0x20, 0x53, 0xd8, 0x36, 0x81,
+ 0x9e, 0x33, 0x71, 0xa6, 0xee, 0xe1, 0x2e, 0x35, 0x11, 0x67, 0x28, 0x22, 0x36, 0x0b, 0x2b, 0x77,
+ 0xf0, 0x12, 0x46, 0xd6, 0x0d, 0x3c, 0xcf, 0x52, 0x8e, 0x1b, 0x5c, 0xf1, 0x00, 0xf6, 0xdf, 0xa2,
+ 0xb0, 0x2a, 0xd8, 0x85, 0x36, 0xd3, 0x27, 0x07, 0x61, 0x9b, 0x25, 0xc1, 0x0b, 0x20, 0xf5, 0xa0,
+ 0x8d, 0x93, 0xfc, 0x74, 0x80, 0xbc, 0x67, 0xdc, 0x4e, 0x33, 0x84, 0xee, 0x8c, 0xdd, 0x30, 0xa1,
+ 0x8e, 0x77, 0x43, 0x0d, 0xc8, 0x18, 0x7a, 0xd9, 0xd5, 0x15, 0x47, 0xe1, 0xb5, 0x95, 0xd9, 0x20,
+ 0xe2, 0x43, 0x9f, 0x67, 0x85, 0x78, 0x87, 0x25, 0xf7, 0x3a, 0x93, 0xce, 0x74, 0x10, 0x2e, 0x70,
+ 0xe5, 0x3b, 0x63, 0x05, 0xf7, 0xb6, 0x96, 0x3e, 0x89, 0xc9, 0x11, 0xf4, 0xde, 0xb0, 0x99, 0xc0,
+ 0xc2, 0xeb, 0x4e, 0x3a, 0x53, 0xf7, 0xf0, 0x1e, 0x5d, 0x2f, 0x85, 0xea, 0x88, 0xd7, 0xa9, 0x28,
+ 0xca, 0xd0, 0x84, 0xfb, 0xcf, 0xc1, 0xad, 0x99, 0xc9, 0x1e, 0x74, 0xae, 0xb1, 0x34, 0x5d, 0x91,
+ 0x9f, 0xb2, 0xfe, 0xaf, 0xd1, 0x6c, 0x8e, 0xaa, 0xd0, 0x41, 0xa8, 0xc1, 0x71, 0xfb, 0x99, 0x13,
+ 0x5c, 0xc0, 0xc1, 0x4a, 0x12, 0xd3, 0xb1, 0x47, 0xd0, 0x37, 0x2d, 0xe1, 0x9e, 0xa3, 0x8a, 0xb1,
+ 0x5b, 0xb6, 0xf0, 0x13, 0x02, 0x5b, 0x29, 0x7e, 0xab, 0x9a, 0xa0, 0xbe, 0x83, 0x4f, 0x30, 0xbc,
+ 0xc8, 0x93, 0xf5, 0x89, 0xb1, 0xf8, 0x92, 0x2d, 0x8c, 0xe2, 0x18, 0x39, 0x37, 0x95, 0x19, 0xa4,
+ 0xda, 0x84, 0xf1, 0xbc, 0x60, 0xa2, 0xf4, 0x3a, 0xca, 0xb3, 0xc0, 0x72, 0x96, 0xac, 0xbb, 0x37,
+ 0xa6, 0xf9, 0x21, 0x0c, 0xcf, 0x70, 0x86, 0x7f, 0x2b, 0x2f, 0xb8, 0x0d, 0x23, 0x2b, 0x4e, 0xa7,
+ 0x0a, 0xbe, 0xb7, 0x61, 0x67, 0xe5, 0xee, 0xb5, 0x97, 0xf9, 0xd0, 0x17, 0x98, 0x46, 0xa9, 0x38,
+ 0x4f, 0xcc, 0xdb, 0x16, 0x58, 0xbe, 0x7a, 0xce, 0xb1, 0x38, 0x4f, 0xcc, 0xdb, 0x0c, 0x52, 0x9d,
+ 0x8c, 0x6e, 0xd0, 0xdb, 0x52, 0x56, 0xf5, 0x2d, 0x6d, 0xa2, 0xcc, 0xd1, 0xeb, 0x6a, 0x9b, 0xfc,
+ 0x96, 0xe7, 0x0b, 0xfc, 0xcc, 0xb2, 0xd4, 0xeb, 0xe9, 0xf3, 0x1a, 0xc9, 0x9c, 0x98, 0x26, 0x79,
+ 0xc6, 0x52, 0xe1, 0x6d, 0xeb, 0x9c, 0x15, 0x26, 0x77, 0x01, 0x2e, 0xe7, 0xf1, 0x35, 0x8a, 0x0f,
+ 0x32, 0x43, 0x5f, 0x79, 0x6b, 0x96, 0x1a, 0x13, 0x83, 0x3f, 0x32, 0x01, 0x16, 0x13, 0x3f, 0x1c,
+ 0xb8, 0x25, 0xa7, 0xe7, 0x63, 0x99, 0xe3, 0xff, 0x5d, 0x95, 0xa7, 0xd6, 0xaa, 0xdc, 0xa1, 0x56,
+ 0x1d, 0xff, 0x7a, 0x4f, 0xce, 0x61, 0x6f, 0x99, 0xc1, 0xcc, 0xdb, 0x7d, 0xe8, 0x4a, 0x3a, 0xaa,
+ 0x0d, 0x71, 0xa9, 0xf4, 0x9a, 0x51, 0xd3, 0x9e, 0xc6, 0xdd, 0x38, 0x05, 0x58, 0x06, 0x2e, 0x38,
+ 0x77, 0x6a, 0x9c, 0x4f, 0xc0, 0x4d, 0x90, 0xc7, 0x05, 0xcb, 0x85, 0x24, 0x59, 0x17, 0x53, 0x37,
+ 0x1d, 0xfe, 0x6a, 0xc3, 0xb6, 0x99, 0x3f, 0x72, 0x02, 0x3b, 0x2b, 0xda, 0x4a, 0x46, 0xb4, 0x49,
+ 0xad, 0xfd, 0x31, 0x6d, 0x94, 0xe0, 0xa0, 0x45, 0x8e, 0x00, 0x96, 0xaa, 0x49, 0x08, 0x5d, 0xd3,
+ 0x59, 0xff, 0x80, 0xae, 0xcb, 0x6a, 0xd0, 0x22, 0xc7, 0xe0, 0xd6, 0xd4, 0x83, 0x1c, 0x34, 0x08,
+ 0x96, 0x3f, 0xa4, 0x0d, 0x02, 0x13, 0xb4, 0x64, 0xd9, 0x2b, 0x6b, 0x4c, 0x46, 0xb4, 0x49, 0x32,
+ 0xfc, 0x31, 0x6d, 0xdc, 0x76, 0x7d, 0xc3, 0xca, 0x76, 0x92, 0x11, 0x6d, 0xda, 0x6a, 0x7f, 0x4c,
+ 0x9b, 0x97, 0xb8, 0x45, 0x1e, 0x43, 0xbf, 0x62, 0x95, 0xec, 0xd9, 0x23, 0xe4, 0xef, 0x53, 0x9b,
+ 0xf2, 0xa0, 0x75, 0xd9, 0x53, 0xbf, 0xc4, 0x27, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x50, 0xb6,
+ 0xe0, 0x3d, 0x23, 0x07, 0x00, 0x00,
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "fmt"
+
+ "github.com/micro/go-micro"
+ "github.com/opensds/multi-cloud/api/pkg/utils/obs"
+ _ "github.com/opensds/multi-cloud/s3/pkg/datastore"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/driver"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/config"
+ gc "github.com/opensds/multi-cloud/s3/pkg/gc"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/redis"
+ handler "github.com/opensds/multi-cloud/s3/pkg/service"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+ _ "go.uber.org/automaxprocs"
+)
+
+func main() {
+ service := micro.NewService(
+ micro.Name("s3"),
+ )
+
+ obs.InitLogs()
+ service.Init(micro.AfterStop(func() error {
+ driver.FreeCloser()
+ gc.Stop()
+ return nil
+ }))
+
+ helper.SetupConfig()
+
+ log.Infof("YIG conf: %+v \n", helper.CONFIG)
+ log.Infof("YIG instance ID: %+v \n", helper.CONFIG.InstanceId)
+
+ if helper.CONFIG.MetaCacheType > 0 || helper.CONFIG.EnableDataCache {
+ cfg := config.CacheConfig{
+ Mode: helper.CONFIG.RedisMode,
+ Address: helper.CONFIG.RedisAddress,
+ }
+ redis.Initialize(&cfg)
+ }
+
+ pb.RegisterS3Handler(service.Server(), handler.NewS3Service())
+ if err := service.Run(); err != nil {
+ fmt.Println(err)
+ }
+}
+
+
+
/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3error
+
+import (
+ "net/http"
+)
+
+type S3Error interface {
+ error
+ AwsErrorCode() string
+ Description() string
+ HttpStatusCode() int
+}
+
+type S3ErrorStruct struct {
+ AwsErrorCode string
+ Description string
+ HttpStatusCode int
+}
+
+// APIErrorCode type of error status.
+type S3ErrorCode int
+
+// Error codes, non exhaustive list - http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+const (
+ ErrNoErr S3ErrorCode = iota
+ ErrAccessDenied
+ ErrBadDigest
+ ErrBucketAlreadyExists
+ ErrEmptyEntity
+ ErrEntityTooLarge
+ ErrIncompleteBody
+ ErrInternalError
+ ErrInvalidAccessKeyID
+ ErrInvalidBucketName
+ ErrInvalidObjectName
+ ErrInvalidDigest
+ ErrInvalidRange
+ ErrInvalidEncodingType
+ ErrInvalidContinuationToken
+ ErrInvalidMaxKeys
+ ErrInvalidMaxUploads
+ ErrInvalidMaxParts
+ ErrInvalidPartNumberMarker
+ ErrInvalidRequestBody
+ ErrInvalidCopySource
+ ErrInvalidCopySourceStorageClass
+ ErrInvalidCopyDest
+ ErrInvalidPrecondition
+ ErrInvalidPolicyDocument
+ ErrInvalidCorsDocument
+ ErrInvalidVersioning
+ ErrMalformedXML
+ ErrMissingContentLength
+ ErrMissingContentMD5
+ ErrMissingRequestBodyError
+ ErrNoSuchBucket
+ ErrNoSuchBucketPolicy
+ ErrNoSuchKey
+ ErrNoSuchUpload
+ ErrNoSuchVersion
+ ErrNotImplemented
+ ErrPreconditionFailed
+ ErrRequestTimeTooSkewed
+ ErrSignatureDoesNotMatch
+ ErrMethodNotAllowed
+ ErrInvalidPart
+ ErrInvalidPartOrder
+ ErrAuthorizationHeaderMalformed
+ ErrMalformedPOSTRequest
+ ErrSignatureVersionNotSupported
+ ErrBucketNotEmpty
+ ErrBucketAccessForbidden
+ ErrMalformedPolicy
+ ErrMissingFields
+ ErrMissingCredTag
+ ErrCredMalformed
+ ErrInvalidRegion
+ ErrInvalidService
+ ErrInvalidRequestVersion
+ ErrMissingSignTag
+ ErrMissingSignHeadersTag
+ ErrMissingRequiredSignedHeader
+ ErrSignedHeadersNotSorted
+ ErrPolicyAlreadyExpired
+ ErrPolicyViolation
+ ErrMalformedDate
+ ErrMalformedExpires
+ ErrAuthHeaderEmpty
+ ErrExpiredPresignRequest
+ ErrMissingDateHeader
+ ErrInvalidQuerySignatureAlgo
+ ErrInvalidQueryParams
+ ErrBucketAlreadyOwnedByYou
+ ErrInvalidCannedAcl
+ ErrInvalidSseHeader
+ ErrTooManyBuckets
+ ErrInvalidPosition
+ ErrObjectNotAppendable
+ ErrPositionNotEqualToLength
+ // Add new error codes here.
+
+ // SSE-S3 related API errors
+ ErrInvalidEncryptionMethod
+
+ // Server-Side-Encryption (with Customer provided key) related API errors.
+ ErrInsecureSSECustomerRequest
+ ErrSSEMultipartEncrypted
+ ErrSSEEncryptedObject
+ ErrInvalidEncryptionParameters
+ ErrInvalidSSECustomerAlgorithm
+ ErrInvalidSSECustomerKey
+ ErrMissingSSECustomerKey
+ ErrMissingSSECustomerKeyMD5
+ ErrSSECustomerKeyMD5Mismatch
+ ErrInvalidSSECustomerParameters
+ ErrIncompatibleEncryptionMethod
+ ErrKMSNotConfigured
+ ErrKMSAuthFailure
+
+ // S3 extended errors.
+ ErrContentSHA256Mismatch
+ // Add new extended error codes here.
+
+ // Add new extended error codes here.
+ ContentNotModified // actually not an error
+ ErrInvalidHeader // supplementary error for golang http lib
+ ErrNoSuchBucketCors
+ ErrPolicyMissingFields
+ ErrInvalidAcl
+ ErrUnsupportedAcl
+ ErrNonUTF8Encode
+ ErrInvalidLc
+ ErrNoSuchBucketLc
+ ErrInvalidStorageClass
+ ErrPutToBackendFailed
+ ErrGetFromBackendFailed
+ ErrDeleteFromBackendFailed
+ ErrBackendInitMultipartFailed
+ ErrBackendCompleteMultipartFailed
+ ErrBackendAbortMultipartFailed
+ ErrGetBackendFailed
+ ErrUnmarshalFailed
+ ErrGetBucketFailed
+ ErrDBError
+)
+
+// error code to APIError structure, these fields carry respective
+// descriptions for all the error responses.
+var ErrorCodeResponse = map[S3ErrorCode]S3ErrorStruct{
+ ErrNoErr: {
+ AwsErrorCode: "OK",
+ Description: "OK",
+ HttpStatusCode: http.StatusOK,
+ },
+ ErrInvalidCopyDest: {
+ AwsErrorCode: "InvalidRequest",
+ Description: "This copy request is illegal because it is trying to copy an object to itself.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidCopySource: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+
+ ErrInvalidCopySourceStorageClass: {
+ AwsErrorCode: "InvalidCopySourceStorageClass",
+ Description: "Storage class of copy source cannot be GLACIER or DEEP_ARCHIVE.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidPrecondition: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "The provided preconditions are not valid(bad time format, rule combination, etc)",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidRequestBody: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Body shouldn't be set for this request.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidEncodingType: {
+ AwsErrorCode: "InvalidEncodingType",
+ Description: "The encoding type you provided is not allowed.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidContinuationToken: {
+ AwsErrorCode: "ErrInvalidContinuationToken",
+ Description: "The continuation token you provided is invalid.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidMaxUploads: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Argument max-uploads must be an integer between 1 and 1000",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidMaxKeys: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Argument maxKeys must be an integer between 1 and 1000",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidMaxParts: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Argument max-parts must be an integer between 1 and 1000",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidPartNumberMarker: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Argument partNumberMarker must be an integer.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidPolicyDocument: {
+ AwsErrorCode: "InvalidPolicyDocument",
+ Description: "The content of the form does not meet the conditions specified in the policy document.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidCorsDocument: {
+ AwsErrorCode: "InvalidCorsDocument",
+ Description: "The CORS XML you provided is invalid",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidVersioning: {
+ AwsErrorCode: "IllegalVersioningConfigurationException",
+ Description: "The versioning configuration specified in the request is invalid.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrAccessDenied: {
+ AwsErrorCode: "AccessDenied",
+ Description: "Access Denied.",
+ HttpStatusCode: http.StatusForbidden,
+ },
+ ErrBadDigest: {
+ AwsErrorCode: "BadDigest",
+ Description: "The Content-Md5 you specified did not match what we received.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrBucketAlreadyExists: {
+ AwsErrorCode: "BucketAlreadyExists",
+ Description: "The requested bucket name is not available.",
+ HttpStatusCode: http.StatusConflict,
+ },
+ ErrEmptyEntity: {
+ AwsErrorCode: "EmptyEntity",
+ Description: "Your upload does not include a valid object",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrEntityTooLarge: {
+ AwsErrorCode: "EntityTooLarge",
+ Description: "Your proposed upload exceeds the maximum allowed object size.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrIncompleteBody: {
+ AwsErrorCode: "IncompleteBody",
+ Description: "You did not provide the number of bytes specified by the Content-Length HTTP header.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInternalError: {
+ AwsErrorCode: "InternalError",
+ Description: "We encountered an internal error, please try again.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+ ErrInvalidAccessKeyID: {
+ AwsErrorCode: "InvalidAccessKeyId",
+ Description: "The access key ID you provided does not exist in our records.",
+ HttpStatusCode: http.StatusForbidden,
+ },
+ ErrInvalidBucketName: {
+ AwsErrorCode: "InvalidBucketName",
+ Description: "The specified bucket name is not valid.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidObjectName: {
+ AwsErrorCode: "InvalidObjectName",
+ Description: "The specified object name is not valid",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidDigest: {
+ AwsErrorCode: "InvalidDigest",
+ Description: "The Content-Md5 you specified is not valid.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidRange: {
+ AwsErrorCode: "InvalidRange",
+ Description: "The requested range is not satisfiable",
+ HttpStatusCode: http.StatusRequestedRangeNotSatisfiable,
+ },
+ ErrMalformedXML: {
+ AwsErrorCode: "MalformedXML",
+ Description: "The XML you provided was not well-formed or did not validate against our published schema.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMissingContentLength: {
+ AwsErrorCode: "MissingContentLength",
+ Description: "You must provide the Content-Length HTTP header.",
+ HttpStatusCode: http.StatusLengthRequired,
+ },
+ ErrMissingContentMD5: {
+ AwsErrorCode: "MissingContentMD5",
+ Description: "Missing required header for this request: Content-Md5.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMissingRequestBodyError: {
+ AwsErrorCode: "MissingRequestBodyError",
+ Description: "Request body is empty.",
+ HttpStatusCode: http.StatusLengthRequired,
+ },
+ ErrNoSuchBucket: {
+ AwsErrorCode: "NoSuchBucket",
+ Description: "The specified bucket does not exist",
+ HttpStatusCode: http.StatusNotFound,
+ },
+ ErrNoSuchBucketPolicy: {
+ AwsErrorCode: "NoSuchBucketPolicy",
+ Description: "The specified bucket does not have a bucket policy.",
+ HttpStatusCode: http.StatusNotFound,
+ },
+ ErrNoSuchKey: {
+ AwsErrorCode: "NoSuchKey",
+ Description: "The specified key does not exist.",
+ HttpStatusCode: http.StatusNotFound,
+ },
+ ErrNoSuchUpload: {
+ AwsErrorCode: "NoSuchUpload",
+ Description: "The specified multipart upload does not exist.",
+ HttpStatusCode: http.StatusNotFound,
+ },
+ ErrNoSuchVersion: {
+ AwsErrorCode: "NoSuchVersion",
+ Description: "The version ID specified in the request does not match an existing version.",
+ HttpStatusCode: http.StatusNotFound,
+ },
+ ErrNotImplemented: {
+ AwsErrorCode: "NotImplemented",
+ Description: "A header you provided implies functionality that is not implemented",
+ HttpStatusCode: http.StatusNotImplemented,
+ },
+ ErrPreconditionFailed: {
+ AwsErrorCode: "PreconditionFailed",
+ Description: "At least one of the pre-conditions you specified did not hold",
+ HttpStatusCode: http.StatusPreconditionFailed,
+ },
+ ErrRequestTimeTooSkewed: {
+ AwsErrorCode: "RequestTimeTooSkewed",
+ Description: "The difference between the request time and the server's time is too large.",
+ HttpStatusCode: http.StatusForbidden,
+ },
+ ErrSignatureDoesNotMatch: {
+ AwsErrorCode: "SignatureDoesNotMatch",
+ Description: "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
+ HttpStatusCode: http.StatusForbidden,
+ },
+ ErrMethodNotAllowed: {
+ AwsErrorCode: "MethodNotAllowed",
+ Description: "The specified method is not allowed against this resource.",
+ HttpStatusCode: http.StatusMethodNotAllowed,
+ },
+ ErrInvalidPart: {
+ AwsErrorCode: "InvalidPart",
+ Description: "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidPartOrder: {
+ AwsErrorCode: "InvalidPartOrder",
+ Description: "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrAuthorizationHeaderMalformed: {
+ AwsErrorCode: "AuthorizationHeaderMalformed",
+ Description: "The authorization header is malformed.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMalformedPOSTRequest: {
+ AwsErrorCode: "MalformedPOSTRequest",
+ Description: "The body of your POST request is not well-formed multipart/form-data.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrSignatureVersionNotSupported: {
+ AwsErrorCode: "AccessDenied",
+ Description: "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
+ HttpStatusCode: http.StatusForbidden,
+ },
+ ErrBucketNotEmpty: {
+ AwsErrorCode: "BucketNotEmpty",
+ Description: "The bucket you tried to delete is not empty.",
+ HttpStatusCode: http.StatusConflict,
+ },
+ ErrBucketAccessForbidden: {
+ AwsErrorCode: "AccessDenied",
+ Description: "You have no access to this bucket.",
+ HttpStatusCode: http.StatusForbidden,
+ },
+ ErrMalformedPolicy: {
+ AwsErrorCode: "MalformedPolicy",
+ Description: "Policy has invalid resource.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMissingFields: {
+ AwsErrorCode: "MissingFields",
+ Description: "Missing fields in request.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMissingCredTag: {
+ AwsErrorCode: "InvalidRequest",
+ Description: "Missing Credential field for this request.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrCredMalformed: {
+ AwsErrorCode: "CredentialMalformed",
+ Description: "Credential field does not follow accessKeyID/credentialScope.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMalformedDate: {
+ AwsErrorCode: "MalformedDate",
+ Description: "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidRegion: {
+ AwsErrorCode: "InvalidRegion",
+ Description: "Region does not match.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidService: {
+ AwsErrorCode: "AccessDenied",
+ Description: "Service scope should be of value 's3'.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidRequestVersion: {
+ AwsErrorCode: "AccessDenied",
+ Description: "Request scope should be of value 'aws4_request'.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMissingSignTag: {
+ AwsErrorCode: "AccessDenied",
+ Description: "Signature header missing Signature field.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMissingSignHeadersTag: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Signature header missing SignedHeaders field.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMissingRequiredSignedHeader: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Missing one or more required signed header",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrSignedHeadersNotSorted: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Signed headers are not ordered",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrPolicyAlreadyExpired: {
+ AwsErrorCode: "AccessDenied",
+ Description: "Invalid according to Policy: Policy expired.",
+ HttpStatusCode: http.StatusForbidden,
+ },
+ ErrPolicyViolation: {
+ AwsErrorCode: "AccessDenied",
+ Description: "File uploading policy violated.",
+ HttpStatusCode: http.StatusForbidden,
+ },
+ ErrMalformedExpires: {
+ AwsErrorCode: "MalformedExpires",
+ Description: "Malformed expires value, should be between 1 and 604800(seven days)",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrAuthHeaderEmpty: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Authorization header is invalid -- one and only one ' ' (space) required.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMissingDateHeader: {
+ AwsErrorCode: "AccessDenied",
+ Description: "AWS authentication requires a valid Date or x-amz-date header",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidQuerySignatureAlgo: {
+ AwsErrorCode: "AuthorizationQueryParametersError",
+ Description: "X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\".",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrExpiredPresignRequest: {
+ AwsErrorCode: "ExpiredToken",
+ Description: "Request has expired.",
+ HttpStatusCode: http.StatusForbidden,
+ },
+ ErrInvalidQueryParams: {
+ AwsErrorCode: "AuthorizationQueryParametersError",
+ Description: "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrBucketAlreadyOwnedByYou: {
+ AwsErrorCode: "BucketAlreadyOwnedByYou",
+ Description: "Your previous request to create the named bucket succeeded and you already own it.",
+ HttpStatusCode: http.StatusConflict,
+ },
+ ErrTooManyBuckets: {
+ AwsErrorCode: "TooManyBuckets",
+ Description: "You have attempted to create more buckets than allowed.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+
+ // SSE-S3 related API errors
+ ErrInvalidEncryptionMethod: {
+ AwsErrorCode: "InvalidRequest",
+ Description: "The encryption method specified is not supported",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+
+ // Server-Side-Encryption (with Customer provided key) related API errors.
+ ErrInsecureSSECustomerRequest: {
+ AwsErrorCode: "InvalidRequest",
+ Description: "Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrSSEMultipartEncrypted: {
+ AwsErrorCode: "InvalidRequest",
+ Description: "The multipart upload initiate requested encryption. Subsequent part requests must include the appropriate encryption parameters.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrSSEEncryptedObject: {
+ AwsErrorCode: "InvalidRequest",
+ Description: "The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidEncryptionParameters: {
+ AwsErrorCode: "InvalidRequest",
+ Description: "The encryption parameters are not applicable to this object.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidSSECustomerAlgorithm: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidSSECustomerKey: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "The secret key was invalid for the specified algorithm.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMissingSSECustomerKey: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrMissingSSECustomerKeyMD5: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrSSECustomerKeyMD5Mismatch: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "The calculated MD5 hash of the key did not match the hash that was provided.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidSSECustomerParameters: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "The provided encryption parameters did not match the ones used originally.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrIncompatibleEncryptionMethod: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Server side encryption specified with both SSE-C and SSE-S3 headers",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrKMSNotConfigured: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Server side encryption specified but KMS is not configured",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrKMSAuthFailure: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "Server side encryption specified but KMS authorization failed",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ /// S3 extensions.
+ ErrContentSHA256Mismatch: {
+ AwsErrorCode: "XAmzContentSHA256Mismatch",
+ Description: "The provided 'x-amz-content-sha256' header does not match what was computed.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidCannedAcl: {
+ AwsErrorCode: "InvalidAcl",
+ Description: "The canned ACL you provided is not valid",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidSseHeader: {
+ AwsErrorCode: "InvalidSseHeader",
+ Description: "The Server-side Encryption configuration is corrupted or invalid",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+
+ ContentNotModified: { // FIXME: This is actually not an error
+ AwsErrorCode: "",
+ Description: "",
+ HttpStatusCode: http.StatusNotModified,
+ },
+ ErrInvalidHeader: {
+ AwsErrorCode: "InvalidRequest",
+ Description: "This request is illegal because some header is malformed.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrNoSuchBucketCors: {
+ AwsErrorCode: "NoSuchBucketCors",
+ Description: "The specified bucket does not have CORS configured.",
+ HttpStatusCode: http.StatusNotFound,
+ },
+ ErrPolicyMissingFields: {
+ AwsErrorCode: "AccessDenied",
+ Description: "Missing policy condition",
+ HttpStatusCode: http.StatusForbidden,
+ },
+ ErrInvalidAcl: {
+ AwsErrorCode: "IllegalAclConfigurationException",
+ Description: "The ACL configuration specified in the request is invalid.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrUnsupportedAcl: {
+ AwsErrorCode: "UnsupportedAclConfigurationException",
+ Description: "The ACL configuration specified in the request is unsupported.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrNonUTF8Encode: {
+ AwsErrorCode: "InvalidArgument",
+ Description: "URL Argument must be UTF8 encoded.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrNoSuchBucketLc: {
+ AwsErrorCode: "NoSuchBucketLc",
+ Description: "The specified bucket does not have LifeCycle configured.",
+ HttpStatusCode: http.StatusNotFound,
+ },
+ ErrInvalidLc: {
+ AwsErrorCode: "IllegalLcConfigurationException",
+ Description: "The LC configuration specified in the request is invalid.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrInvalidPosition: {
+ AwsErrorCode: "InvalidPosition",
+ Description: "The argument position specified in the request must be non-negative integer.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrObjectNotAppendable: {
+ AwsErrorCode: "ObjectNotAppendable",
+ Description: "Cannot perform an AppendObject operation on a non-Appendable Object.",
+ HttpStatusCode: http.StatusConflict,
+ },
+ ErrPositionNotEqualToLength: {
+ AwsErrorCode: "PositionNotEqualToLength",
+ Description: "The value of position does not match the length of the current Object.",
+ HttpStatusCode: http.StatusConflict,
+ },
+ ErrInvalidStorageClass: {
+ AwsErrorCode: "InvalidStorageClass",
+ Description: "The storage class you specified in header is invalid.",
+ HttpStatusCode: http.StatusBadRequest,
+ },
+ ErrPutToBackendFailed: {
+ AwsErrorCode: "PutToBackendFailed",
+ Description: "Put object to backend failed.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+ ErrGetFromBackendFailed: {
+ AwsErrorCode: "GetFromBackendFailed",
+ Description: "Get object from backend failed.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+ ErrDeleteFromBackendFailed: {
+ AwsErrorCode: "DeleteFromBackendFailed",
+ Description: "Delete object from backend failed.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+ ErrBackendInitMultipartFailed: {
+ AwsErrorCode: "BackendInitMultipartFailed",
+ Description: "Backend init multipart upload failed.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+ ErrBackendCompleteMultipartFailed: {
+ AwsErrorCode: "BackendCompleteMultipartFailed",
+ Description: "Backend complete multipart upload failed.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+ ErrBackendAbortMultipartFailed: {
+ AwsErrorCode: "BackendAbortMultipartFailed",
+ Description: "Backend abort multipart upload failed.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+ ErrGetBackendFailed: {
+ AwsErrorCode: "GetBackendFailed",
+ Description: "Backend is not exist, or get it failed.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+ ErrUnmarshalFailed: {
+ AwsErrorCode: "UnmarshalFailed",
+ Description: "Unmarshal failed.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+ ErrGetBucketFailed: {
+ AwsErrorCode: "GetBucketFailed",
+ Description: "Bucket is not exist, or get it failed.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+ ErrDBError: {
+ AwsErrorCode: "InternalError",
+ Description: "DB error.",
+ HttpStatusCode: http.StatusInternalServerError,
+ },
+}
+
+func (e S3ErrorCode) AwsErrorCode() string {
+ awsError, ok := ErrorCodeResponse[e]
+ if !ok {
+ return "InternalError"
+ }
+ return awsError.AwsErrorCode
+}
+
+func (e S3ErrorCode) Description() string {
+ awsError, ok := ErrorCodeResponse[e]
+ if !ok {
+ return "We encountered an internal error, please try again."
+ }
+ return awsError.Description
+}
+
+func (e S3ErrorCode) Error() string {
+ return e.Description()
+}
+
+func (e S3ErrorCode) HttpStatusCode() int {
+ awsError, ok := ErrorCodeResponse[e]
+ if !ok {
+ return http.StatusInternalServerError
+ }
+ return awsError.HttpStatusCode
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "io/ioutil"
+ "strconv"
+ "time"
+
+ "crypto/md5"
+ "encoding/base64"
+ "encoding/hex"
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/session"
+ awss3 "github.com/aws/aws-sdk-go/service/s3"
+ "github.com/aws/aws-sdk-go/service/s3/s3manager"
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ . "github.com/opensds/multi-cloud/s3/error"
+ dscommon "github.com/opensds/multi-cloud/s3/pkg/datastore/common"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ osdss3 "github.com/opensds/multi-cloud/s3/pkg/service"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+type AwsAdapter struct {
+ backend *backendpb.BackendDetail
+ session *session.Session
+}
+
+type s3Cred struct {
+ ak string
+ sk string
+}
+
+func (myc *s3Cred) Retrieve() (credentials.Value, error) {
+ cred := credentials.Value{AccessKeyID: myc.ak, SecretAccessKey: myc.sk}
+ return cred, nil
+}
+
+func (myc *s3Cred) IsExpired() bool {
+ return false
+}
+
+func (ad *AwsAdapter) Put(ctx context.Context, stream io.Reader, object *pb.Object) (dscommon.PutResult, error) {
+ bucket := ad.backend.BucketName
+ objectId := object.BucketName + "/" + object.ObjectKey
+ result := dscommon.PutResult{}
+ userMd5 := dscommon.GetMd5FromCtx(ctx)
+ size := object.Size
+ log.Infof("put object[OBS], objectId:%s, bucket:%s, size=%d, userMd5=%s\n", objectId, bucket, size, userMd5)
+
+ // Limit the reader to its provided size if specified.
+ var limitedDataReader io.Reader
+ if size > 0 { // request.ContentLength is -1 if length is unknown
+ limitedDataReader = io.LimitReader(stream, size)
+ } else {
+ limitedDataReader = stream
+ }
+ md5Writer := md5.New()
+ dataReader := io.TeeReader(limitedDataReader, md5Writer)
+
+ if object.Tier == 0 {
+ // default
+ object.Tier = utils.Tier1
+ }
+ storClass, err := osdss3.GetNameFromTier(object.Tier, utils.OSTYPE_AWS)
+ if err != nil {
+ log.Infof("translate tier[%d] to aws storage class failed\n", object.Tier)
+ return result, ErrInternalError
+ }
+
+ uploader := s3manager.NewUploader(ad.session)
+ input := &s3manager.UploadInput{
+ Body: dataReader,
+ Bucket: aws.String(bucket),
+ Key: aws.String(objectId),
+ StorageClass: aws.String(storClass),
+ }
+ if userMd5 != "" {
+ md5Bytes, err := hex.DecodeString(userMd5)
+ if err != nil {
+ log.Warnf("user input md5 is abandoned, cause decode md5 failed, err:%v\n", err)
+ } else {
+ input.ContentMD5 = aws.String(base64.StdEncoding.EncodeToString(md5Bytes))
+ log.Debugf("input.ContentMD5=%s\n", *input.ContentMD5)
+ }
+ }
+ log.Infof("upload object[AWS S3] start, objectId:%s\n", objectId)
+ ret, err := uploader.Upload(input)
+ if err != nil {
+ log.Errorf("put object[AWS S3] failed, objectId:%s, err:%v\n", objectId, err)
+ return result, ErrPutToBackendFailed
+ }
+ log.Infof("put object[AWS S3] end, objectId:%s\n", objectId)
+
+ calculatedMd5 := hex.EncodeToString(md5Writer.Sum(nil))
+ log.Debug("calculatedMd5:", calculatedMd5, ", userMd5:", userMd5)
+ if userMd5 != "" && userMd5 != calculatedMd5 {
+ log.Error("### MD5 not match, calculatedMd5:", calculatedMd5, ", userMd5:", userMd5)
+ return result, ErrBadDigest
+ }
+
+ if ret.VersionID != nil {
+ result.Meta = *ret.VersionID
+ }
+ result.UpdateTime = time.Now().Unix()
+ result.ObjectId = objectId
+ result.Etag = calculatedMd5
+ result.Written = size
+ log.Infof("put object[AWS S3] successfully, objectId:%s, UpdateTime is:%v\n", objectId, result.UpdateTime)
+
+ return result, nil
+}
+
+func (ad *AwsAdapter) Get(ctx context.Context, object *pb.Object, start int64, end int64) (io.ReadCloser, error) {
+ bucket := ad.backend.BucketName
+ objectId := object.ObjectId
+ getObjectInput := awss3.GetObjectInput{
+ Bucket: &bucket,
+ Key: &objectId,
+ }
+ log.Infof("get object[AWS S3], objectId:%s, start = %d, end = %d\n", objectId, start, end)
+ if start != 0 || end != 0 {
+ strStart := strconv.FormatInt(start, 10)
+ strEnd := strconv.FormatInt(end, 10)
+ rangestr := "bytes=" + strStart + "-" + strEnd
+ getObjectInput.SetRange(rangestr)
+ }
+
+ svc := awss3.New(ad.session)
+ result, err := svc.GetObject(&getObjectInput)
+ if err != nil {
+ log.Errorf("get object[AWS S3] failed, objectId:%s, err:%v", objectId, err)
+ return nil, ErrGetFromBackendFailed
+ }
+
+ log.Infof("get object[AWS S3] succeed, objectId:%s, ContentLength:%d\n", objectId, *result.ContentLength)
+ return result.Body, nil
+}
+
+func (ad *AwsAdapter) Delete(ctx context.Context, input *pb.DeleteObjectInput) error {
+ bucket := ad.backend.BucketName
+ objectId := input.Bucket + "/" + input.Key
+ deleteInput := awss3.DeleteObjectInput{Bucket: &bucket, Key: &objectId}
+
+ log.Infof("delete object[AWS S3], objectId:%s.\n", objectId)
+ svc := awss3.New(ad.session)
+ _, err := svc.DeleteObject(&deleteInput)
+ if err != nil {
+ log.Errorf("delete object[AWS S3] failed, objectId:%s, err:%v.\n", objectId, err)
+ return ErrDeleteFromBackendFailed
+ }
+
+ log.Infof("delete object[AWS S3] succeed, objectId:%s.\n", objectId)
+
+ return nil
+}
+
+func (ad *AwsAdapter) Copy(ctx context.Context, stream io.Reader, target *pb.Object) (result dscommon.PutResult, err error) {
+ log.Errorf("copy[AWS S3] is not supported.")
+ err = ErrInternalError
+ return
+}
+
+func (ad *AwsAdapter) ChangeStorageClass(ctx context.Context, object *pb.Object, newClass *string) error {
+ objectId := object.ObjectId
+ log.Infof("change storage class[AWS S3] of object[%s] to %s .\n", objectId, *newClass)
+
+ svc := awss3.New(ad.session)
+ input := &awss3.CopyObjectInput{
+ Bucket: aws.String(ad.backend.BucketName),
+ Key: aws.String(objectId),
+ CopySource: aws.String(ad.backend.BucketName + "/" + objectId),
+ }
+ input.StorageClass = aws.String(*newClass)
+ _, err := svc.CopyObject(input)
+ if err != nil {
+ log.Errorf("change storage class[AWS S3] of object[%s] to %s failed: %v.\n", objectId, *newClass, err)
+ return ErrPutToBackendFailed
+ }
+
+ log.Infof("change storage class[AWS S3] of object[%s] to %s succeed.\n", objectId, *newClass)
+ return nil
+}
+
+func (ad *AwsAdapter) InitMultipartUpload(ctx context.Context, object *pb.Object) (*pb.MultipartUpload, error) {
+ bucket := ad.backend.BucketName
+ objectId := object.BucketName + "/" + object.ObjectKey
+ log.Infof("init multipart upload[AWS S3], bucket = %v,objectId = %v\n", bucket, objectId)
+
+ storClass, err := osdss3.GetNameFromTier(object.Tier, utils.OSTYPE_AWS)
+ if err != nil {
+ log.Warnf("translate tier[%d] to aws storage class failed, use default value.\n", object.Tier)
+ return nil, ErrInternalError
+ }
+
+ multipartUpload := &pb.MultipartUpload{}
+ multiUpInput := &awss3.CreateMultipartUploadInput{
+ Bucket: &bucket,
+ Key: &objectId,
+ StorageClass: aws.String(storClass),
+ }
+
+ svc := awss3.New(ad.session)
+ res, err := svc.CreateMultipartUpload(multiUpInput)
+ if err != nil {
+ log.Fatalf("init multipart upload[AWS S3] failed, err:%v\n", err)
+ return nil, ErrBackendInitMultipartFailed
+ } else {
+ log.Infof("init multipart upload[AWS S3] succeed, UploadId:%s\n", *res.UploadId)
+ multipartUpload.Bucket = object.BucketName
+ multipartUpload.Key = object.ObjectKey
+ multipartUpload.UploadId = *res.UploadId
+ multipartUpload.ObjectId = objectId
+ return multipartUpload, nil
+ }
+}
+
+func (ad *AwsAdapter) UploadPart(ctx context.Context, stream io.Reader, multipartUpload *pb.MultipartUpload,
+ partNumber int64, upBytes int64) (*model.UploadPartResult, error) {
+ bucket := ad.backend.BucketName
+ bytess, err := ioutil.ReadAll(stream)
+ if err != nil {
+ log.Errorf("read data failed, err:%v\n", err)
+ return nil, ErrInternalError
+ }
+ upPartInput := &awss3.UploadPartInput{
+ Body: bytes.NewReader(bytess),
+ Bucket: &bucket,
+ Key: &multipartUpload.ObjectId,
+ PartNumber: aws.Int64(partNumber),
+ UploadId: &multipartUpload.UploadId,
+ ContentLength: aws.Int64(upBytes),
+ }
+ log.Infof("upload part[AWS S3], input:%v\n", *upPartInput)
+
+ svc := awss3.New(ad.session)
+ upRes, err := svc.UploadPart(upPartInput)
+ if err != nil {
+ log.Errorf("upload part[AWS S3] failed. err:%v\n", err)
+ return nil, ErrPutToBackendFailed
+ } else {
+ log.Infof("upload object[AWS S3], objectId:%s, part #%d succeed, ETag:%s\n", multipartUpload.ObjectId,
+ partNumber, *upRes.ETag)
+ result := &model.UploadPartResult{
+ Xmlns: model.Xmlns,
+ ETag: *upRes.ETag,
+ PartNumber: partNumber}
+ return result, nil
+ }
+
+ log.Error("upload part[AWS S3]: should not be here.")
+ return nil, ErrInternalError
+}
+
+func (ad *AwsAdapter) CompleteMultipartUpload(ctx context.Context, multipartUpload *pb.MultipartUpload,
+ completeUpload *model.CompleteMultipartUpload) (*model.CompleteMultipartUploadResult, error) {
+ bucket := ad.backend.BucketName
+ log.Infof("complete multipart upload[AWS S3], bucket:%s, objectId:%s.\n", bucket, multipartUpload.ObjectId)
+
+ var completeParts []*awss3.CompletedPart
+ for _, p := range completeUpload.Parts {
+ completePart := &awss3.CompletedPart{
+ ETag: aws.String(p.ETag),
+ PartNumber: aws.Int64(p.PartNumber),
+ }
+ completeParts = append(completeParts, completePart)
+ }
+ completeInput := &awss3.CompleteMultipartUploadInput{
+ Bucket: &bucket,
+ Key: &multipartUpload.ObjectId,
+ UploadId: &multipartUpload.UploadId,
+ MultipartUpload: &awss3.CompletedMultipartUpload{
+ Parts: completeParts,
+ },
+ }
+
+ log.Infof("completeInput %v\n", *completeInput)
+ svc := awss3.New(ad.session)
+ resp, err := svc.CompleteMultipartUpload(completeInput)
+ if err != nil {
+ log.Errorf("complete multipart upload[AWS S3] failed, err:%v\n", err)
+ return nil, ErrBackendCompleteMultipartFailed
+ }
+ result := &model.CompleteMultipartUploadResult{
+ Xmlns: model.Xmlns,
+ Location: *resp.Location,
+ Bucket: multipartUpload.Bucket,
+ Key: multipartUpload.Key,
+ ETag: *resp.ETag,
+ }
+
+ log.Infof("complete multipart upload[AWS S3] successfully, resp:%v\n", resp)
+ return result, nil
+}
+
+func (ad *AwsAdapter) AbortMultipartUpload(ctx context.Context, multipartUpload *pb.MultipartUpload) error {
+ bucket := ad.backend.BucketName
+ log.Infof("abort multipart upload[AWS S3], bucket:%s, objectId:%s.\n", bucket, multipartUpload.ObjectId)
+
+ abortInput := &awss3.AbortMultipartUploadInput{
+ Bucket: &bucket,
+ Key: &multipartUpload.ObjectId,
+ UploadId: &multipartUpload.UploadId,
+ }
+
+ svc := awss3.New(ad.session)
+ rsp, err := svc.AbortMultipartUpload(abortInput)
+ if err != nil {
+ log.Errorf("abort multipart upload[AWS S3] failed, err:%v\n", err)
+ return ErrBackendAbortMultipartFailed
+ }
+
+ log.Infof("complete multipart upload[AWS S3] successfully, rsp:%v\n", rsp)
+ return nil
+}
+
+func (ad *AwsAdapter) ListParts(ctx context.Context, multipartUpload *pb.ListParts) (*model.ListPartsOutput, error) {
+ return nil, errors.New("not implemented yet.")
+}
+
+func (ad *AwsAdapter) Close() error {
+ // TODO:
+ return nil
+}
+
+
+
package aws
+
+import (
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/opensds/multi-cloud/backend/pkg/utils/constants"
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/driver"
+)
+
+type AwsS3DriverFactory struct {
+}
+
+func (factory *AwsS3DriverFactory) CreateDriver(backend *backendpb.BackendDetail) (driver.StorageDriver, error) {
+ endpoint := backend.Endpoint
+ AccessKeyID := backend.Access
+ AccessKeySecret := backend.Security
+ region := backend.Region
+
+ s3aksk := s3Cred{ak: AccessKeyID, sk: AccessKeySecret}
+ creds := credentials.NewCredentials(&s3aksk)
+
+ disableSSL := true
+ sess, err := session.NewSession(&aws.Config{
+ Region: ®ion,
+ Endpoint: &endpoint,
+ Credentials: creds,
+ DisableSSL: &disableSSL,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ adap := &AwsAdapter{backend: backend, session: sess}
+
+ return adap, nil
+}
+
+func init() {
+ driver.RegisterDriverFactory(constants.BackendTypeAws, &AwsS3DriverFactory{})
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azure
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/binary"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "time"
+
+ "encoding/hex"
+ "github.com/Azure/azure-storage-blob-go/azblob"
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ . "github.com/opensds/multi-cloud/s3/error"
+ dscommon "github.com/opensds/multi-cloud/s3/pkg/datastore/common"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ osdss3 "github.com/opensds/multi-cloud/s3/pkg/service"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+ "strconv"
+)
+
+// TryTimeout indicates the maximum time allowed for any single try of an HTTP request.
+var MaxTimeForSingleHttpRequest = 50 * time.Minute
+
+type AzureAdapter struct {
+ backend *backendpb.BackendDetail
+ containerURL azblob.ContainerURL
+}
+
+/*func Init(backend *backendpb.BackendDetail) *AzureAdapter {
+ endpoint := backend.Endpoint
+ AccessKeyID := backend.Access
+ AccessKeySecret := backend.Security
+ ad := AzureAdapter{}
+ containerURL, err := ad.createContainerURL(endpoint, AccessKeyID, AccessKeySecret)
+ if err != nil {
+ log.Infof("AzureAdapter Init container URL faild:%v\n", err)
+ return nil
+ }
+ adap := &AzureAdapter{backend: backend, containerURL: containerURL}
+ log.Log("AzureAdapter Init succeed, container URL:", containerURL.String())
+ return adap
+}*/
+
+func (ad *AzureAdapter) createContainerURL(endpoint string, acountName string, accountKey string) (azblob.ContainerURL,
+ error) {
+ credential, err := azblob.NewSharedKeyCredential(acountName, accountKey)
+
+ if err != nil {
+ log.Infof("create credential[Azure Blob] failed, err:%v\n", err)
+ return azblob.ContainerURL{}, err
+ }
+
+ //create containerURL
+ p := azblob.NewPipeline(credential, azblob.PipelineOptions{
+ Retry: azblob.RetryOptions{
+ TryTimeout: MaxTimeForSingleHttpRequest,
+ },
+ })
+ URL, _ := url.Parse(endpoint)
+
+ return azblob.NewContainerURL(*URL, p), nil
+}
+
+func (ad *AzureAdapter) Put(ctx context.Context, stream io.Reader, object *pb.Object) (dscommon.PutResult, error) {
+ objectId := object.BucketName + "/" + object.ObjectKey
+ blobURL := ad.containerURL.NewBlockBlobURL(objectId)
+ result := dscommon.PutResult{}
+ userMd5 := dscommon.GetMd5FromCtx(ctx)
+ log.Infof("put object[Azure Blob], objectId:%s, blobURL:%v, userMd5:%s, size:%d\n", objectId, blobURL, userMd5, object.Size)
+
+ log.Infof("put object[Azure Blob] begin, objectId:%s\n", objectId)
+ options := azblob.UploadStreamToBlockBlobOptions{BufferSize: 2 * 1024 * 1024, MaxBuffers: 2}
+
+ uploadResp, err := azblob.UploadStreamToBlockBlob(ctx, stream, blobURL, options)
+ log.Infof("put object[Azure Blob] end, objectId:%s\n", objectId)
+ if err != nil {
+ log.Errorf("put object[Azure Blob], objectId:%s, err:%v\n", objectId, err)
+ return result, ErrPutToBackendFailed
+ }
+ if uploadResp.Response().StatusCode != http.StatusCreated {
+ log.Errorf("put object[Azure Blob], objectId:%s, StatusCode:%d\n", objectId, uploadResp.Response().StatusCode)
+ return result, ErrPutToBackendFailed
+ }
+
+ if object.Tier == 0 {
+ // default
+ object.Tier = utils.Tier1
+ }
+ storClass, err := osdss3.GetNameFromTier(object.Tier, utils.OSTYPE_Azure)
+ if err != nil {
+ log.Infof("translate tier[%d] to aws storage class failed\n", object.Tier)
+ return result, ErrInternalError
+ }
+
+ resultMd5 := uploadResp.Response().Header.Get("Content-MD5")
+ resultMd5Bytes, err := base64.StdEncoding.DecodeString(resultMd5)
+ if err != nil {
+ log.Errorf("decode Content-MD5 failed, err:%v\n", err)
+ return result, ErrBadDigest
+ }
+ decodedMd5 := hex.EncodeToString(resultMd5Bytes)
+ if userMd5 != "" && userMd5 != decodedMd5 {
+ log.Error("### MD5 not match, resultMd5:", resultMd5, ", decodedMd5:", decodedMd5, ", userMd5:", userMd5)
+ return result, ErrBadDigest
+ }
+
+ // Currently, only support Hot
+ _, err = blobURL.SetTier(ctx, azblob.AccessTierType(storClass), azblob.LeaseAccessConditions{})
+ if err != nil {
+ log.Errorf("set azure blob tier[%s] failed:%v\n", object.Tier, err)
+ return result, ErrPutToBackendFailed
+ }
+
+ result.UpdateTime = time.Now().Unix()
+ result.ObjectId = objectId
+ result.Etag = decodedMd5
+ result.Meta = uploadResp.Version()
+ result.Written = object.Size
+ log.Infof("upload object[Azure Blob] succeed, objectId:%s, UpdateTime is:%v\n", objectId, result.UpdateTime)
+
+ return result, nil
+}
+
+func (ad *AzureAdapter) Get(ctx context.Context, object *pb.Object, start int64, end int64) (io.ReadCloser, error) {
+ bucket := ad.backend.BucketName
+ log.Infof("get object[Azure Blob], bucket:%s, objectId:%s\n", bucket, object.ObjectId)
+
+ blobURL := ad.containerURL.NewBlobURL(object.ObjectId)
+ log.Infof("blobURL:%v, size:%d\n", blobURL, object.Size)
+
+ var buf []byte
+ if end < object.Size - 1 {
+ count := end - start + 1
+ buf = make([]byte, count)
+ log.Debugf("start=%d, end=%d, count=%d\n", start, end, count)
+ err := azblob.DownloadBlobToBuffer(ctx, blobURL, start, count, buf, azblob.DownloadFromBlobOptions{})
+ if err != nil {
+ log.Errorf("get object[Azure Blob] failed, objectId:%s, err:%v\n", object.ObjectId, err)
+ return nil, ErrGetFromBackendFailed
+ }
+ body := bytes.NewReader(buf)
+ ioReaderClose := ioutil.NopCloser(body)
+ return ioReaderClose, nil
+ } else {
+ downloadResp, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{},
+ false)
+ if err != nil {
+ log.Errorf("get object[Azure Blob] failed, objectId:%s, err:%v\n", object.ObjectId, err)
+ return nil, ErrGetFromBackendFailed
+ }
+ log.Infof("get object[Azure Blob] successfully, objectId:%s\n", object.ObjectId)
+ return downloadResp.Response().Body, nil
+ }
+
+ log.Error("get object[Azure Blob]: should not be here")
+ return nil, ErrInternalError
+}
+
+func (ad *AzureAdapter) Delete(ctx context.Context, input *pb.DeleteObjectInput) error {
+ bucket := ad.backend.BucketName
+ objectId := input.Bucket + "/" + input.Key
+ log.Infof("delete object[Azure Blob], objectId:%s, bucket:%s\n", objectId, bucket)
+
+ blobURL := ad.containerURL.NewBlockBlobURL(objectId)
+ log.Infof("blobURL is %v\n", blobURL)
+ delRsp, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
+ if err != nil {
+ if serr, ok := err.(azblob.StorageError); ok { // This error is a Service-specific
+ log.Infof("delete service code:%s\n", serr.ServiceCode())
+ if string(serr.ServiceCode()) == string(azblob.StorageErrorCodeBlobNotFound) {
+ return nil
+ }
+ }
+
+ log.Errorf("delete object[Azure Blob] failed, objectId:%s, err:%v\n", objectId, err)
+ return ErrDeleteFromBackendFailed
+ }
+
+ if delRsp.StatusCode() != http.StatusOK && delRsp.StatusCode() != http.StatusAccepted {
+ log.Errorf("delete object[Azure Blob] failed, objectId:%s, status code:%d\n", objectId, delRsp.StatusCode())
+ return ErrDeleteFromBackendFailed
+ }
+
+ log.Infof("delete object[Azure Blob] succeed, objectId:%s\n", objectId)
+ return nil
+}
+
+func (ad *AzureAdapter) Copy(ctx context.Context, stream io.Reader, target *pb.Object) (result dscommon.PutResult, err error) {
+ log.Errorf("copy[Azure Blob] is not supported.")
+ err = ErrInternalError
+ return
+}
+
+func (ad *AzureAdapter) ChangeStorageClass(ctx context.Context, object *pb.Object, newClass *string) error {
+ objectId := object.ObjectId
+ blobURL := ad.containerURL.NewBlockBlobURL(objectId)
+ log.Infof("change storage class[Azure Blob], objectId:%s, blobURL is %v\n", objectId, blobURL)
+
+ var res *azblob.BlobSetTierResponse
+ var err error
+ switch *newClass {
+ case string(azblob.AccessTierHot):
+ res, err = blobURL.SetTier(ctx, azblob.AccessTierHot, azblob.LeaseAccessConditions{})
+ case string(azblob.AccessTierCool):
+ res, err = blobURL.SetTier(ctx, azblob.AccessTierCool, azblob.LeaseAccessConditions{})
+ case string(azblob.AccessTierArchive):
+ res, err = blobURL.SetTier(ctx, azblob.AccessTierArchive, azblob.LeaseAccessConditions{})
+ default:
+ log.Errorf("change storage class[Azure Blob] of object[%s] to %s failed, err: invalid storage class.\n",
+ object.ObjectKey, newClass)
+ return ErrInvalidStorageClass
+ }
+ if err != nil {
+ log.Errorf("change storage class[Azure Blob] of object[%s] to %s failed, err:%v\n", object.ObjectKey,
+ newClass, err)
+ return ErrInternalError
+ } else {
+ log.Errorf("change storage class[Azure Blob] of object[%s] to %s succeed, res:%v\n", object.ObjectKey,
+ newClass, res.Response())
+ }
+
+ return nil
+}
+
+func (ad *AzureAdapter) GetObjectInfo(bucketName string, key string, context context.Context) (*pb.Object, error) {
+ object := pb.Object{}
+ object.BucketName = bucketName
+ object.ObjectKey = key
+ return &object, nil
+}
+
+func (ad *AzureAdapter) InitMultipartUpload(ctx context.Context, object *pb.Object) (*pb.MultipartUpload, error) {
+ bucket := ad.backend.BucketName
+ log.Infof("bucket is %v\n", bucket)
+ multipartUpload := &pb.MultipartUpload{}
+ multipartUpload.Key = object.ObjectKey
+
+ multipartUpload.Bucket = object.BucketName
+ multipartUpload.UploadId = object.ObjectKey + "_" + strconv.FormatInt(time.Now().UnixNano(), 10)
+ multipartUpload.ObjectId = object.BucketName + "/" + object.ObjectKey
+ return multipartUpload, nil
+}
+
+func (ad *AzureAdapter) Int64ToBase64(blockID int64) string {
+ buf := (&[8]byte{})[:]
+ binary.LittleEndian.PutUint64(buf, uint64(blockID))
+ return ad.BinaryToBase64(buf)
+}
+
+func (ad *AzureAdapter) BinaryToBase64(binaryID []byte) string {
+ return base64.StdEncoding.EncodeToString(binaryID)
+}
+
+func (ad *AzureAdapter) Base64ToInt64(base64ID string) int64 {
+ bin, _ := base64.StdEncoding.DecodeString(base64ID)
+ return int64(binary.LittleEndian.Uint64(bin))
+}
+
+func (ad *AzureAdapter) UploadPart(ctx context.Context, stream io.Reader, multipartUpload *pb.MultipartUpload,
+ partNumber int64, upBytes int64) (*model.UploadPartResult, error) {
+ bucket := ad.backend.BucketName
+ log.Infof("upload part[Azure Blob], bucket:%s, objectId:%s\n", bucket, multipartUpload.ObjectId)
+
+ blobURL := ad.containerURL.NewBlockBlobURL(multipartUpload.ObjectId)
+ base64ID := ad.Int64ToBase64(partNumber)
+ bytess, _ := ioutil.ReadAll(stream)
+ log.Debugf("blobURL=%+v\n", blobURL)
+ rsp, err := blobURL.StageBlock(ctx, base64ID, bytes.NewReader(bytess), azblob.LeaseAccessConditions{}, nil)
+ if err != nil {
+ log.Errorf("stage block[#%d,base64ID:%s] failed:%v\n", partNumber, base64ID, err)
+ return nil, ErrPutToBackendFailed
+ }
+
+ etag := hex.EncodeToString(rsp.ContentMD5())
+ log.Infof("stage block[#%d,base64ID:%s] succeed, etag:%s.\n", partNumber, base64ID, etag)
+ result := &model.UploadPartResult{PartNumber: partNumber, ETag: etag}
+
+ return result, nil
+}
+
+func (ad *AzureAdapter) CompleteMultipartUpload(ctx context.Context, multipartUpload *pb.MultipartUpload,
+ completeUpload *model.CompleteMultipartUpload) (*model.CompleteMultipartUploadResult, error) {
+ bucket := ad.backend.BucketName
+ result := model.CompleteMultipartUploadResult{}
+ result.Bucket = multipartUpload.Bucket
+ result.Key = multipartUpload.Key
+ result.Location = ad.backend.Name
+ log.Infof("complete multipart upload[Azure Blob], bucket:%s, objectId:%s\n", bucket, multipartUpload.ObjectId)
+
+ blobURL := ad.containerURL.NewBlockBlobURL(multipartUpload.ObjectId)
+ var completeParts []string
+ for _, p := range completeUpload.Parts {
+ base64ID := ad.Int64ToBase64(p.PartNumber)
+ completeParts = append(completeParts, base64ID)
+ }
+ log.Debugf("commit block list, blobURL:%+v, completeParts:%+v\n", blobURL, completeParts)
+ _, err := blobURL.CommitBlockList(ctx, completeParts, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{})
+ if err != nil {
+ log.Errorf("commit blocks[bucket:%s, objectId:%s] failed:%v\n", bucket, multipartUpload.ObjectId, err)
+ return nil, ErrBackendCompleteMultipartFailed
+ } else {
+ storClass, err := osdss3.GetNameFromTier(multipartUpload.Tier, utils.OSTYPE_Azure)
+ if err != nil {
+ log.Errorf("translate tier[%d] to aws storage class failed\n", multipartUpload.Tier)
+ return nil, ErrInternalError
+ }
+
+ // set tier
+ _, err = blobURL.SetTier(ctx, azblob.AccessTierType(storClass), azblob.LeaseAccessConditions{})
+ if err != nil {
+ log.Errorf("set blob[objectId:%s] tier failed:%v\n", multipartUpload.ObjectId, err)
+ return nil, ErrBackendCompleteMultipartFailed
+ }
+ }
+
+ log.Infof("complete multipart upload[Azure Blob], bucket:%s, objectId:%s succeed\n",
+ bucket, multipartUpload.ObjectId)
+ return &result, nil
+}
+
+func (ad *AzureAdapter) AbortMultipartUpload(ctx context.Context, multipartUpload *pb.MultipartUpload) error {
+ bucket := ad.backend.BucketName
+ log.Infof("no need to abort multipart upload[objkey:%s].\n", bucket)
+ return nil
+}
+
+func (ad *AzureAdapter) ListParts(ctx context.Context, multipartUpload *pb.ListParts) (*model.ListPartsOutput, error) {
+ return nil, ErrNotImplemented
+}
+
+func (ad *AzureAdapter) Close() error {
+ // TODO:
+ return nil
+}
+
+
+
package azure
+
+import (
+ "github.com/opensds/multi-cloud/backend/pkg/utils/constants"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/driver"
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+)
+
+type AzureBlobDriverFactory struct {
+}
+
+func (factory *AzureBlobDriverFactory) CreateDriver(backend *backendpb.BackendDetail) (driver.StorageDriver, error) {
+ endpoint := backend.Endpoint
+ AccessKeyID := backend.Access
+ AccessKeySecret := backend.Security
+ ad := AzureAdapter{}
+ containerURL, err := ad.createContainerURL(endpoint, AccessKeyID, AccessKeySecret)
+ if err != nil {
+ return nil, err
+ }
+
+ adap := &AzureAdapter{backend: backend, containerURL: containerURL}
+
+ return adap, nil
+}
+
+func init() {
+ driver.RegisterDriverFactory(constants.BackendTypeAzure, &AzureBlobDriverFactory{})
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ceph
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "time"
+
+ "crypto/md5"
+ "encoding/hex"
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ . "github.com/opensds/multi-cloud/s3/error"
+ dscommon "github.com/opensds/multi-cloud/s3/pkg/datastore/common"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+ "github.com/webrtcn/s3client"
+ . "github.com/webrtcn/s3client"
+ "github.com/webrtcn/s3client/models"
+)
+
+type CephAdapter struct {
+ backend *backendpb.BackendDetail
+ session *s3client.Client
+}
+
+func (ad *CephAdapter) Put(ctx context.Context, stream io.Reader, object *pb.Object) (result dscommon.PutResult, err error) {
+ bucketName := ad.backend.BucketName
+ objectId := object.BucketName + "/" + object.ObjectKey
+ log.Infof("put object[Ceph S3], bucket:%s, objectId:%s\n", bucketName, objectId)
+
+ userMd5 := dscommon.GetMd5FromCtx(ctx)
+ size := object.Size
+
+ // Limit the reader to its provided size if specified.
+ var limitedDataReader io.Reader
+ if size > 0 { // request.ContentLength is -1 if length is unknown
+ limitedDataReader = io.LimitReader(stream, size)
+ } else {
+ limitedDataReader = stream
+ }
+ md5Writer := md5.New()
+ dataReader := io.TeeReader(limitedDataReader, md5Writer)
+
+ bucket := ad.session.NewBucket()
+ cephObject := bucket.NewObject(bucketName)
+ body := ioutil.NopCloser(dataReader)
+ log.Infof("put object[Ceph S3] begin, objectId:%s\n", objectId)
+ err = cephObject.Create(objectId, userMd5, "", object.Size, body, models.Private)
+ log.Infof("put object[Ceph S3] end, objectId:%s\n", objectId)
+ if err != nil {
+ log.Infof("upload object[Ceph S3] failed, objectId:%s, err:%v", objectId, err)
+ return result, ErrPutToBackendFailed
+ }
+
+ calculatedMd5 := "\"" + hex.EncodeToString(md5Writer.Sum(nil)) + "\""
+ if userMd5 != "" && userMd5 != calculatedMd5 {
+ log.Error("### MD5 not match, calculatedMd5:", calculatedMd5, "userMd5:", userMd5)
+ return result, ErrBadDigest
+ }
+
+ result.UpdateTime = time.Now().Unix()
+ result.ObjectId = objectId
+ result.Etag = calculatedMd5
+ result.Written = size
+ log.Infof("upload object[Ceph S3] succeed, objectId:%s, UpdateTime is:%v, etag:\n", objectId,
+ result.UpdateTime, result.Etag)
+
+ return result, nil
+}
+
+func (ad *CephAdapter) Get(ctx context.Context, object *pb.Object, start int64, end int64) (io.ReadCloser, error) {
+ log.Infof("get object[Ceph S3], bucket:%s, objectId:%s\n", object.BucketName, object.ObjectId)
+
+ getObjectOption := GetObjectOption{}
+ if start != 0 || end != 0 {
+ rangeObj := Range{
+ Begin: start,
+ End: end,
+ }
+ getObjectOption = GetObjectOption{
+ Range: &rangeObj,
+ }
+ }
+
+ bucket := ad.session.NewBucket()
+ cephObject := bucket.NewObject(ad.backend.BucketName)
+ getObject, err := cephObject.Get(object.ObjectId, &getObjectOption)
+ if err != nil {
+ fmt.Println(err)
+ log.Infof("get object[Ceph S3], objectId:%s failed:%v", object.ObjectId, err)
+ return nil, ErrGetFromBackendFailed
+ }
+
+ log.Infof("get object[Ceph S3] succeed, objectId:%s, bytes:%d\n", object.ObjectId, getObject.ContentLength)
+ return getObject.Body, nil
+}
+
+func (ad *CephAdapter) Delete(ctx context.Context, object *pb.DeleteObjectInput) error {
+ bucket := ad.session.NewBucket()
+ objectId := object.Bucket + "/" + object.Key
+ log.Infof("delete object[Ceph S3], objectId:%s, bucket:%s\n", objectId, bucket)
+
+ cephObject := bucket.NewObject(ad.backend.BucketName)
+ err := cephObject.Remove(objectId)
+ if err != nil {
+ log.Infof("delete object[Ceph S3] failed, objectId:%s, err:%v\n", objectId, err)
+ return ErrDeleteFromBackendFailed
+ }
+
+ log.Infof("delete object[Ceph S3] succeed, objectId:%s.\n", objectId)
+ return nil
+}
+
+func (ad *CephAdapter) Copy(ctx context.Context, stream io.Reader, target *pb.Object) (result dscommon.PutResult, err error) {
+ log.Errorf("copy[Ceph S3] is not supported.")
+ err = ErrInternalError
+ return
+}
+
+func (ad *CephAdapter) ChangeStorageClass(ctx context.Context, object *pb.Object, newClass *string) error {
+ log.Errorf("change storage class[Ceph S3] is not supported.")
+ return ErrInternalError
+}
+
+/*func (ad *CephAdapter) GetObjectInfo(context context.Context, bucketName string, key string) (*pb.Object, error) {
+ bucket := ad.backend.BucketName
+ newKey := bucketName + "/" + key
+
+ bucketO := ad.session.NewBucket()
+ bucketResp, err := bucketO.Get(bucket, newKey, "", "", 1000)
+ if err != nil {
+ log.Infof("error occured during get Object Info, err:%v\n", err)
+ return nil, err
+ }
+
+ for _, content := range bucketResp.Contents {
+ realKey := bucketName + "/" + key
+ if realKey != content.Key {
+ break
+ }
+ obj := &pb.Object{
+ BucketName: bucketName,
+ ObjectKey: key,
+ Size: content.Size,
+ }
+
+ return obj, nil
+ }
+
+ log.Infof("can not find specified object(%s).\n", key)
+ return nil, NoSuchObject.Error()
+}*/
+
+func (ad *CephAdapter) InitMultipartUpload(ctx context.Context, object *pb.Object) (*pb.MultipartUpload, error) {
+ bucket := ad.session.NewBucket()
+ objectId := object.BucketName + "/" + object.ObjectKey
+ log.Infof("init multipart upload[Ceph S3], bucket = %v,objectId = %v\n", bucket, objectId)
+ cephObject := bucket.NewObject(ad.backend.BucketName)
+ uploader := cephObject.NewUploads(objectId)
+ multipartUpload := &pb.MultipartUpload{}
+
+ res, err := uploader.Initiate(nil)
+
+ if err != nil {
+ log.Fatalf("init multipart upload[Ceph S3] failed, objectId:%s, err:%v\n", objectId, err)
+ return nil, err
+ } else {
+ log.Infof("init multipart upload[Ceph S3] succeed, objectId:%s, UploadId:%s\n", objectId, res.UploadID)
+ multipartUpload.Bucket = object.BucketName
+ multipartUpload.Key = object.ObjectKey
+ multipartUpload.UploadId = res.UploadID
+ multipartUpload.ObjectId = objectId
+ return multipartUpload, nil
+ }
+}
+
+func (ad *CephAdapter) UploadPart(ctx context.Context, stream io.Reader, multipartUpload *pb.MultipartUpload,
+ partNumber int64, upBytes int64) (*model.UploadPartResult, error) {
+ bucket := ad.session.NewBucket()
+ log.Infof("upload part[Ceph S3], objectId:%s, bucket:%s\n", multipartUpload.ObjectId, bucket)
+
+ cephObject := bucket.NewObject(ad.backend.BucketName)
+ uploader := cephObject.NewUploads(multipartUpload.ObjectId)
+
+ d, err := ioutil.ReadAll(stream)
+ data := []byte(d)
+ body := ioutil.NopCloser(bytes.NewReader(data))
+ contentMD5 := utils.Md5Content(data)
+ part, err := uploader.UploadPart(int(partNumber), multipartUpload.UploadId, contentMD5, "", upBytes, body)
+ if err != nil {
+ log.Errorf("upload part[Ceph S3] failed, err:%v\n", err)
+ return nil, ErrPutToBackendFailed
+ } else {
+ log.Infof("uploaded part[Ceph S3] #%d successfully, ETag:%s\n", partNumber, part.Etag)
+ result := &model.UploadPartResult{
+ Xmlns: model.Xmlns,
+ ETag: part.Etag,
+ PartNumber: partNumber}
+ return result, nil
+ }
+
+ log.Error("upload part[Ceph S3]: should not be here.")
+ return nil, ErrInternalError
+}
+
+func (ad *CephAdapter) CompleteMultipartUpload(ctx context.Context, multipartUpload *pb.MultipartUpload,
+ completeUpload *model.CompleteMultipartUpload) (*model.CompleteMultipartUploadResult, error) {
+ bucket := ad.session.NewBucket()
+ log.Infof("complete multipart upload[Ceph S3], objectId:%s, bucket:%s\n", multipartUpload.ObjectId, bucket)
+
+ cephObject := bucket.NewObject(ad.backend.BucketName)
+ uploader := cephObject.NewUploads(multipartUpload.ObjectId)
+ var completeParts []CompletePart
+ for _, p := range completeUpload.Parts {
+ completePart := CompletePart{
+ Etag: p.ETag,
+ PartNumber: int(p.PartNumber),
+ }
+ completeParts = append(completeParts, completePart)
+ }
+ resp, err := uploader.Complete(multipartUpload.UploadId, completeParts)
+ if err != nil {
+ log.Infof("complete multipart upload[Ceph S3] failed, objectId:%s, err:%v\n", multipartUpload.ObjectId, err)
+ return nil, ErrBackendCompleteMultipartFailed
+ }
+ result := &model.CompleteMultipartUploadResult{
+ Xmlns: model.Xmlns,
+ Location: ad.backend.Endpoint,
+ Bucket: multipartUpload.Bucket,
+ Key: multipartUpload.Key,
+ ETag: resp.Etag,
+ }
+
+ log.Infof("complete multipart upload[Ceph S3] succeed, objectId:%s, resp:%v\n", multipartUpload.ObjectId, resp)
+ return result, nil
+}
+
+func (ad *CephAdapter) AbortMultipartUpload(ctx context.Context, multipartUpload *pb.MultipartUpload) error {
+ bucket := ad.session.NewBucket()
+ cephObject := bucket.NewObject(ad.backend.BucketName)
+ uploader := cephObject.NewUploads(multipartUpload.ObjectId)
+ log.Infof("abort multipart upload[Ceph S3], objectId:%s, bucket:%s\n", multipartUpload.ObjectId, bucket)
+
+ err := uploader.RemoveUploads(multipartUpload.UploadId)
+ if err != nil {
+ log.Infof("abort multipart upload[Ceph S3] failed, objectId:%s, err:%v\n", multipartUpload.ObjectId, err)
+ return ErrBackendAbortMultipartFailed
+ } else {
+ log.Infof("abort multipart upload[Ceph S3] succeed, objectId:%s, err:%v\n", multipartUpload.ObjectId, err)
+ }
+
+ return nil
+}
+
+/*func (ad *CephAdapter) ListParts(context context.Context, listParts *pb.ListParts) (*model.ListPartsOutput, error) {
+ newObjectKey := listParts.Bucket + "/" + listParts.Key
+ bucket := ad.session.NewBucket()
+ cephObject := bucket.NewObject(ad.backend.BucketName)
+ uploader := cephObject.NewUploads(newObjectKey)
+
+ listPartsResult, err := uploader.ListPart(listParts.UploadId)
+ if err != nil {
+ log.Infof("list parts failed, err:%v\n", err)
+ return nil, S3Error{Code: 500, Description: err.Error()}.Error()
+ } else {
+ log.Infof("List parts successful\n")
+ var parts []model.Part
+ for _, p := range listPartsResult.Parts {
+ part := model.Part{
+ ETag: p.Etag,
+ PartNumber: int64(p.PartNumber),
+ }
+ parts = append(parts, part)
+ }
+ listPartsOutput := &model.ListPartsOutput{
+ Xmlns: model.Xmlns,
+ Key: listPartsResult.Key,
+ Bucket: listParts.Bucket,
+ IsTruncated: listPartsResult.IsTruncated,
+ MaxParts: listPartsResult.MaxParts,
+ Owner: model.Owner{
+ ID: listPartsResult.Owner.OwnerID,
+ DisplayName: listPartsResult.Owner.DisplayName,
+ },
+ UploadId: listPartsResult.UploadID,
+ Parts: parts,
+ }
+
+ return listPartsOutput, nil
+ }
+}*/
+
+func (ad *CephAdapter) ListParts(ctx context.Context, multipartUpload *pb.ListParts) (*model.ListPartsOutput, error) {
+ return nil, ErrNotImplemented
+}
+
+func (ad *CephAdapter) Close() error {
+ // TODO
+ return nil
+}
+
+
+
package ceph
+
+import (
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/driver"
+ "github.com/webrtcn/s3client"
+ "github.com/opensds/multi-cloud/backend/pkg/utils/constants"
+)
+
+type CephS3DriverFactory struct {
+}
+
+func (cdf *CephS3DriverFactory) CreateDriver(backend *backendpb.BackendDetail) (driver.StorageDriver, error) {
+ endpoint := backend.Endpoint
+ AccessKeyID := backend.Access
+ AccessKeySecret := backend.Security
+ sess := s3client.NewClient(endpoint, AccessKeyID, AccessKeySecret)
+ adap := &CephAdapter{backend: backend, session: sess}
+
+ return adap, nil
+}
+
+func init() {
+ driver.RegisterDriverFactory(constants.BackendTypeCeph, &CephS3DriverFactory{})
+}
+
+
+
package common
+
+import (
+ "context"
+)
+
+func GetMd5FromCtx(ctx context.Context) (md5val string) {
+ // md5 provided by user for uploading object.
+ if val := ctx.Value(CONTEXT_KEY_MD5); val != nil {
+ md5val = val.(string)
+ }
+
+ return
+}
+
+func TrimQuot(in string) string {
+ s := in
+ l := len(s)
+ if l <= 0 {
+ return ""
+ }
+ if s[l-1] == '"' {
+ s = s[:l-1]
+ }
+ if s[0] == '"' {
+ s = s[1:]
+ }
+
+ return s
+}
+
+
+
package driver
+
+type Closer interface {
+ Close()
+}
+
+var closers []Closer
+
+func AddCloser(closer Closer) {
+ closers = append(closers, closer)
+}
+
+func FreeCloser() {
+ for _, c := range closers {
+ c.Close()
+ }
+}
+
+
+
package driver
+
+import (
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ exp "github.com/opensds/multi-cloud/s3/pkg/exception"
+)
+
+type DriverFactory interface {
+ CreateDriver(detail *backendpb.BackendDetail) (StorageDriver, error)
+}
+
+var driverFactoryMgr = make(map[string]DriverFactory)
+
+func RegisterDriverFactory(driverType string, factory DriverFactory) {
+ driverFactoryMgr[driverType] = factory
+}
+
+func CreateStorageDriver(driverType string, detail *backendpb.BackendDetail) (StorageDriver, error) {
+ if factory, ok := driverFactoryMgr[driverType]; ok {
+ return factory.CreateDriver(detail)
+ }
+ return nil, exp.NoSuchType.Error()
+}
+
+
+
package hws
+
+import (
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/driver"
+ "github.com/opensds/multi-cloud/api/pkg/utils/obs"
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ "github.com/opensds/multi-cloud/backend/pkg/utils/constants"
+)
+
+type HWObsDriverFactory struct {
+
+}
+
+func (cdf *HWObsDriverFactory) CreateDriver(backend *backendpb.BackendDetail) (driver.StorageDriver, error) {
+ endpoint := backend.Endpoint
+ AccessKeyID := backend.Access
+ AccessKeySecret := backend.Security
+
+ client, err := obs.New(AccessKeyID, AccessKeySecret, endpoint)
+ if err != nil {
+ return nil, err
+ }
+
+ adap := &OBSAdapter{backend: backend, client: client}
+
+ return adap, nil
+}
+
+func init() {
+ driver.RegisterDriverFactory(constants.BackendTypeObs, &HWObsDriverFactory{})
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hws
+
+import (
+ "context"
+ "io"
+ "time"
+
+ "encoding/base64"
+ "encoding/hex"
+ "github.com/opensds/multi-cloud/api/pkg/utils/obs"
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ . "github.com/opensds/multi-cloud/s3/error"
+ dscommon "github.com/opensds/multi-cloud/s3/pkg/datastore/common"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ osdss3 "github.com/opensds/multi-cloud/s3/pkg/service"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+type OBSAdapter struct {
+ backend *backendpb.BackendDetail
+ client *obs.ObsClient
+}
+
+func (ad *OBSAdapter) Put(ctx context.Context, stream io.Reader, object *pb.Object) (dscommon.PutResult, error) {
+ bucket := ad.backend.BucketName
+ objectId := object.BucketName + "/" + object.ObjectKey
+ result := dscommon.PutResult{}
+ userMd5 := dscommon.GetMd5FromCtx(ctx)
+ size := object.Size
+ log.Infof("put object[OBS], objectId:%s, bucket:%s, size=%d, userMd5=%s\n", objectId, bucket, size, userMd5)
+
+ if object.Tier == 0 {
+ // default
+ object.Tier = utils.Tier1
+ }
+ storClass, err := osdss3.GetNameFromTier(object.Tier, utils.OSTYPE_OBS)
+ if err != nil {
+ log.Errorf("translate tier[%d] to aws storage class failed\n", object.Tier)
+ return result, ErrInternalError
+ }
+
+ input := &obs.PutObjectInput{}
+ input.Bucket = bucket
+ input.Key = objectId
+ input.Body = stream
+ input.ContentLength = size
+ input.StorageClass = obs.StorageClassType(storClass)
+ if userMd5 != "" {
+ md5Bytes, err := hex.DecodeString(userMd5)
+ if err != nil {
+ log.Warnf("user input md5 is abandoned, cause decode md5 failed, err:%v\n", err)
+ } else {
+ input.ContentMD5 = base64.StdEncoding.EncodeToString(md5Bytes)
+ log.Debugf("input.ContentMD5=%s\n", input.ContentMD5)
+ }
+ }
+
+ log.Infof("upload object[OBS] begin, objectId:%s\n", objectId)
+ out, err := ad.client.PutObject(input)
+ log.Infof("upload object[OBS] end, objectId:%s\n", objectId)
+ if err != nil {
+ log.Errorf("upload object[OBS] failed, objectId:%s, err:%v", objectId, err)
+ return result, ErrPutToBackendFailed
+ }
+
+ result.Etag = dscommon.TrimQuot(out.ETag)
+ if userMd5 != "" && userMd5 != result.Etag {
+ log.Error("### MD5 not match, result.Etag:", result.Etag, ", userMd5:", userMd5)
+ return result, ErrBadDigest
+ }
+
+ result.ObjectId = objectId
+ result.UpdateTime = time.Now().Unix()
+ result.Meta = out.VersionId
+ result.Written = size
+ log.Infof("upload object[OBS] succeed, objectId:%s, UpdateTime is:%v\n", objectId, result.UpdateTime)
+
+ return result, nil
+}
+
+func (ad *OBSAdapter) Get(ctx context.Context, object *pb.Object, start int64, end int64) (io.ReadCloser, error) {
+ bucket := ad.backend.BucketName
+ objectId := object.ObjectId
+ log.Infof("get object[OBS], objectId:%s, bucket:%s\n", objectId, bucket)
+
+ input := &obs.GetObjectInput{}
+ input.Bucket = bucket
+ input.Key = objectId
+ if start != 0 || end != 0 {
+ input.RangeStart = start
+ input.RangeEnd = end
+ }
+
+ out, err := ad.client.GetObject(input)
+ if err != nil {
+ log.Infof("get object[OBS] failed, objectId:%,s err:%v", objectId, err)
+ return nil, ErrGetFromBackendFailed
+ }
+
+ log.Infof("get object[OBS] succeed, objectId:%s\n", objectId)
+ return out.Body, nil
+}
+
+func (ad *OBSAdapter) Delete(ctx context.Context, object *pb.DeleteObjectInput) error {
+ objectId := object.Bucket + "/" + object.Key
+ log.Infof("delete object[OBS], objectId:%s\n", objectId)
+
+ deleteObjectInput := obs.DeleteObjectInput{Bucket: ad.backend.BucketName, Key: objectId}
+ _, err := ad.client.DeleteObject(&deleteObjectInput)
+ if err != nil {
+ log.Infof("delete object[OBS] failed, objectId:%s, :%v", objectId, err)
+ return ErrDeleteFromBackendFailed
+ }
+
+ log.Infof("delete object[OBS] succeed, objectId:%s.\n", objectId)
+ return nil
+}
+
+func (ad *OBSAdapter) ChangeStorageClass(ctx context.Context, object *pb.Object, newClass *string) error {
+ log.Infof("change storage class[OBS] of object[%s] to %s .\n", object.ObjectId, newClass)
+
+ input := &obs.CopyObjectInput{}
+ input.Bucket = ad.backend.BucketName
+ input.Key = object.ObjectId
+ input.CopySourceBucket = ad.backend.BucketName
+ input.CopySourceKey = object.ObjectId
+ input.MetadataDirective = obs.CopyMetadata
+ switch *newClass {
+ case "STANDARD_IA":
+ input.StorageClass = obs.StorageClassWarm
+ case "GLACIER":
+ input.StorageClass = obs.StorageClassCold
+ default:
+ log.Infof("[OBS] unspport storage class:%s", newClass)
+ return ErrInvalidStorageClass
+ }
+ _, err := ad.client.CopyObject(input)
+ if err != nil {
+ log.Errorf("[OBS] change storage class of object[%s] to %s failed: %v\n", object.ObjectId, newClass, err)
+ return ErrPutToBackendFailed
+ } else {
+ log.Infof("[OBS] change storage class of object[%s] to %s succeed.\n", object.ObjectId, newClass)
+ }
+
+ return nil
+}
+
+func (ad *OBSAdapter) Copy(ctx context.Context, stream io.Reader, target *pb.Object) (result dscommon.PutResult, err error) {
+ return
+}
+
+/*func (ad *OBSAdapter) GetObjectInfo(bucketName string, key string, context context.Context) (*pb.Object, S3Error) {
+ return nil, nil
+}*/
+
+func (ad *OBSAdapter) InitMultipartUpload(ctx context.Context, object *pb.Object) (*pb.MultipartUpload, error) {
+ bucket := ad.backend.BucketName
+ objectId := object.BucketName + "/" + object.ObjectKey
+ multipartUpload := &pb.MultipartUpload{}
+ log.Infof("init multipart upload[OBS], objectId:%s, bucket:%s\n", objectId, bucket)
+
+ input := &obs.InitiateMultipartUploadInput{}
+ input.Bucket = bucket
+ input.Key = objectId
+ storClass, err := osdss3.GetNameFromTier(object.Tier, utils.OSTYPE_OBS)
+ if err != nil {
+ log.Errorf("translate tier[%d] to obs storage class failed\n", object.Tier)
+ return nil, ErrInternalError
+ }
+ input.StorageClass = obs.StorageClassType(storClass)
+
+ out, err := ad.client.InitiateMultipartUpload(input)
+ if err != nil {
+ log.Infof("init multipart upload[OBS] failed, objectId:%s, err:%v", objectId, err)
+ return nil, ErrBackendInitMultipartFailed
+ }
+
+ multipartUpload.Bucket = out.Bucket
+ multipartUpload.Key = out.Key
+ multipartUpload.UploadId = out.UploadId
+ multipartUpload.ObjectId = objectId
+ log.Infof("init multipart upload[OBS] succeed, objectId:%s\n", objectId)
+ return multipartUpload, nil
+}
+
+func (ad *OBSAdapter) UploadPart(ctx context.Context, stream io.Reader, multipartUpload *pb.MultipartUpload,
+ partNumber int64, upBytes int64) (*model.UploadPartResult, error) {
+ bucket := ad.backend.BucketName
+ objectId := multipartUpload.Bucket + "/" + multipartUpload.Key
+ log.Infof("upload part[OBS], objectId:%s, partNum:%d, bytes:%d\n", objectId, partNumber, upBytes)
+
+ input := &obs.UploadPartInput{}
+ input.Bucket = bucket
+ input.Key = objectId
+ input.Body = stream
+ input.PartNumber = int(partNumber)
+ input.PartSize = upBytes
+ log.Infof(" multipartUpload.UploadId is %v", multipartUpload.UploadId)
+ input.UploadId = multipartUpload.UploadId
+ out, err := ad.client.UploadPart(input)
+
+ if err != nil {
+ log.Infof("upload part[OBS] failed, objectId:%s, err:%v", objectId, err)
+ return nil, ErrPutToBackendFailed
+ }
+
+ log.Infof("upload part[OBS] succeed, objectId:%s, partNum:%d\n", objectId, out.PartNumber)
+ result := &model.UploadPartResult{ETag: out.ETag, PartNumber: partNumber}
+
+ return result, nil
+}
+
+func (ad *OBSAdapter) CompleteMultipartUpload(ctx context.Context, multipartUpload *pb.MultipartUpload,
+ completeUpload *model.CompleteMultipartUpload) (*model.CompleteMultipartUploadResult, error) {
+ bucket := ad.backend.BucketName
+ objectId := multipartUpload.Bucket + "/" + multipartUpload.Key
+ log.Infof("complete multipart upload[OBS], objectId:%s, bucket:%s\n", objectId, bucket)
+
+ input := &obs.CompleteMultipartUploadInput{}
+ input.Bucket = bucket
+ input.Key = objectId
+ input.UploadId = multipartUpload.UploadId
+ for _, p := range completeUpload.Parts {
+ part := obs.Part{
+ PartNumber: int(p.PartNumber),
+ ETag: p.ETag,
+ }
+ input.Parts = append(input.Parts, part)
+ }
+ resp, err := ad.client.CompleteMultipartUpload(input)
+ if err != nil {
+ log.Errorf("complete multipart upload[OBS] failed, objectid:%s, err:%v\n", objectId, err)
+ return nil, ErrBackendCompleteMultipartFailed
+ }
+ result := &model.CompleteMultipartUploadResult{
+ Xmlns: model.Xmlns,
+ Location: resp.Location,
+ Bucket: resp.Bucket,
+ Key: resp.Key,
+ ETag: resp.ETag,
+ }
+ if err != nil {
+ log.Infof("complete multipart upload[OBS] failed, objectid:%s, err:%v", objectId, err)
+ return nil, ErrBackendCompleteMultipartFailed
+ }
+
+ log.Infof("complete multipart upload[OBS] succeed, objectId:%s\n", objectId)
+ return result, nil
+}
+
+func (ad *OBSAdapter) AbortMultipartUpload(ctx context.Context, multipartUpload *pb.MultipartUpload) error {
+ bucket := ad.backend.BucketName
+ objectId := multipartUpload.Bucket + "/" + multipartUpload.Key
+ log.Infof("abort multipart upload[OBS], objectId:%s, bucket:%s\n", objectId, bucket)
+
+ input := &obs.AbortMultipartUploadInput{}
+ input.UploadId = multipartUpload.UploadId
+ input.Bucket = bucket
+ input.Key = objectId
+ _, err := ad.client.AbortMultipartUpload(input)
+ if err != nil {
+ log.Infof("abort multipart upload[OBS] failed, objectId:%s, err:%v", objectId, err)
+ return ErrBackendAbortMultipartFailed
+ }
+
+ log.Infof("abort multipart upload[OBS] succeed, objectId:%s\n", objectId)
+ return nil
+}
+
+func (ad *OBSAdapter) ListParts(context context.Context, listParts *pb.ListParts) (*model.ListPartsOutput, error) {
+ bucket := ad.backend.BucketName
+ if context.Value("operation") == "listParts" {
+ input := &obs.ListPartsInput{}
+ input.Bucket = bucket
+ input.Key = listParts.Key
+ input.UploadId = listParts.UploadId
+ input.MaxParts = int(listParts.MaxParts)
+ listPartsOutput, err := ad.client.ListParts(input)
+ listParts := &model.ListPartsOutput{}
+ listParts.Bucket = listPartsOutput.Bucket
+ listParts.Key = listPartsOutput.Key
+ listParts.UploadId = listPartsOutput.UploadId
+ listParts.MaxParts = listPartsOutput.MaxParts
+
+ for _, p := range listPartsOutput.Parts {
+ part := model.Part{
+ PartNumber: int64(p.PartNumber),
+ ETag: p.ETag,
+ }
+ listParts.Parts = append(listParts.Parts, part)
+ }
+
+ if err != nil {
+ log.Infof("ListPartsListParts is nil:%v\n", err)
+ return nil, err
+ } else {
+ log.Infof("ListParts successfully")
+ return listParts, nil
+ }
+ }
+ return nil, nil
+}
+
+func (ad *OBSAdapter) Close() error {
+ //TODO
+ return nil
+}
+
+
+
package ibmcos
+
+import (
+ "github.com/opensds/multi-cloud/backend/pkg/utils/constants"
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/aws"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/driver"
+)
+
+type IBMCOSDriverFactory struct {
+}
+
+func (factory *IBMCOSDriverFactory) CreateDriver(backend *backendpb.BackendDetail) (driver.StorageDriver, error) {
+ awss3Fac := &aws.AwsS3DriverFactory{}
+ return awss3Fac.CreateDriver(backend)
+}
+
+func init() {
+ driver.RegisterDriverFactory(constants.BackendTypeAws, &IBMCOSDriverFactory{})
+}
+
+
+
package config
+
+import (
+ "errors"
+ "strings"
+
+ "github.com/spf13/viper"
+)
+
+const (
+ DEFAULT_DB_MAX_IDLE_CONNS = 1024
+ DEFAULT_DB_MAX_OPEN_CONNS = 1024
+ DEFAULT_GC_CHECK_TIME = 5
+)
+
+type Config struct {
+ Endpoint EndpointConfig
+ Log LogConfig
+ StorageCfg StorageConfig
+ Database DatabaseConfig
+}
+
+func (config *Config) Parse() error {
+ endpoint := viper.GetStringMap("endpoint")
+ log := viper.GetStringMap("log")
+ storageCfg := viper.GetStringMap("storage")
+ db := viper.GetStringMap("database")
+
+ (&config.Endpoint).Parse(endpoint)
+ (&config.Log).Parse(log)
+ (&config.StorageCfg).Parse(storageCfg)
+ (&config.Database).Parse(db)
+
+ return nil
+}
+
+type CommonConfig struct {
+ Log LogConfig
+ Cache CacheConfig
+}
+
+func (cc *CommonConfig) Parse() error {
+ log := viper.GetStringMap("log")
+ cache := viper.GetStringMap("cache")
+
+ cc.Log.Parse(log)
+ cc.Cache.Parse(cache)
+
+ return nil
+}
+
+type EndpointConfig struct {
+ Url string
+ MachineId int
+ // how frequency to perform a gc in seconds.
+ GcCheckTime int64
+}
+
+func (ec *EndpointConfig) Parse(vals map[string]interface{}) error {
+ ec.GcCheckTime = DEFAULT_GC_CHECK_TIME
+ if url, ok := vals["url"]; ok {
+ ec.Url = url.(string)
+ return nil
+ } else {
+ return errors.New("no url found")
+ }
+
+ if id, ok := vals["machine_id"]; ok {
+ ec.MachineId = id.(int)
+ return nil
+ } else {
+ return errors.New("no machine_id found")
+ }
+ if gc, ok := vals["gc_check_time"]; ok {
+ ec.GcCheckTime = gc.(int64)
+ return nil
+ }
+ return nil
+}
+
+type LogConfig struct {
+ Path string
+ Level int
+}
+
+func (lc *LogConfig) Parse(vals map[string]interface{}) error {
+ if p, ok := vals["log_path"]; ok {
+ lc.Path = p.(string)
+ }
+ if l, ok := vals["log_level"]; ok {
+ lc.Level = int(l.(int64))
+ }
+ return nil
+}
+
+type StorageConfig struct {
+ CephPath string
+}
+
+func (sc *StorageConfig) Parse(vals map[string]interface{}) error {
+ if p, ok := vals["ceph_dir"]; ok {
+ sc.CephPath = p.(string)
+ }
+ return nil
+}
+
+type CacheConfig struct {
+ Mode int
+ Nodes []string
+ Master string
+ Address string
+ Password string
+ ConnectionTimeout int
+ ReadTimeout int
+ WriteTimeout int
+ KeepAlive int
+ PoolMaxIdle int
+ PoolIdleTimeout int
+}
+
+func (cc *CacheConfig) Parse(vals map[string]interface{}) error {
+ if m, ok := vals["redis_mode"]; ok {
+ cc.Mode = int(m.(int64))
+ }
+ if n, ok := vals["redis_nodes"]; ok {
+ nodes := n.(string)
+ cc.Nodes = strings.Split(nodes, ",")
+ }
+ if master, ok := vals["redis_master_name"]; ok {
+ cc.Master = master.(string)
+ }
+ if addr, ok := vals["redis_address"]; ok {
+ cc.Address = addr.(string)
+ }
+ if password, ok := vals["redis_password"]; ok {
+ cc.Password = password.(string)
+ }
+ if ct, ok := vals["redis_connect_timeout"]; ok {
+ cc.ConnectionTimeout = int(ct.(int64))
+ }
+ if rt, ok := vals["redis_read_timeout"]; ok {
+ cc.ReadTimeout = int(rt.(int64))
+ }
+ if wt, ok := vals["redis_write_timeout"]; ok {
+ cc.WriteTimeout = int(wt.(int64))
+ }
+ if ka, ok := vals["redis_keepalive"]; ok {
+ cc.KeepAlive = int(ka.(int64))
+ }
+ if pa, ok := vals["redis_pool_max_idle"]; ok {
+ cc.PoolMaxIdle = int(pa.(int64))
+ }
+ if pt, ok := vals["redis_pool_idle_timeout"]; ok {
+ cc.PoolIdleTimeout = int(pt.(int64))
+ }
+
+ return nil
+}
+
+type DatabaseConfig struct {
+ DbType string
+ DbUrl string
+ DbPassword string
+ MaxIdleConns int
+ MaxOpenConns int
+}
+
+func (dc *DatabaseConfig) Parse(vals map[string]interface{}) error {
+ dc.MaxIdleConns = DEFAULT_DB_MAX_IDLE_CONNS
+ dc.MaxOpenConns = DEFAULT_DB_MAX_OPEN_CONNS
+
+ if dt, ok := vals["db_type"]; ok {
+ dc.DbType = dt.(string)
+ }
+ if du, ok := vals["db_url"]; ok {
+ dc.DbUrl = du.(string)
+ }
+ if dp, ok := vals["db_password"]; ok {
+ dc.DbPassword = dp.(string)
+ }
+ if mi, ok := vals["db_maxidleconns"]; ok {
+ dc.MaxIdleConns = mi.(int)
+ }
+ if mc, ok := vals["db_maxopenconns"]; ok {
+ dc.MaxOpenConns = mc.(int)
+ }
+
+ return nil
+}
+
+
+
package config
+
+import (
+ "os"
+ "path/filepath"
+
+ "github.com/spf13/viper"
+)
+
+type FuncConfigParse func(config *Config) error
+
+func ReadCommonConfig(dir string) (*CommonConfig, error) {
+ viper.AddConfigPath(dir)
+ viper.SetConfigName("common")
+ err := viper.ReadInConfig()
+ if err != nil {
+ return nil, err
+ }
+ cc := &CommonConfig{}
+ err = cc.Parse()
+ if err != nil {
+ return nil, err
+ }
+ return cc, nil
+}
+
+func ReadConfigs(dir string, funcConfigParse FuncConfigParse) error {
+ err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+
+ if info.Name() == "common.toml" {
+ return nil
+ }
+
+ viper.SetConfigFile(path)
+ err = viper.ReadInConfig()
+ if err != nil {
+ // skip the config file which is failed to parse and continue the next one.
+ return nil
+ }
+ config := &Config{}
+ err = config.Parse()
+ if err != nil {
+ return err
+ }
+ err = funcConfigParse(config)
+ if err != nil {
+ return err
+ }
+ return nil
+ })
+
+ return err
+}
+
+
+
package config
+
+import (
+ "errors"
+ "strings"
+ "sync"
+
+ "github.com/fsnotify/fsnotify"
+ "github.com/spf13/viper"
+)
+
+const (
+ CFG_SUFFIX = "toml"
+)
+
+type ConfigWatcher struct {
+ FuncConfigParse FuncConfigParse
+ watcher *fsnotify.Watcher
+ stopSignal chan bool
+ wg sync.WaitGroup
+ err error
+}
+
+func (cw *ConfigWatcher) Error() error {
+ return cw.err
+}
+
+func (cw *ConfigWatcher) Stop() {
+ cw.stopSignal <- true
+ close(cw.stopSignal)
+ cw.wg.Wait()
+ cw.watcher.Close()
+}
+
+func (cw *ConfigWatcher) Watch(dir string) {
+ mask := fsnotify.Write | fsnotify.Create
+ cw.wg.Add(1)
+ go func() {
+ defer cw.wg.Done()
+ for {
+ select {
+ case event, ok := <-cw.watcher.Events:
+ if !ok {
+ cw.err = errors.New("failed to read watcher events.")
+ return
+ }
+ if event.Op&mask != 0 {
+ // got we need.
+ if strings.HasSuffix(event.Name, CFG_SUFFIX) {
+ viper.SetConfigFile(event.Name)
+ err := viper.ReadInConfig()
+ if err != nil {
+ cw.err = err
+ return
+ }
+ cfg := &Config{}
+ err = cfg.Parse()
+ if err != nil {
+ cw.err = err
+ return
+ }
+ err = cw.FuncConfigParse(cfg)
+ if err != nil {
+ cw.err = err
+ return
+ }
+ }
+ }
+ case err := <-cw.watcher.Errors:
+ if err != nil {
+ cw.err = err
+ return
+ }
+ case stopped := <-cw.stopSignal:
+ if stopped {
+ cw.err = nil
+ return
+ }
+ }
+ }
+ }()
+}
+
+func NewConfigWatcher(funcConfigParse FuncConfigParse) (*ConfigWatcher, error) {
+ watcher, err := fsnotify.NewWatcher()
+ if err != nil {
+ return nil, err
+ }
+
+ cw := &ConfigWatcher{
+ FuncConfigParse: funcConfigParse,
+ watcher: watcher,
+ stopSignal: make(chan bool),
+ err: nil,
+ }
+ return cw, nil
+}
+
+
+
package crypto
+
+import (
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ log "github.com/sirupsen/logrus"
+)
+
+func NewKMS() KMS {
+ switch helper.CONFIG.KMS.Type {
+ case "vault":
+ c, err := NewVaultConfig()
+ if err != nil {
+ panic("read kms vault err:" + err.Error())
+ }
+ vault, err := NewVault(c)
+ if err != nil {
+ panic("create vault err:" + err.Error())
+ }
+ return vault
+
+ //extention case here
+
+ default:
+ log.Error("not support kms type", helper.CONFIG.KMS.Type)
+ return nil
+ }
+}
+
+
+
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import "errors"
+
+// Error is the generic type for any error happening during decrypting
+// an object. It indicates that the object itself or its metadata was
+// modified accidentally or maliciously.
+type Error struct{ msg string }
+
+func (e Error) Error() string { return e.msg }
+
+var (
+ // ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
+ // is not supported.
+ ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+
+ // ErrInvalidCustomerAlgorithm indicates that the specified SSE-C algorithm
+ // is not supported.
+ ErrInvalidCustomerAlgorithm = errors.New("The SSE-C algorithm is not supported")
+
+ // ErrMissingCustomerKey indicates that the HTTP headers contains no SSE-C client key.
+ ErrMissingCustomerKey = errors.New("The SSE-C request is missing the customer key")
+
+ // ErrMissingCustomerKeyMD5 indicates that the HTTP headers contains no SSE-C client key
+ // MD5 checksum.
+ ErrMissingCustomerKeyMD5 = errors.New("The SSE-C request is missing the customer key MD5")
+
+ // ErrInvalidCustomerKey indicates that the SSE-C client key is not valid - e.g. not a
+ // base64-encoded string or not 256 bits long.
+ ErrInvalidCustomerKey = errors.New("The SSE-C client key is invalid")
+
+ // ErrSecretKeyMismatch indicates that the provided secret key (SSE-C client key / SSE-S3 KMS key)
+ // does not match the secret key used during encrypting the object.
+ ErrSecretKeyMismatch = errors.New("The secret key does not match the secret key used during upload")
+
+ // ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the
+ // computed MD5 sum. This means that the client provided either the wrong key for
+ // a certain MD5 checksum or the wrong MD5 for a certain key.
+ ErrCustomerKeyMD5Mismatch = errors.New("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
+ // ErrIncompatibleEncryptionMethod indicates that both SSE-C headers and SSE-S3 headers were specified, and are incompatible
+ // The client needs to remove the SSE-S3 header or the SSE-C headers
+ ErrIncompatibleEncryptionMethod = errors.New("Server side encryption specified with both SSE-C and SSE-S3 headers")
+)
+
+var (
+ errMissingInternalIV = Error{"The object metadata is missing the internal encryption IV"}
+ errMissingInternalSealAlgorithm = Error{"The object metadata is missing the internal seal algorithm"}
+
+ errInvalidInternalIV = Error{"The internal encryption IV is malformed"}
+ errInvalidInternalSealAlgorithm = Error{"The internal seal algorithm is invalid and not supported"}
+)
+
+var (
+ // errOutOfEntropy indicates that the a source of randomness (PRNG) wasn't able
+ // to produce enough random data. This is fatal error and should cause a panic.
+ errOutOfEntropy = errors.New("Unable to read enough randomness from the system")
+)
+
+
+
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+ "bytes"
+ "crypto/md5"
+ "encoding/base64"
+ "net/http"
+ "strings"
+)
+
+// SSEHeader is the general AWS SSE HTTP header key.
+const SSEHeader = "X-Amz-Server-Side-Encryption"
+
+const (
+ // SSEKmsID is the HTTP header key referencing the SSE-KMS
+ // key ID.
+ SSEKmsID = SSEHeader + "-Aws-Kms-Key-Id"
+
+ // SSEKmsContext is the HTTP header key referencing the
+ // SSE-KMS encryption context.
+ SSEKmsContext = SSEHeader + "-Context"
+)
+
+const (
+ // SSECAlgorithm is the HTTP header key referencing
+ // the SSE-C algorithm.
+ SSECAlgorithm = SSEHeader + "-Customer-Algorithm"
+
+ // SSECKey is the HTTP header key referencing the
+ // SSE-C client-provided key..
+ SSECKey = SSEHeader + "-Customer-Key"
+
+ // SSECKeyMD5 is the HTTP header key referencing
+ // the MD5 sum of the client-provided key.
+ SSECKeyMD5 = SSEHeader + "-Customer-Key-Md5"
+)
+
+const (
+ // SSECopyAlgorithm is the HTTP header key referencing
+ // the SSE-C algorithm for SSE-C copy requests.
+ SSECopyAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
+
+ // SSECopyKey is the HTTP header key referencing the SSE-C
+ // client-provided key for SSE-C copy requests.
+ SSECopyKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
+
+ // SSECopyKeyMD5 is the HTTP header key referencing the
+ // MD5 sum of the client key for SSE-C copy requests.
+ SSECopyKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5"
+)
+
+const (
+ // SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
+ // For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
+ // For SSE-C see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
+ SSEAlgorithmAES256 = "AES256"
+
+ // SSEAlgorithmKMS is the value of 'X-Amz-Server-Side-Encryption' for SSE-KMS.
+ // See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+ SSEAlgorithmKMS = "aws:kms"
+)
+
+// RemoveSensitiveHeaders removes confidential encryption
+// information - e.g. the SSE-C key - from the HTTP headers.
+// It has the same semantics as RemoveSensitiveEntires.
+func RemoveSensitiveHeaders(h http.Header) {
+ h.Del(SSECKey)
+ h.Del(SSECopyKey)
+}
+
+// S3 represents AWS SSE-S3. It provides functionality to handle
+// SSE-S3 requests.
+var S3 = s3{}
+
+type s3 struct{}
+
+// IsRequested returns true if the HTTP headers indicates that
+// the S3 client requests SSE-S3.
+func (s3) IsRequested(h http.Header) bool {
+ _, ok := h[SSEHeader]
+ return ok && strings.ToLower(h.Get(SSEHeader)) != SSEAlgorithmKMS // Return only true if the SSE header is specified and does not contain the SSE-KMS value
+}
+
+// ParseHTTP parses the SSE-S3 related HTTP headers and checks
+// whether they contain valid values.
+func (s3) ParseHTTP(h http.Header) (err error) {
+ if h.Get(SSEHeader) != SSEAlgorithmAES256 {
+ err = ErrInvalidEncryptionMethod
+ }
+ return
+}
+
+// S3KMS represents AWS SSE-KMS. It provides functionality to
+// handle SSE-KMS requests.
+var S3KMS = s3KMS{}
+
+type s3KMS struct{}
+
+// IsRequested returns true if the HTTP headers indicates that
+// the S3 client requests SSE-KMS.
+func (s3KMS) IsRequested(h http.Header) bool {
+ if _, ok := h[SSEKmsID]; ok {
+ return true
+ }
+ if _, ok := h[SSEKmsContext]; ok {
+ return true
+ }
+ if _, ok := h[SSEHeader]; ok {
+ return strings.ToUpper(h.Get(SSEHeader)) != SSEAlgorithmAES256 // Return only true if the SSE header is specified and does not contain the SSE-S3 value
+ }
+ return false
+}
+
+var (
+ // SSEC represents AWS SSE-C. It provides functionality to handle
+ // SSE-C requests.
+ SSEC = ssec{}
+
+ // SSECopy represents AWS SSE-C for copy requests. It provides
+ // functionality to handle SSE-C copy requests.
+ SSECopy = ssecCopy{}
+)
+
+type ssec struct{}
+type ssecCopy struct{}
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C header. SSE-C copy headers are ignored.
+func (ssec) IsRequested(h http.Header) bool {
+ if _, ok := h[SSECAlgorithm]; ok {
+ return true
+ }
+ if _, ok := h[SSECKey]; ok {
+ return true
+ }
+ if _, ok := h[SSECKeyMD5]; ok {
+ return true
+ }
+ return false
+}
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C copy header. Regular SSE-C headers
+// are ignored.
+func (ssecCopy) IsRequested(h http.Header) bool {
+ if _, ok := h[SSECopyAlgorithm]; ok {
+ return true
+ }
+ if _, ok := h[SSECopyKey]; ok {
+ return true
+ }
+ if _, ok := h[SSECopyKeyMD5]; ok {
+ return true
+ }
+ return false
+}
+
+// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
+// on success. SSE-C copy headers are ignored.
+func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
+ if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
+ return key, ErrInvalidCustomerAlgorithm
+ }
+ if h.Get(SSECKey) == "" {
+ return key, ErrMissingCustomerKey
+ }
+ if h.Get(SSECKeyMD5) == "" {
+ return key, ErrMissingCustomerKeyMD5
+ }
+
+ clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECKey))
+ if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+ return key, ErrInvalidCustomerKey
+ }
+ keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECKeyMD5))
+ if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+ return key, ErrCustomerKeyMD5Mismatch
+ }
+ copy(key[:], clientKey)
+ return key, nil
+}
+
+// ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key
+// on success. Regular SSE-C headers are ignored.
+func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) {
+ if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
+ return key, ErrInvalidCustomerAlgorithm
+ }
+ if h.Get(SSECopyKey) == "" {
+ return key, ErrMissingCustomerKey
+ }
+ if h.Get(SSECopyKeyMD5) == "" {
+ return key, ErrMissingCustomerKeyMD5
+ }
+
+ clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKey))
+ if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+ return key, ErrInvalidCustomerKey
+ }
+ keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKeyMD5))
+ if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+ return key, ErrCustomerKeyMD5Mismatch
+ }
+ copy(key[:], clientKey)
+ return key, nil
+}
+
+
+
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+ "crypto/hmac"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/binary"
+ "io"
+
+ log "github.com/sirupsen/logrus"
+)
+
+// ObjectKey is a 256 bit secret key used to encrypt the object.
+// It must never be stored in plaintext.
+type ObjectKey [32]byte
+
+// GenerateKey generates a unique ObjectKey from a 256 bit external key
+// and a source of randomness. If random is nil the default PRNG of the
+// system (crypto/rand) is used.
+func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
+ if random == nil {
+ random = rand.Reader
+ }
+ var nonce [32]byte
+ if _, err := io.ReadFull(random, nonce[:]); err != nil {
+ log.Error(errOutOfEntropy)
+ return key
+ }
+ sha := sha256.New()
+ sha.Write(extKey[:])
+ sha.Write(nonce[:])
+ sha.Sum(key[:0])
+ return key
+}
+
+// SealedKey represents a sealed object key. It can be stored
+// at an untrusted location.
+type SealedKey struct {
+ Key [64]byte // The encrypted and authenticted object-key.
+ IV [32]byte // The random IV used to encrypt the object-key.
+ Algorithm string // The sealing algorithm used to encrypt the object key.
+}
+
+// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index.
+func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
+ var bin [4]byte
+ binary.LittleEndian.PutUint32(bin[:], id)
+
+ mac := hmac.New(sha256.New, key[:])
+ mac.Write(bin[:])
+ mac.Sum(partKey[:0])
+ return partKey
+}
+
+
+
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+ "fmt"
+ "io"
+ "sort"
+)
+
+// Context is a list of key-value pairs cryptographically
+// associated with a certain object.
+type Context map[string]string
+
+// WriteTo writes the context in a canonical from to w.
+// It returns the number of bytes and the first error
+// encounter during writing to w, if any.
+//
+// WriteTo sorts the context keys and writes the sorted
+// key-value pairs as canonical JSON object to w.
+func (c Context) WriteTo(w io.Writer) (n int64, err error) {
+ sortedKeys := make(sort.StringSlice, 0, len(c))
+ for k := range c {
+ sortedKeys = append(sortedKeys, k)
+ }
+ sort.Sort(sortedKeys)
+
+ nn, err := io.WriteString(w, "{")
+ if err != nil {
+ return n + int64(nn), err
+ }
+ n += int64(nn)
+ for i, k := range sortedKeys {
+ s := fmt.Sprintf("\"%s\":\"%s\",", k, c[k])
+ if i == len(sortedKeys)-1 {
+ s = s[:len(s)-1] // remove last ','
+ }
+
+ nn, err = io.WriteString(w, s)
+ if err != nil {
+ return n + int64(nn), err
+ }
+ n += int64(nn)
+ }
+ nn, err = io.WriteString(w, "}")
+ return n + int64(nn), err
+}
+
+// KMS represents an active and authenticted connection
+// to a Key-Management-Service. It supports generating
+// data key generation and unsealing of KMS-generated
+// data keys.
+type KMS interface {
+ // GenerateKey generates a new random data key using
+ // the master key referenced by the keyID. It returns
+ // the plaintext key and the sealed plaintext key
+ // on success.
+ //
+ // The context is cryptographically bound to the
+ // generated key. The same context must be provided
+ // again to unseal the generated key.
+ GenerateKey(keyID string, context Context) (key [32]byte, sealedKey []byte, err error)
+
+ // UnsealKey unseals the sealedKey using the master key
+ // referenced by the keyID. The provided context must
+ // match the context used to generate the sealed key.
+ UnsealKey(keyID string, sealedKey []byte, context Context) (key [32]byte, err error)
+
+ GetKeyID() string
+}
+
+
+
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+// RemoveSensitiveEntries removes confidential encryption
+// information - e.g. the SSE-C key - from the metadata map.
+// It has the same semantics as RemoveSensitiveHeaders.
+func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
+ delete(metadata, SSECKey)
+ delete(metadata, SSECopyKey)
+}
+
+// IsETagSealed returns true if the etag seems to be encrypted.
+func IsETagSealed(etag []byte) bool { return len(etag) > 16 }
+
+
+
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+// String returns the SSE domain as string. For SSE-S3 the
+// domain is "SSE-S3".
+func (s3) String() string { return "SSE-S3" }
+
+// String returns the SSE domain as string. For SSE-C the
+// domain is "SSE-C".
+func (ssec) String() string { return "SSE-C" }
+
+func (s3KMS) String() string { return "SSE-KMS" }
+
+
+
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+ "bytes"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "strings"
+ "time"
+
+ vault "github.com/hashicorp/vault/api"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ DEBUG_ROOT_TOKEN = "myroot"
+ DEBUG_LEASE_DURATION = 60 * 60 * 24 * 30 // 30 days
+)
+
+var (
+ //ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
+ ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
+)
+
+type vaultService struct {
+ config *VaultConfig
+ client *vault.Client
+ leaseDuration time.Duration
+}
+
+// return transit secret engine's path for generate data key operation
+func (v *vaultService) genDataKeyEndpoint(key string) string {
+ return "/transit/datakey/plaintext/" + key
+}
+
+// return transit secret engine's path for decrypt operation
+func (v *vaultService) decryptEndpoint(key string) string {
+ return "/transit/decrypt/" + key
+}
+
+// VaultKey represents vault encryption key-id name & version
+type VaultKey struct {
+ Name string `json:"name"`
+ Version int `json:"version"`
+}
+
+// VaultAuth represents vault auth type to use. For now, AppRole is the only supported
+// auth type.
+type VaultAuth struct {
+ Type string `json:"type"`
+ AppRole VaultAppRole `json:"approle"`
+}
+
+// VaultAppRole represents vault approle credentials
+type VaultAppRole struct {
+ ID string `json:"id"`
+ Secret string `json:"secret"`
+}
+
+// VaultConfig holds config required to start vault service
+type VaultConfig struct {
+ Endpoint string `json:"endpoint"`
+ Auth VaultAuth `json:"auth"`
+ Key VaultKey `json:"key-id"`
+}
+
+// validate whether all required env variables needed to start vault service have
+// been set
+func validateVaultConfig(c *VaultConfig) error {
+ if c.Endpoint == "" {
+ return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", "c.Endpoint")
+ }
+ if strings.ToLower(c.Auth.Type) != "approle" {
+ return fmt.Errorf("Unsupported hashicorp vault auth type - %s", "c.Auth.Type")
+ }
+ if c.Auth.AppRole.ID == "" {
+ return fmt.Errorf("Missing hashicorp vault AppRole ID - %s is empty", "c.Auth.AppRole.ID")
+ }
+ if c.Auth.AppRole.Secret == "" {
+ return fmt.Errorf("Missing hashicorp vault AppSecret ID - %s is empty", "c.Auth.AppRole.Secret")
+ }
+ if c.Key.Name == "" {
+ return fmt.Errorf("Invalid value set in environment variable %s", "c.Key.Name")
+ }
+ if c.Key.Version < 0 {
+ return fmt.Errorf("Invalid value set in environment variable %s", "c.Key.Version")
+ }
+ return nil
+}
+
+// authenticate to vault with app role id and app role secret, and get a client access token, lease duration
+func getVaultAccessToken(client *vault.Client, appRoleID, appSecret string) (token string, duration int, err error) {
+ data := map[string]interface{}{
+ "role_id": appRoleID,
+ "secret_id": appSecret,
+ }
+ resp, e := client.Logical().Write("auth/approle/login", data)
+ if e != nil {
+ return token, duration, e
+ }
+ if resp.Auth == nil {
+ return token, duration, ErrKMSAuthLogin
+ }
+ return resp.Auth.ClientToken, resp.Auth.LeaseDuration, nil
+}
+
+// NewVaultConfig sets KMSConfig from environment
+// variables and performs validations.
+func NewVaultConfig() (KMSConfig, error) {
+ kc := KMSConfig{}
+ config := VaultConfig{
+ Endpoint: helper.CONFIG.KMS.Endpoint,
+ Auth: VaultAuth{
+ Type: "approle",
+ AppRole: VaultAppRole{
+ ID: helper.CONFIG.KMS.Id,
+ Secret: helper.CONFIG.KMS.Secret,
+ },
+ },
+ Key: VaultKey{
+ Version: helper.CONFIG.KMS.Version,
+ Name: helper.CONFIG.KMS.Keyname,
+ },
+ }
+
+ // return if none of the vault env variables are configured
+ if (config.Endpoint == "") && (config.Auth.AppRole.ID == "") && (config.Auth.AppRole.Secret == "") &&
+ (config.Key.Name == "") && (config.Key.Version == 0) {
+ return kc, nil
+ }
+
+ if err := validateVaultConfig(&config); err != nil {
+ return kc, err
+ }
+ kc.Vault = config
+ return kc, nil
+}
+
+// NewVault initializes Hashicorp Vault KMS by
+// authenticating to Vault with the credentials in KMSConfig,
+// and gets a client token for future api calls.
+func NewVault(kmsConf KMSConfig) (KMS, error) {
+ config := kmsConf.Vault
+ vconfig := &vault.Config{
+ Address: config.Endpoint,
+ }
+
+ c, err := vault.NewClient(vconfig)
+ if err != nil {
+ return nil, err
+ }
+
+ var accessToken string
+ var leaseDuration int
+ if helper.CONFIG.DebugMode == true {
+ accessToken = DEBUG_ROOT_TOKEN
+ leaseDuration = DEBUG_LEASE_DURATION
+ } else {
+ accessToken, leaseDuration, err = getVaultAccessToken(c, config.Auth.AppRole.ID, config.Auth.AppRole.Secret)
+ log.Info("get access token:", accessToken, "lease duration:", leaseDuration)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // authenticate and get the access token
+ log.Info("get vault token:", accessToken, "lease duration:", leaseDuration)
+ c.SetToken(accessToken)
+ v := vaultService{client: c, config: &config, leaseDuration: time.Duration(leaseDuration)}
+ v.renewToken(c)
+ return &v, nil
+}
+
+func (v *vaultService) renewToken(c *vault.Client) {
+ retryDelay := 1 * time.Minute
+ go func() {
+ for {
+ s, err := c.Auth().Token().RenewSelf(int(v.leaseDuration))
+ if err != nil {
+ time.Sleep(retryDelay)
+ continue
+ }
+ nextRenew := s.Auth.LeaseDuration / 2
+ time.Sleep(time.Duration(nextRenew) * time.Second)
+ }
+ }()
+}
+
+// Generates a random plain text key, sealed plain text key from
+// Vault. It returns the plaintext key and sealed plaintext key on success
+func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
+ contextStream := new(bytes.Buffer)
+ ctx.WriteTo(contextStream)
+
+ payload := map[string]interface{}{
+ "context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+ }
+ s, err := v.client.Logical().Write(v.genDataKeyEndpoint(keyID), payload)
+
+ if err != nil {
+ return key, sealedKey, err
+ }
+ sealKey := s.Data["ciphertext"].(string)
+ plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
+ if err != nil {
+ return key, sealedKey, err
+ }
+ copy(key[:], []byte(plainKey))
+ return key, []byte(sealKey), nil
+}
+
+// unsealKMSKey unseals the sealedKey using the Vault master key
+// referenced by the keyID. The plain text key is returned on success.
+func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
+ contextStream := new(bytes.Buffer)
+ ctx.WriteTo(contextStream)
+ payload := map[string]interface{}{
+ "ciphertext": string(sealedKey),
+ "context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+ }
+ s, err := v.client.Logical().Write(v.decryptEndpoint(keyID), payload)
+ if err != nil {
+ return key, err
+ }
+ base64Key := s.Data["plaintext"].(string)
+ plainKey, err1 := base64.StdEncoding.DecodeString(base64Key)
+ if err1 != nil {
+ return key, err
+ }
+ copy(key[:], []byte(plainKey))
+
+ return key, nil
+}
+
+func (v *vaultService) GetKeyID() string {
+ return v.config.Key.Name
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package yig
+
+import (
+ "errors"
+ "fmt"
+ "math/rand"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/opensds/multi-cloud/backend/pkg/utils/constants"
+ backendpb "github.com/opensds/multi-cloud/backend/proto"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/driver"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/config"
+ _ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/db"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/storage"
+ log "github.com/sirupsen/logrus"
+)
+
+type YigDriverFactory struct {
+ Drivers sync.Map
+ cfgWatcher *config.ConfigWatcher
+ initLock sync.Mutex
+ initFlag int32
+}
+
+func (ydf *YigDriverFactory) CreateDriver(backend *backendpb.BackendDetail) (driver.StorageDriver, error) {
+ err := ydf.Init()
+ if err != nil {
+ log.Errorf("failed to perform YigDriverFactory init, err: %v", err)
+ return nil, err
+ }
+ // if driver already exists, just return it.
+ if driver, ok := ydf.Drivers.Load(backend.Endpoint); ok {
+ return driver.(*storage.YigStorage), nil
+ }
+
+ log.Infof("no storage driver for yig endpoint %s", backend.Endpoint)
+ return nil, errors.New(fmt.Sprintf("no storage driver for yig endpoint: %s", backend.Endpoint))
+}
+
+func (ydf *YigDriverFactory) Init() error {
+ // check
+ if atomic.LoadInt32(&ydf.initFlag) == 1 {
+ return nil
+ }
+
+ // lock
+ ydf.initLock.Lock()
+ defer ydf.initLock.Unlock()
+
+ // check
+ if ydf.initFlag == 1 {
+ return nil
+ }
+
+ // create the driver.
+ rand.Seed(time.Now().UnixNano())
+
+ // read the config.
+ err := config.ReadConfigs("/etc/yig", ydf.driverInit)
+ if err != nil {
+ log.Errorf("failed to read yig configs, err: %v", err)
+ return nil
+ }
+
+ // init config watcher.
+ watcher, err := config.NewConfigWatcher(ydf.driverInit)
+ if err != nil {
+ log.Errorf("failed to new config watcher, err: %v", err)
+ return err
+ }
+ ydf.cfgWatcher = watcher
+ ydf.cfgWatcher.Watch("/etc/yig")
+
+ atomic.StoreInt32(&ydf.initFlag, 1)
+ return nil
+}
+
+func (ydf *YigDriverFactory) Close() {
+ var keys []interface{}
+ // stop config watcher
+ if ydf.cfgWatcher != nil {
+ ydf.cfgWatcher.Stop()
+ }
+ // close the drivers
+ ydf.Drivers.Range(func(k, v interface{}) bool {
+ drv := v.(*storage.YigStorage)
+ drv.Close()
+ keys = append(keys, k)
+ return true
+ })
+
+ // remove the drivers
+ for _, k := range keys {
+ ydf.Drivers.Delete(k)
+ }
+}
+
+func (ydf *YigDriverFactory) driverInit(cfg *config.Config) error {
+ yigStorage, err := storage.New(cfg)
+ if err != nil {
+ log.Errorf("failed to create driver for %s, err: %v", cfg.Endpoint.Url, err)
+ return err
+ }
+
+ ydf.Drivers.Store(cfg.Endpoint.Url, yigStorage)
+
+ return nil
+}
+
+func init() {
+ yigDf := &YigDriverFactory{}
+ err := yigDf.Init()
+ if err != nil {
+ return
+ }
+ driver.AddCloser(yigDf)
+ driver.RegisterDriverFactory(constants.BackendTypeYIGS3, yigDf)
+}
+
+
+
package log
+
+import (
+ "fmt"
+ "io"
+ "log"
+ "os"
+)
+
+// These flags define which text to prefix to each log entry generated by the Logger.
+const (
+ // Bits or'ed together to control what's printed.
+ // There is no control over the order they appear (the order listed
+ // here) or the format they present (as described in the comments).
+ // The prefix is followed by a colon only when Llongfile or Lshortfile
+ // is specified.
+ // For example, flags Ldate | Ltime (or LstdFlags) produce,
+ // 2009/01/23 01:23:23 message
+ // while flags Ldate | Ltime | Lmicroseconds | Llongfile produce,
+ // 2009/01/23 01:23:23.123123 /a/b/c/d.go:23: message
+ Ldate = 1 << iota // the date in the local time zone: 2009/01/23
+ Ltime // the time in the local time zone: 01:23:23
+ Lmicroseconds // microsecond resolution: 01:23:23.123123. assumes Ltime.
+ Llongfile // full file name and line number: /a/b/c/d.go:23
+ Lshortfile // final file name element and line number: d.go:23. overrides Llongfile
+ LUTC // if Ldate or Ltime is set, use UTC rather than the local time zone
+ LstdFlags = Ldate | Ltime // initial values for the standard logger
+)
+
+type Logger struct {
+ Logger *log.Logger
+ LogLevel int
+}
+
+func New(out io.Writer, prefix string, flag int, level int) *Logger {
+ var logger Logger
+ logger.LogLevel = level
+ logger.Logger = log.New(out, prefix, flag)
+ return &logger
+}
+
+// Printf calls l.Output to print to the logger.
+// Arguments are handled in the manner of fmt.Printf.
+func (l *Logger) Printf(level int, format string, v ...interface{}) {
+ if l.LogLevel >= level {
+ l.Logger.Output(2, fmt.Sprintf(format, v...))
+ }
+}
+
+// Print calls l.Output to print to the logger.
+// Arguments are handled in the manner of fmt.Print.
+func (l *Logger) Print(level int, v ...interface{}) {
+ if l.LogLevel >= level {
+ l.Logger.Output(2, fmt.Sprint(v...))
+ }
+}
+
+// Println calls l.Output to print to the logger.
+// Arguments are handled in the manner of fmt.Println.
+func (l *Logger) Println(level int, v ...interface{}) {
+ if l.LogLevel >= level {
+ l.Logger.Output(2, fmt.Sprintln(v...))
+ }
+}
+
+// Fatal is equivalent to l.Print() followed by a call to os.Exit(1).
+func (l *Logger) Fatal(level int, v ...interface{}) {
+ if l.LogLevel >= level {
+ l.Logger.Output(2, fmt.Sprint(v...))
+ }
+ os.Exit(1)
+}
+
+// Fatalf is equivalent to l.Printf() followed by a call to os.Exit(1).
+func (l *Logger) Fatalf(level int, format string, v ...interface{}) {
+ if l.LogLevel >= level {
+ l.Logger.Output(2, fmt.Sprintf(format, v...))
+ }
+ os.Exit(1)
+}
+
+// Fatalln is equivalent to l.Println() followed by a call to os.Exit(1).
+func (l *Logger) Fatalln(level int, v ...interface{}) {
+ if l.LogLevel >= level {
+ l.Logger.Output(2, fmt.Sprintln(v...))
+ }
+ os.Exit(1)
+}
+
+// Panic is equivalent to l.Print() followed by a call to panic().
+func (l *Logger) Panic(level int, v ...interface{}) {
+ s := fmt.Sprint(v...)
+ if l.LogLevel >= level {
+ l.Logger.Output(2, s)
+ }
+ panic(s)
+}
+
+// Panicf is equivalent to l.Printf() followed by a call to panic().
+func (l *Logger) Panicf(level int, format string, v ...interface{}) {
+ s := fmt.Sprintf(format, v...)
+ if l.LogLevel >= level {
+ l.Logger.Output(2, s)
+ }
+ panic(s)
+}
+
+// Panicln is equivalent to l.Println() followed by a call to panic().
+func (l *Logger) Panicln(level int, v ...interface{}) {
+ s := fmt.Sprintln(v...)
+ if l.LogLevel >= level {
+ l.Logger.Output(2, s)
+ }
+ panic(s)
+}
+
+
+
package meta
+
+import (
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/types"
+)
+
+func (m *Meta) GetCluster(fsid, poolName string) (types.Cluster, error) {
+ return m.db.GetCluster(fsid, poolName)
+}
+
+
+
package driver
+
+import (
+ "errors"
+ "fmt"
+ "sync"
+
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/config"
+)
+
+var (
+ driversLock = sync.RWMutex{}
+ drivers = make(map[string]DBDriver)
+)
+
+func Open(dbCfg config.DatabaseConfig) (DB, error) {
+ driversLock.RLock()
+ driver, ok := drivers[dbCfg.DbType]
+ driversLock.RUnlock()
+ if !ok {
+ return nil, errors.New(fmt.Sprintf("unknow db driver %s", dbCfg.DbType))
+ }
+ return driver.OpenDB(dbCfg)
+}
+
+func RegisterDBDriver(dbType string, dbDriver DBDriver) {
+ if dbDriver == nil {
+ return
+ }
+
+ driversLock.Lock()
+ defer driversLock.Unlock()
+ drivers[dbType] = dbDriver
+}
+
+
+
package tidb
+
+import (
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/types"
+)
+
+//cluster
+func (t *Tidb) GetCluster(fsid, pool string) (cluster types.Cluster, err error) {
+ sqltext := "select fsid,pool,weight from cluster where fsid=? and pool=?"
+ err = t.DB.QueryRow(sqltext, fsid, pool).Scan(
+ &cluster.Fsid,
+ &cluster.Pool,
+ &cluster.Weight,
+ )
+ return
+}
+
+
+
package tidb
+
+import (
+ "database/sql"
+)
+
+type Tidb struct {
+ DB *sql.DB
+}
+
+func (t *Tidb) Close() {
+ t.DB.Close()
+}
+
+
+
package tidb
+
+import (
+ "database/sql"
+
+ _ "github.com/go-sql-driver/mysql"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/config"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/db/driver"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ TIDB_DRIVER_ID = "tidb"
+)
+
+type TidbDriver struct {
+}
+
+func (td *TidbDriver) OpenDB(dbCfg config.DatabaseConfig) (driver.DB, error) {
+ db := &Tidb{}
+ var err error
+ db.DB, err = sql.Open("mysql", dbCfg.DbUrl)
+ if err != nil {
+ log.Errorf("failed to open db: %s, err: %v", dbCfg.DbUrl, err)
+ return nil, err
+ }
+ log.Info("connected to tidb ...")
+ db.DB.SetMaxIdleConns(dbCfg.MaxIdleConns)
+ db.DB.SetMaxOpenConns(dbCfg.MaxOpenConns)
+ return db, nil
+}
+
+func init() {
+ tidbDriver := &TidbDriver{}
+ driver.RegisterDBDriver(TIDB_DRIVER_ID, tidbDriver)
+}
+
+
+
package tidb
+
+import (
+ "database/sql"
+ "time"
+
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/types"
+ mtypes "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ log "github.com/sirupsen/logrus"
+)
+
+func (t *Tidb) PutPartsInGc(parts []*types.PartInfo) (err error) {
+ var tx *sql.Tx
+ var stmt *sql.Stmt
+ tx, err = t.DB.Begin()
+ if err != nil {
+ log.Errorf("PutPartsInGc(%v) failed, err: %v", parts, err)
+ return err
+ }
+
+ defer func() {
+ if err == nil {
+ if err = tx.Commit(); err != nil {
+ log.Errorf("PutPartsInGc(%v) failed, failed to commit, err: %v", parts, err)
+ }
+ return
+ }
+ if rErr := tx.Rollback(); rErr != nil {
+ log.Errorf("PutPartsInGc(%v) failed, failed to rollback, err: %v", parts, err)
+ }
+ }()
+
+ // put all the parts into gc.
+ stmt, err = tx.Prepare("insert into gc(location, pool, object_id) values(?, ?, ?)")
+ if err != nil {
+ log.Errorf("PutPartsInGc(%v) failed, failed to prepare insert to gc, err: %v", parts, err)
+ return err
+ }
+ for _, p := range parts {
+ _, err = stmt.Exec(p.Location, p.Pool, p.ObjectId)
+ if err != nil {
+ log.Errorf("PutPartsInGc(%v) failed, failed to exec insert gc stmt(%v), err: %v", parts, p, err)
+ stmt.Close()
+ return err
+ }
+ }
+ stmt.Close()
+ // remove all the parts from multiparts
+ stmt, err = tx.Prepare("delete from multiparts where upload_id=? and part_num=?")
+ if err != nil {
+ log.Errorf("PutPartsInGc(%v) failed, failed to prepare to remove multiparts, err: %v", parts, err)
+ return err
+ }
+ for _, p := range parts {
+ _, err = stmt.Exec(p.UploadId, p.PartNum)
+ if err != nil {
+ log.Errorf("PutPartsInGc(%v) failed, failed to exec remove multiparts stmt(%v), err: %v", parts, p, err)
+ stmt.Close()
+ return err
+ }
+ }
+ stmt.Close()
+ return nil
+}
+
+// delete objects
+func (t *Tidb) PutGcObjects(objects ...*types.GcObject) (err error) {
+ var tx *sql.Tx
+ var stmt *sql.Stmt
+ tx, err = t.DB.Begin()
+ if err != nil {
+ log.Errorf("PutGcObjects(%v) failed, err: %v", objects, err)
+ return err
+ }
+
+ defer func() {
+ if err == nil {
+ if err = tx.Commit(); err != nil {
+ log.Errorf("PutGcObjects(%v) failed, failed to commit, err: %v", objects, err)
+ }
+ return
+ }
+ if rErr := tx.Rollback(); rErr != nil {
+ log.Errorf("PutGcObjects(%v) failed, failed to rollback, err: %v", objects, err)
+ }
+ }()
+
+ stmt, err = tx.Prepare("insert into gc(location, pool, object_id) values(?, ?, ?)")
+ if err != nil {
+ log.Errorf("PutGcObjects(%v) failed, failed to prepare, err: %v", objects, err)
+ return err
+ }
+ defer stmt.Close()
+ for _, o := range objects {
+ _, err = stmt.Exec(o.Location, o.Pool, o.ObjectId)
+ if err != nil {
+ log.Errorf("PutGcObjects(%v) failed, failed to exec(%v), err: %v", objects, o, err)
+ return err
+ }
+ }
+ return nil
+}
+
+func (t *Tidb) GetGcObjects(marker int64, limit int) ([]*types.GcObject, error) {
+ sqlText := "select id, location, pool, object_id, create_time from gc where id>=? order by create_time limit ?"
+ var out []*types.GcObject
+ rows, err := t.DB.Query(sqlText, marker, limit)
+ if err != nil {
+ log.Errorf("failed to GetGcObjects(%d, %d), err: %v", marker, limit, err)
+ return nil, err
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var createTime sql.NullString
+ o := &types.GcObject{}
+ err = rows.Scan(&o.Id, &o.Location, &o.Pool, &o.ObjectId, &createTime)
+ if err != nil {
+ log.Errorf("GetGcObjects(%d, %d) failed, failed to perform scan, err: %v", marker, limit, err)
+ return nil, err
+ }
+ if createTime.Valid {
+ o.CreateTime, err = time.Parse(mtypes.TIME_LAYOUT_TIDB, createTime.String)
+ if err != nil {
+ log.Errorf("GetGcObjects(%d, %d) failed, failed to parse create_time: %s, err: %v", marker, limit, createTime.String, err)
+ return nil, err
+ }
+ }
+ out = append(out, o)
+ }
+ if err = rows.Err(); err != nil {
+ log.Errorf("GetGcObjects(%d, %d) failed, rows return error: %v", marker, limit, err)
+ return nil, err
+ }
+
+ return out, nil
+}
+
+// delete gc objects meta.
+func (t *Tidb) DeleteGcObjects(objects ...*types.GcObject) (err error) {
+ var tx *sql.Tx
+ var stmt *sql.Stmt
+ tx, err = t.DB.Begin()
+ if err != nil {
+ log.Errorf("DeleteGcObjects(%v) failed, err: %v", objects, err)
+ return err
+ }
+
+ defer func() {
+ if err == nil {
+ if err = tx.Commit(); err != nil {
+ log.Errorf("DeleteGcObjects(%v) failed, failed to commit, err: %v", objects, err)
+ }
+ return
+ }
+ if rErr := tx.Rollback(); rErr != nil {
+ log.Errorf("DeleteGcObjects(%v) failed, failed to rollback, err: %v", objects, err)
+ }
+ }()
+
+ stmt, err = tx.Prepare("delete from gc where object_id=?")
+ if err != nil {
+ log.Errorf("DeleteGcObjects(%v) failed, failed to prepare, err: %v", objects, err)
+ return err
+ }
+ defer stmt.Close()
+ for _, o := range objects {
+ _, err = stmt.Exec(o.ObjectId)
+ if err != nil {
+ log.Errorf("DeleteGcObjects(%v) failed, failed to exec(%v), err: %v", objects, o, err)
+ return err
+ }
+ }
+ return nil
+}
+
+
+
package tidb
+
+import (
+ "database/sql"
+ "sort"
+ "time"
+
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/types"
+ mtypes "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ log "github.com/sirupsen/logrus"
+)
+
+/*
+* ListParts: list all the parts which belong to the uploadId.
+* @return: the related parts which are sorted by PartNum asc.
+*
+ */
+func (t *Tidb) ListParts(uploadId uint64) ([]*types.PartInfo, error) {
+ sqlText := "select upload_id, part_num, object_id, location, pool, offset, size, etag, flag, create_time, update_time from multiparts where upload_id = ? order by part_num"
+ var parts []*types.PartInfo
+ rows, err := t.DB.Query(sqlText, uploadId)
+ if err != nil {
+ log.Errorf("failed to query parts for uploadId %d, err: %v", uploadId, err)
+ return nil, err
+ }
+ defer rows.Close()
+ for rows.Next() {
+ part := &types.PartInfo{}
+ var createTime sql.NullString
+ var updateTime sql.NullString
+ err = rows.Scan(&part.UploadId, &part.PartNum, &part.ObjectId, &part.Location, &part.Pool, &part.Offset, &part.Size, &part.Etag, &part.Flag, &createTime, &updateTime)
+ if err != nil {
+ log.Errorf("failed to scan rows for uploadId %d, err: %v", uploadId, err)
+ return nil, err
+ }
+ if createTime.Valid {
+ part.CreateTime, err = time.Parse(mtypes.TIME_LAYOUT_TIDB, createTime.String)
+ if err != nil {
+ log.Errorf("failed to parse create_time: %s, err: %v", createTime.String, err)
+ return nil, err
+ }
+ }
+ if updateTime.Valid {
+ part.UpdateTime, err = time.Parse(mtypes.TIME_LAYOUT_TIDB, updateTime.String)
+ if err != nil {
+ log.Errorf("failed to parse update_time: %s, err: %v", updateTime.String, err)
+ return nil, err
+ }
+ }
+ parts = append(parts, part)
+ }
+ err = rows.Err()
+ if err != nil {
+ log.Errorf("failed to iterate the rows for uploadId %d, err: %v", uploadId, err)
+ return nil, err
+ }
+ sort.Sort(types.ByPartNum(parts))
+ return parts, nil
+}
+
+func (t *Tidb) PutPart(partInfo *types.PartInfo) (err error) {
+ sqlText := "insert into multiparts(upload_id, part_num, object_id, location, pool, offset, size, etag, flag) values(?, ?, ?, ?, ?, ?, ?, ?, ?)"
+ var tx *sql.Tx
+ tx, err = t.DB.Begin()
+ if err != nil {
+ log.Errorf("failed to Begin a transaction for %v, err: %v", partInfo, err)
+ return err
+ }
+
+ defer func() {
+ if err != nil {
+ // do not use the err since since by doing so, it will overwite the original error.
+ if rErr := tx.Rollback(); rErr != nil {
+ log.Errorf("perform rollback for partInfo(%v) failed with err: %v", partInfo, rErr)
+ }
+ } else {
+ // should check whether the transaction commit succeeds or not.
+ if err = tx.Commit(); err != nil {
+ log.Errorf("perform commit for partInfo(%v) failed with err: %v", partInfo, err)
+ }
+ }
+ }()
+
+ _, err = tx.Exec(sqlText, partInfo.UploadId, partInfo.PartNum, partInfo.ObjectId, partInfo.Location, partInfo.Pool, partInfo.Offset, partInfo.Size, partInfo.Etag, partInfo.Flag)
+ if err != nil {
+ log.Errorf("failed to save partInfo(%v), err: %v", partInfo, err)
+ return err
+ }
+ // must return err instead of nil, because commit may return error.
+ return err
+}
+
+func (t *Tidb) DeleteParts(uploadId uint64) error {
+ sqlText := "delete from multiparts where upload_id=?"
+ _, err := t.DB.Exec(sqlText, uploadId)
+ if err != nil {
+ log.Errorf("failed to remove parts from meta for uploadId(%d), err: %v", uploadId, err)
+ return err
+ }
+ return nil
+}
+
+func (t *Tidb) CompleteParts(uploadId uint64, parts []*types.PartInfo) (err error) {
+ sqlText := "update multiparts set offset=?, flag=? where upload_id=? and part_num=?"
+ var tx *sql.Tx
+ var stmt *sql.Stmt
+
+ tx, err = t.DB.Begin()
+ if err != nil {
+ log.Errorf("failed to complete parts for uploadId(%d), it was fail to create transaction, err: %v", uploadId, err)
+ return err
+ }
+
+ defer func() {
+ if err == nil {
+ if err = tx.Commit(); err != nil {
+ log.Errorf("failed to commit tranaction when completing uploadId(%d), err: %v", uploadId, err)
+ }
+ } else {
+ if rErr := tx.Rollback(); rErr != nil {
+ log.Errorf("failed to rollback when completing uploadId(%d), err: %v", uploadId, rErr)
+ }
+ }
+ }()
+
+ stmt, err = tx.Prepare(sqlText)
+ if err != nil {
+ log.Errorf("failed to complete uploadId(%d), it was fail to prepare sql, err: %v", uploadId, err)
+ return err
+ }
+
+ defer stmt.Close()
+
+ for _, part := range parts {
+ _, err := stmt.Exec(part.Offset, part.Flag, uploadId, part.PartNum)
+ if err != nil {
+ log.Errorf("failed to complete uploadId(%d), it was fail to perform stmt exec, err: %v", uploadId, err)
+ return err
+ }
+ }
+
+ return err
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package meta
+
+import (
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/types"
+)
+
+// delete multipart uploaded part objects and put them into gc
+func (m *Meta) PutPartsInGc(parts []*types.PartInfo) error {
+ return m.db.PutPartsInGc(parts)
+}
+
+func (m *Meta) PutGcObjects(objects ...*types.GcObject) error {
+ return m.db.PutGcObjects(objects...)
+}
+
+// get gc objects by marker and limit
+func (m *Meta) GetGcObjects(marker int64, limit int) ([]*types.GcObject, error) {
+ return m.db.GetGcObjects(marker, limit)
+}
+
+// delete gc objects meta.
+func (m *Meta) DeleteGcObjects(objects ...*types.GcObject) error {
+ return m.db.DeleteGcObjects(objects...)
+}
+
+
+
package meta
+
+import (
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/config"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/db/driver"
+)
+
+type MetaConfig struct {
+ Dbcfg config.DatabaseConfig
+}
+
+type Meta struct {
+ db driver.DB
+}
+
+func (m *Meta) Close() {
+ m.db.Close()
+}
+
+func New(cfg MetaConfig) (*Meta, error) {
+ db, err := driver.Open(cfg.Dbcfg)
+ if err != nil {
+ return nil, err
+ }
+
+ m := &Meta{
+ db: db,
+ }
+ return m, nil
+}
+
+
+
package meta
+
+import (
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/types"
+)
+
+func (m *Meta) ListParts(uploadId uint64) ([]*types.PartInfo, error) {
+ return m.db.ListParts(uploadId)
+}
+
+func (m *Meta) PutPart(partInfo *types.PartInfo) error {
+ return m.db.PutPart(partInfo)
+}
+
+func (m *Meta) DeleteParts(uploadId uint64) error {
+ return m.db.DeleteParts(uploadId)
+}
+
+func (m *Meta) CompleteParts(uploadId uint64, parts []*types.PartInfo) error {
+ return m.db.CompleteParts(uploadId, parts)
+}
+
+
+
package types
+
+import (
+ "time"
+)
+
+const (
+ MULTIPART_UPLOAD_IN_PROCESS = 0
+ MULTIPART_UPLOAD_COMPLETE = 1
+)
+
+type PartInfo struct {
+ // global unique upload id
+ // Note: this upload id is increasing monoatomicly only in one node.
+ UploadId uint64
+ // part number in this upload
+ PartNum int64
+ // object id in ceph which relates to the part.
+ ObjectId string
+ // ceph cluster
+ Location string
+ // ceph pool name
+ Pool string
+ // offset of this part in the whole object
+ Offset uint64
+ // size of this part
+ Size uint64
+ // etag of this part
+ Etag string
+ // Flag determin whether this part is completed or not.
+ // 0 by default.
+ Flag uint8
+ // create time of this upload
+ CreateTime time.Time
+ // this record changed time.
+ UpdateTime time.Time
+}
+
+type ByPartNum []*PartInfo
+
+func (bpn ByPartNum) Len() int { return len(bpn) }
+
+func (bpn ByPartNum) Swap(i, j int) { bpn[i], bpn[j] = bpn[j], bpn[i] }
+
+func (bpn ByPartNum) Less(i, j int) bool { return bpn[i].PartNum <= bpn[j].PartNum }
+
+
+
package storage
+
+import (
+ "bytes"
+ "container/list"
+ "errors"
+ "io"
+ "sync"
+
+ "fmt"
+ "github.com/journeymidnight/radoshttpd/rados"
+ log "github.com/sirupsen/logrus"
+ "time"
+)
+
+const (
+ MON_TIMEOUT = "10"
+ OSD_TIMEOUT = "10"
+ STRIPE_UNIT = 512 << 10 /* 512K */
+ STRIPE_COUNT = 2
+ OBJECT_SIZE = 8 << 20 /* 8M */
+ BUFFER_SIZE = 1 << 20 /* 1M */
+ MIN_CHUNK_SIZE = 512 << 10 /* 512K */
+ MAX_CHUNK_SIZE = 8 * BUFFER_SIZE /* 8M */
+ SMALL_FILE_POOLNAME = "rabbit"
+ BIG_FILE_POOLNAME = "tiger"
+ BIG_FILE_THRESHOLD = 128 << 10 /* 128K */
+ AIO_CONCURRENT = 4
+)
+
+type CephStorage struct {
+ Name string
+ Conn *rados.Conn
+ InstanceId uint64
+ CountMutex *sync.Mutex
+ Counter uint64
+ BufPool *sync.Pool
+ BigBufPool *sync.Pool
+}
+
+func NewCephStorage(configFile string) *CephStorage {
+ log.Infof("Loading Ceph file %s\n", configFile)
+
+ Rados, err := rados.NewConn("admin")
+ Rados.SetConfigOption("rados_mon_op_timeout", MON_TIMEOUT)
+ Rados.SetConfigOption("rados_osd_op_timeout", OSD_TIMEOUT)
+
+ err = Rados.ReadConfigFile(configFile)
+ if err != nil {
+ log.Errorf("Failed to open ceph.conf: %s\n", configFile)
+ return nil
+ }
+
+ err = Rados.Connect()
+ if err != nil {
+ log.Errorf("Failed to connect to remote cluster: %s\n", configFile)
+ return nil
+ }
+
+ name, err := Rados.GetFSID()
+ if err != nil {
+ log.Errorf("Failed to get FSID: %s\n", configFile)
+ Rados.Shutdown()
+ return nil
+ }
+
+ id := Rados.GetInstanceID()
+
+ cluster := CephStorage{
+ Conn: Rados,
+ Name: name,
+ InstanceId: id,
+ CountMutex: new(sync.Mutex),
+ BufPool: &sync.Pool{
+ New: func() interface{} {
+ return bytes.NewBuffer(make([]byte, BIG_FILE_THRESHOLD))
+ },
+ },
+ BigBufPool: &sync.Pool{
+ New: func() interface{} {
+ return make([]byte, MAX_CHUNK_SIZE)
+ },
+ },
+ }
+
+ log.Infof("Ceph Cluster %s is ready, InstanceId is %d\n", name, id)
+ return &cluster
+}
+
+func setStripeLayout(p *rados.StriperPool) int {
+ var ret int = 0
+ if ret = p.SetLayoutStripeUnit(STRIPE_UNIT); ret < 0 {
+ return ret
+ }
+ if ret = p.SetLayoutObjectSize(OBJECT_SIZE); ret < 0 {
+ return ret
+ }
+ if ret = p.SetLayoutStripeCount(STRIPE_COUNT); ret < 0 {
+ return ret
+ }
+ return ret
+}
+
+func pending_has_completed(p *list.List) bool {
+ if p.Len() == 0 {
+ return false
+ }
+ e := p.Front()
+ c := e.Value.(*rados.AioCompletion)
+ ret := c.IsComplete()
+ if ret == 0 {
+ return false
+ } else {
+ return true
+ }
+}
+
+func wait_pending_front(p *list.List) int {
+ /* remove AioCompletion from list */
+ e := p.Front()
+ p.Remove(e)
+ c := e.Value.(*rados.AioCompletion)
+ c.WaitForComplete()
+ ret := c.GetReturnValue()
+ c.Release()
+ return ret
+}
+
+func drain_pending(p *list.List) int {
+ var ret int
+ for p.Len() > 0 {
+ ret = wait_pending_front(p)
+ }
+ return ret
+}
+
+func (cluster *CephStorage) GetUniqUploadName() string {
+ cluster.CountMutex.Lock()
+ defer cluster.CountMutex.Unlock()
+ cluster.Counter += 1
+ oid := fmt.Sprintf("%d:%d", cluster.InstanceId, cluster.Counter)
+ return oid
+}
+
+func (c *CephStorage) Shutdown() {
+ c.Conn.Shutdown()
+}
+
+func (cluster *CephStorage) doSmallPut(poolname string, oid string, data io.Reader) (size int64, err error) {
+ tstart := time.Now()
+ pool, err := cluster.Conn.OpenPool(poolname)
+ if err != nil {
+ return 0, errors.New("Bad poolname")
+ }
+ defer pool.Destroy()
+ tpool := time.Now()
+ dur := tpool.Sub(tstart).Nanoseconds() / 1000000
+ if dur >= 10 {
+ log.Warnf("slow log: doSmallPut OpenPool(%s, %s) spent %d", poolname, oid, dur)
+ }
+
+ buffer := cluster.BufPool.Get().(*bytes.Buffer)
+ buffer.Reset()
+ defer cluster.BufPool.Put(buffer)
+ written, err := buffer.ReadFrom(data)
+ if err != nil {
+ log.Errorf("failed to read data for pool %s, oid %s, err: %v", poolname, oid, err)
+ return 0, err
+ }
+
+ size = written
+
+ tread := time.Now()
+ dur = tread.Sub(tpool).Nanoseconds() / 1000000
+ if dur >= 10 {
+ log.Warnf("slow log: doSmallPut read body(%s, %s) spent %d", poolname, oid, dur)
+ }
+
+ err = pool.WriteSmallObject(oid, buffer.Bytes())
+ if err != nil {
+ return 0, err
+ }
+ twrite := time.Now()
+ dur = twrite.Sub(tread).Nanoseconds() / 1000000
+ if dur >= 50 {
+ log.Warnf("slow log: doSmallPut ceph write(%s, %s) spent %d", poolname, oid, dur)
+ }
+
+ dur = twrite.Sub(tstart).Nanoseconds() / 1000000
+ if dur >= 100 {
+ log.Warnf("slow log: doSmallPut fin(%s, %s) spent %d", poolname, oid, dur)
+ }
+
+ return size, nil
+}
+
+type RadosSmallDownloader struct {
+ oid string
+ offset int64
+ remaining int64
+ pool *rados.Pool
+}
+
+func (rd *RadosSmallDownloader) Read(p []byte) (n int, err error) {
+ if rd.remaining <= 0 {
+ return 0, io.EOF
+ }
+ if int64(len(p)) > rd.remaining {
+ p = p[:rd.remaining]
+ }
+ count, err := rd.pool.Read(rd.oid, p, uint64(rd.offset))
+ if count == 0 {
+ return 0, io.EOF
+ }
+ rd.offset += int64(count)
+ rd.remaining -= int64(count)
+ return count, err
+}
+
+func (rd *RadosSmallDownloader) Seek(offset int64, whence int) (int64, error) {
+ switch whence {
+ case 0:
+ rd.offset = offset
+ case 1:
+ rd.offset += offset
+ case 2:
+ panic("Not implemented")
+ }
+ return rd.offset, nil
+}
+
+func (rd *RadosSmallDownloader) Close() error {
+ rd.pool.Destroy()
+ return nil
+}
+
+func (cluster *CephStorage) Put(poolname string, oid string, data io.Reader) (size int64, err error) {
+ if poolname == SMALL_FILE_POOLNAME {
+ return cluster.doSmallPut(poolname, oid, data)
+ }
+
+ pool, err := cluster.Conn.OpenPool(poolname)
+ if err != nil {
+ return 0, fmt.Errorf("Bad poolname %s", poolname)
+ }
+ defer pool.Destroy()
+
+ striper, err := pool.CreateStriper()
+ if err != nil {
+ return 0, fmt.Errorf("Bad ioctx of pool %s", poolname)
+ }
+ defer striper.Destroy()
+
+ setStripeLayout(&striper)
+
+ /* if the data len in pending_data is bigger than current_upload_window, I will flush the data to ceph */
+ /* current_upload_window could not dynamically increase or shrink */
+
+ var c *rados.AioCompletion
+ pending := list.New()
+ var current_upload_window = MIN_CHUNK_SIZE /* initial window size as MIN_CHUNK_SIZE, max size is MAX_CHUNK_SIZE */
+ var pending_data = cluster.BigBufPool.Get().([]byte)
+ defer func() {
+ cluster.BigBufPool.Put(pending_data)
+ }()
+
+ var slice_offset = 0
+ var slow_count = 0
+ // slice is the buffer size of reader, the size is equal to remain size of pending_data
+ var slice = pending_data[0:current_upload_window]
+
+ var offset uint64 = 0
+
+ for {
+ start := time.Now()
+ count, err := data.Read(slice)
+ if err != nil && err != io.EOF {
+ drain_pending(pending)
+ return 0, fmt.Errorf("Read from client failed. pool:%s oid:%s", poolname, oid)
+ }
+ if count == 0 {
+ break
+ }
+ // it's used to calculate next upload window
+ elapsed_time := time.Since(start)
+
+ slice_offset += count
+ slice = pending_data[slice_offset:]
+
+ //is pending_data full?
+ if slice_offset < len(pending_data) {
+ continue
+ }
+
+ /* pending data is full now */
+ c = new(rados.AioCompletion)
+ c.Create()
+ _, err = striper.WriteAIO(c, oid, pending_data, offset)
+ if err != nil {
+ c.Release()
+ drain_pending(pending)
+ return 0, fmt.Errorf("Bad io. pool:%s oid:%s", poolname, oid)
+ }
+ pending.PushBack(c)
+
+ for pending_has_completed(pending) {
+ if ret := wait_pending_front(pending); ret < 0 {
+ drain_pending(pending)
+ return 0, fmt.Errorf("Error drain_pending in pending_has_completed(%d). pool:%s oid:%s", ret, poolname, oid)
+ }
+ }
+
+ if pending.Len() > AIO_CONCURRENT {
+ if ret := wait_pending_front(pending); ret < 0 {
+ drain_pending(pending)
+ return 0, fmt.Errorf("Error wait_pending_front(%d). pool:%s oid:%s", ret, poolname, oid)
+ }
+ }
+ offset += uint64(len(pending_data))
+
+ /* Resize current upload window */
+ expected_time := count * 1000 * 1000 * 1000 / current_upload_window /* 1000 * 1000 * 1000 means use Nanoseconds */
+
+ // If the upload speed is less than half of the current upload window, reduce the upload window by half.
+ // If upload speed is larger than current window size per second, used the larger window and twice
+ if elapsed_time.Nanoseconds() > 2*int64(expected_time) {
+ if slow_count > 2 && current_upload_window > MIN_CHUNK_SIZE {
+ current_upload_window = current_upload_window >> 1
+ slow_count = 0
+ }
+ slow_count += 1
+ } else if int64(expected_time) > elapsed_time.Nanoseconds() {
+ /* if upload speed is fast enough, enlarge the current_upload_window a bit */
+ current_upload_window = current_upload_window << 1
+ if current_upload_window > MAX_CHUNK_SIZE {
+ current_upload_window = MAX_CHUNK_SIZE
+ }
+ slow_count = 0
+ }
+ /* allocate a new pending data */
+ slice_offset = 0
+ slice = pending_data[0:current_upload_window]
+ }
+
+ size = int64(uint64(slice_offset) + offset)
+ //write all remaining data
+ if slice_offset > 0 {
+ c = new(rados.AioCompletion)
+ c.Create()
+ striper.WriteAIO(c, oid, pending_data[:slice_offset], offset)
+ pending.PushBack(c)
+ }
+
+ //drain_pending
+ if ret := drain_pending(pending); ret < 0 {
+ return 0, fmt.Errorf("Error wait_pending_front(%d). pool:%s oid:%s", ret, poolname, oid)
+ }
+ return size, nil
+}
+
+func (cluster *CephStorage) Append(poolname string, oid string, data io.Reader, offset uint64, isExist bool) (size int64, err error) {
+ if poolname != BIG_FILE_POOLNAME {
+ return 0, errors.New("specified pool must be used for storing big file.")
+ }
+
+ pool, err := cluster.Conn.OpenPool(poolname)
+ if err != nil {
+ return 0, fmt.Errorf("Bad poolname %s", poolname)
+ }
+ defer pool.Destroy()
+
+ striper, err := pool.CreateStriper()
+ if err != nil {
+ return 0, fmt.Errorf("Bad ioctx of pool %s", poolname)
+ }
+ defer striper.Destroy()
+
+ setStripeLayout(&striper)
+
+ var current_upload_window = MIN_CHUNK_SIZE /* initial window size as MIN_CHUNK_SIZE, max size is MAX_CHUNK_SIZE */
+ var pending_data = make([]byte, current_upload_window)
+
+ var origin_offset = offset
+ var slice_offset = 0
+ var slow_count = 0
+ // slice is the buffer size of reader, the size is equal to remain size of pending_data
+ var slice = pending_data[0:current_upload_window]
+ for {
+ start := time.Now()
+ count, err := data.Read(slice)
+
+ if count == 0 {
+ break
+ }
+ // it's used to calculate next upload window
+ elapsed_time := time.Since(start)
+
+ slice_offset += count
+ slice = pending_data[slice_offset:]
+
+ //is pending_data full?
+ if slice_offset < len(pending_data) {
+ continue
+ }
+
+ /* pending data is full now */
+ _, err = striper.Write(oid, pending_data, offset)
+ if err != nil {
+ return 0, fmt.Errorf("Bad io. pool:%s oid:%s", poolname, oid)
+ }
+
+ offset += uint64(len(pending_data))
+
+ /* Resize current upload window */
+ expected_time := count * 1000 * 1000 * 1000 / current_upload_window /* 1000 * 1000 * 1000 means use Nanoseconds */
+
+ // If the upload speed is less than half of the current upload window, reduce the upload window by half.
+ // If upload speed is larger than current window size per second, used the larger window and twice
+ if elapsed_time.Nanoseconds() > 2*int64(expected_time) {
+ if slow_count > 2 && current_upload_window > MIN_CHUNK_SIZE {
+ current_upload_window = current_upload_window >> 1
+ slow_count = 0
+ }
+ slow_count += 1
+ } else if int64(expected_time) > elapsed_time.Nanoseconds() {
+ /* if upload speed is fast enough, enlarge the current_upload_window a bit */
+ current_upload_window = current_upload_window << 1
+ if current_upload_window > MAX_CHUNK_SIZE {
+ current_upload_window = MAX_CHUNK_SIZE
+ }
+ slow_count = 0
+ }
+ /* allocate a new pending data */
+ pending_data = make([]byte, current_upload_window)
+ slice_offset = 0
+ slice = pending_data[0:current_upload_window]
+ }
+
+ size = int64(uint64(slice_offset) + offset - origin_offset)
+ //write all remaining data
+ if slice_offset > 0 {
+ _, err = striper.Write(oid, pending_data, offset)
+ if err != nil {
+ return 0, fmt.Errorf("Bad io. pool:%s oid:%s", poolname, oid)
+ }
+ }
+
+ return size, nil
+}
+
+type RadosDownloader struct {
+ striper *rados.StriperPool
+ oid string
+ offset int64
+ remaining int64
+ pool *rados.Pool
+}
+
+func (rd *RadosDownloader) Read(p []byte) (n int, err error) {
+ if rd.remaining <= 0 {
+ return 0, io.EOF
+ }
+ if int64(len(p)) > rd.remaining {
+ p = p[:rd.remaining]
+ }
+ count, err := rd.striper.Read(rd.oid, p, uint64(rd.offset))
+ if count == 0 {
+ return 0, io.EOF
+ }
+ rd.offset += int64(count)
+ rd.remaining -= int64(count)
+ return count, err
+}
+
+func (rd *RadosDownloader) Seek(offset int64, whence int) (int64, error) {
+ switch whence {
+ case 0:
+ rd.offset = offset
+ case 1:
+ rd.offset += offset
+ case 2:
+ panic("Not implemented")
+ }
+ return rd.offset, nil
+}
+
+func (rd *RadosDownloader) Close() error {
+ rd.striper.Destroy()
+ rd.pool.Destroy()
+ return nil
+}
+
+func (cluster *CephStorage) getReader(poolName string, oid string, startOffset int64,
+ length int64) (reader io.ReadCloser, err error) {
+
+ if poolName == SMALL_FILE_POOLNAME {
+ pool, e := cluster.Conn.OpenPool(poolName)
+ if e != nil {
+ err = errors.New("bad poolname")
+ return
+ }
+ radosSmallReader := &RadosSmallDownloader{
+ oid: oid,
+ offset: startOffset,
+ pool: pool,
+ remaining: length,
+ }
+
+ return radosSmallReader, nil
+ }
+
+ pool, err := cluster.Conn.OpenPool(poolName)
+ if err != nil {
+ err = errors.New("bad poolname")
+ return
+ }
+
+ striper, err := pool.CreateStriper()
+ if err != nil {
+ err = errors.New("bad ioctx")
+ return
+ }
+
+ radosReader := &RadosDownloader{
+ striper: &striper,
+ oid: oid,
+ offset: startOffset,
+ pool: pool,
+ remaining: length,
+ }
+
+ return radosReader, nil
+}
+
+// Works together with `wrapAlignedEncryptionReader`, see comments there.
+func (cluster *CephStorage) getAlignedReader(poolName string, oid string, startOffset int64,
+ length int64) (reader io.ReadCloser, err error) {
+
+ alignedOffset := startOffset / AES_BLOCK_SIZE * AES_BLOCK_SIZE
+ length += startOffset - alignedOffset
+ return cluster.getReader(poolName, oid, alignedOffset, length)
+}
+
+func (cluster *CephStorage) doSmallRemove(poolname string, oid string) error {
+ pool, err := cluster.Conn.OpenPool(poolname)
+ if err != nil {
+ return errors.New("Bad poolname")
+ }
+ defer pool.Destroy()
+ return pool.Delete(oid)
+}
+
+func (cluster *CephStorage) Remove(poolname string, oid string) error {
+
+ if poolname == SMALL_FILE_POOLNAME {
+ return cluster.doSmallRemove(poolname, oid)
+ }
+
+ pool, err := cluster.Conn.OpenPool(poolname)
+ if err != nil {
+ return errors.New("Bad poolname")
+ }
+ defer pool.Destroy()
+
+ striper, err := pool.CreateStriper()
+ if err != nil {
+ return errors.New("Bad ioctx")
+ }
+ defer striper.Destroy()
+
+ return striper.Delete(oid)
+}
+
+func (cluster *CephStorage) GetUsedSpacePercent() (pct int, err error) {
+ stat, err := cluster.Conn.GetClusterStats()
+ if err != nil {
+ return 0, errors.New("Stat error")
+ }
+ pct = int(stat.Kb_used * uint64(100) / stat.Kb)
+ return
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package storage
+
+import (
+ "context"
+ "errors"
+ "runtime"
+ "sync"
+ "time"
+
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/types"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ GC_OBJECT_LIMIT_NUM = 10000
+ CEPH_OBJ_NON_EXIST_ERR = "rados: ret=-2"
+)
+
+type GcMgr struct {
+ // context to cancel the operations.
+ ctx context.Context
+ cancelFunc context.CancelFunc
+ yig *YigStorage
+ loopTime int64
+ wg sync.WaitGroup
+}
+
+func (gm *GcMgr) Start() {
+ // query the current available cpus
+ threadNum := runtime.GOMAXPROCS(0)
+ gm.wg.Add(1)
+ go func() {
+ for {
+ select {
+ case <-gm.ctx.Done():
+ log.Infof("GcMgr is stopping.")
+ gm.wg.Done()
+ return
+ case <-time.After(time.Second * time.Duration(gm.loopTime)):
+ }
+ var chs []<-chan *GcObjectResult
+ // get all the gc objects for this loop
+ gcChan := gm.QueryGcObjectStream()
+ // by default, we will start the go routines with the number of available cpus.
+ for i := 0; i < threadNum; i++ {
+ // remove the gc objects from ceph storage
+ ch := gm.CreateObjectDeleteStream(gcChan)
+ chs = append(chs, ch)
+ }
+ // clear the removed gc objects from gc table.
+ chResult := gm.CreateGcObjectRecordCleanStream(chs...)
+ // record the success or failure.
+ for result := range chResult {
+ if result.ErrNo == ErrNoErr {
+ log.Debugf("succeed to remove object: %s", result.ObjectId)
+ continue
+ }
+ log.Errorf("failed to remove object: %s, err: %s", result.ObjectId, result.Err)
+ }
+ }
+ }()
+}
+
+func (gm *GcMgr) Stop() {
+ log.Infof("try to stop GcMgr...")
+ gm.cancelFunc()
+ gm.wg.Wait()
+ log.Infof("GcMgr has stopped.")
+}
+
+func (gm *GcMgr) QueryGcObjectStream() <-chan *types.GcObject {
+ out := make(chan *types.GcObject)
+ go func() {
+ defer close(out)
+ start := int64(0)
+ for {
+ gcObjects, err := gm.yig.MetaStorage.GetGcObjects(start, GC_OBJECT_LIMIT_NUM)
+ if err != nil {
+ log.Errorf("failed to get gc objects(%d), err: %v", start, err)
+ return
+ }
+ if gcObjects == nil || len(gcObjects) == 0 {
+ log.Debugf("got empty gc objects(%d)", start)
+ return
+ }
+ // set the next marker to query the gc objects.
+ start = gcObjects[len(gcObjects)-1].Id + 1
+ for _, o := range gcObjects {
+ select {
+ case out <- o:
+ case <-gm.ctx.Done():
+ return
+ }
+ }
+ // check whether it is finished to read in this loop.
+ if len(gcObjects) < GC_OBJECT_LIMIT_NUM {
+ return
+ }
+ }
+ }()
+
+ return out
+}
+
+type GcObjectResult struct {
+ ErrNo S3ErrorCode
+ Err error
+ Id int64
+ ObjectId string
+}
+
+func (gm *GcMgr) CreateObjectDeleteStream(in <-chan *types.GcObject) <-chan *GcObjectResult {
+ out := make(chan *GcObjectResult)
+
+ go func() {
+ defer close(out)
+ for o := range in {
+ result := &GcObjectResult{
+ Id: o.Id,
+ ObjectId: o.ObjectId,
+ }
+ ceph, ok := gm.yig.DataStorage[o.Location]
+ if !ok {
+ log.Errorf("cannot find the ceph storage for gc object(%s, %s, %s)", o.Location, o.Pool, o.ObjectId)
+ result.ErrNo = ErrNoSuchKey
+ result.Err = errors.New("cannot find the ceph storage")
+ select {
+ case out <- result:
+ case <-gm.ctx.Done():
+ return
+ }
+ continue
+ }
+ err := ceph.Remove(o.Pool, o.ObjectId)
+ if err != nil && err.Error() != CEPH_OBJ_NON_EXIST_ERR {
+ log.Errorf("failed to remove object(%s, %s, %s) from ceph, err: %v", o.Location, o.Pool, o.ObjectId, err)
+ result.ErrNo = ErrInternalError
+ result.Err = err
+ select {
+ case out <- result:
+ case <-gm.ctx.Done():
+ return
+ }
+ // just continue to remove next object.
+ continue
+ }
+ result.Err = nil
+ result.ErrNo = ErrNoErr
+ select {
+ case out <- result:
+ case <-gm.ctx.Done():
+ return
+ }
+ }
+ }()
+
+ return out
+}
+
+func (gm *GcMgr) CreateGcObjectRecordCleanStream(in ...<-chan *GcObjectResult) <-chan *GcObjectResult {
+ wg := sync.WaitGroup{}
+ out := make(chan *GcObjectResult)
+ clearfunc := func(ch <-chan *GcObjectResult) {
+ defer wg.Done()
+ count := 0
+ var gcObjects []*types.GcObject
+ for result := range ch {
+ // check error of the result
+ if result.ErrNo != ErrNoErr {
+ select {
+ case out <- result:
+ continue
+ case <-gm.ctx.Done():
+ return
+ }
+ }
+ //batch clean the gc objects from gc table.
+ gcObj := &types.GcObject{
+ ObjectId: result.ObjectId,
+ }
+ gcObjects = append(gcObjects, gcObj)
+ count += 1
+ if count >= GC_OBJECT_LIMIT_NUM {
+ err := gm.yig.MetaStorage.DeleteGcObjects(gcObjects...)
+ if err != nil {
+ for _, o := range gcObjects {
+ clearResult := &GcObjectResult{
+ ErrNo: ErrInternalError,
+ Err: err,
+ ObjectId: o.ObjectId,
+ }
+ select {
+ case out <- clearResult:
+ case <-gm.ctx.Done():
+ return
+ }
+ }
+ } else {
+ for _, o := range gcObjects {
+ clearResult := &GcObjectResult{
+ ErrNo: ErrNoErr,
+ Err: nil,
+ ObjectId: o.ObjectId,
+ }
+ select {
+ case out <- clearResult:
+ case <-gm.ctx.Done():
+ return
+ }
+ }
+ }
+ // free the buffer slice and re-calculate again.
+ count = 0
+ gcObjects = nil
+ }
+ }
+ // clear the remaining gc objects.
+ if len(gcObjects) > 0 {
+ err := gm.yig.MetaStorage.DeleteGcObjects(gcObjects...)
+ if err != nil {
+ for _, o := range gcObjects {
+ clearResult := &GcObjectResult{
+ ErrNo: ErrInternalError,
+ Err: err,
+ ObjectId: o.ObjectId,
+ }
+ select {
+ case out <- clearResult:
+ case <-gm.ctx.Done():
+ return
+ }
+ }
+ } else {
+ for _, o := range gcObjects {
+ clearResult := &GcObjectResult{
+ ErrNo: ErrNoErr,
+ Err: nil,
+ ObjectId: o.ObjectId,
+ }
+ select {
+ case out <- clearResult:
+ case <-gm.ctx.Done():
+ return
+ }
+ }
+ }
+ }
+ }
+ for _, ch := range in {
+ wg.Add(1)
+ go clearfunc(ch)
+ }
+ go func() {
+ wg.Wait()
+ close(out)
+ }()
+ return out
+}
+
+func NewGcMgr(ctx context.Context, yig *YigStorage, loopTime int64) *GcMgr {
+ cancelCtx, cancelFunc := context.WithCancel(ctx)
+ return &GcMgr{
+ ctx: cancelCtx,
+ cancelFunc: cancelFunc,
+ yig: yig,
+ loopTime: loopTime,
+ wg: sync.WaitGroup{},
+ }
+}
+
+
+
package storage
+
+import (
+ "context"
+ "crypto/md5"
+ "encoding/hex"
+ "io"
+ "sort"
+ "strconv"
+
+ s3err "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/common"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/types"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ MAX_PART_SIZE = 5 << 30 // 5GB
+ MIN_PART_SIZE = 5 << 20 // 5MB
+ MIN_PART_NUMBER = 1
+ MAX_PART_NUMBER = 10000
+)
+
+/*
+* Below is the process of multipart upload:
+* 1. InitMultipartUpload will return a upload id and object id, and the caller should save them.
+* 2. caller should call UploadPart and input upload id and part number and then transfer the data.
+* backend will save the upload id and the part number and other related information.
+* 3. caller will call CompleteMultipartUpload and input the upload id, the backend
+* will mark all the parts identified by upload id as completed. If the caller calls
+* AbortMultipartUpload, backend will remove all the parts belongs to the upload id.
+*
+ */
+
+func (yig *YigStorage) InitMultipartUpload(ctx context.Context, object *pb.Object) (*pb.MultipartUpload, error) {
+ uploadId := uploadId2Str(yig.idGen.GetId())
+ mu := &pb.MultipartUpload{
+ Bucket: object.BucketName,
+ Key: object.ObjectKey,
+ UploadId: uploadId,
+ ObjectId: uploadId,
+ }
+ return mu, nil
+}
+
+/*
+* ctx: the context should contain below information: md5
+*
+ */
+
+func (yig *YigStorage) UploadPart(ctx context.Context, stream io.Reader, multipartUpload *pb.MultipartUpload,
+ partNumber int64, upBytes int64) (*model.UploadPartResult, error) {
+ // check the limiation https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/qfacts.html
+ if upBytes > MAX_PART_SIZE {
+ log.Errorf("UploadPart(%s, %s, %s, %d) failed, the size %d exceeds the maximum size.", multipartUpload.Bucket, multipartUpload.Key, multipartUpload.UploadId, partNumber, upBytes)
+ return nil, s3err.ErrEntityTooLarge
+ }
+ if partNumber < MIN_PART_NUMBER || partNumber > MAX_PART_NUMBER {
+ log.Errorf("UploadPart(%s, %s, %s, %d) failed, got invalid partNumber.", multipartUpload.Bucket, multipartUpload.Key, multipartUpload.UploadId, partNumber)
+ return nil, s3err.ErrInvalidPart
+ }
+ uploadId, err := str2UploadId(multipartUpload.UploadId)
+ if err != nil {
+ log.Errorf("UploadPart(%s, %s, %s, %d) failed, failed to convert uploadId to int64, err: %v", multipartUpload.Bucket, multipartUpload.Key, multipartUpload.UploadId, partNumber, err)
+ return nil, s3err.ErrNoSuchUpload
+ }
+ // TODO we need check whether the part number exists or not, if it already exists, we need to override it.
+ md5Writer := md5.New()
+ inputMd5 := ""
+ if val := ctx.Value(common.CONTEXT_KEY_MD5); val != nil {
+ inputMd5 = val.(string)
+ }
+
+ limitedDataReader := io.LimitReader(stream, upBytes)
+ cephCluster, poolName := yig.PickOneClusterAndPool(multipartUpload.Bucket, multipartUpload.Key, upBytes, false)
+ if cephCluster == nil {
+ log.Errorf("UploadPart(%s, %s, %s, %d) failed, cannot find the cluster", multipartUpload.Bucket, multipartUpload.Key, multipartUpload.UploadId, partNumber)
+ return nil, s3err.ErrInternalError
+ }
+ oid := cephCluster.GetUniqUploadName()
+ dataReader := io.TeeReader(limitedDataReader, md5Writer)
+ bytesWritten, err := cephCluster.Put(poolName, oid, dataReader)
+ if err != nil {
+ log.Errorf("failed to UploadPart(%s, %s, %s, %d), err: %v", multipartUpload.Bucket, multipartUpload.Key, multipartUpload.UploadId, partNumber, err)
+ return nil, err
+ }
+ // Should metadata update failed, put the object to gc,
+ // so the object in Ceph could be removed asynchronously
+ shouldGc := false
+ defer func() {
+ if shouldGc {
+ if err := yig.putObjToGc(cephCluster.Name, poolName, oid); err != nil {
+ log.Errorf("failed to put(%s, %s, %s) to gc, err: %v", cephCluster.Name, poolName, oid, err)
+ }
+ }
+ }()
+
+ if bytesWritten < upBytes {
+ shouldGc = true
+ log.Errorf("failed to UploadPart(%s, %s, %s, %d), written: %d, total: %d", multipartUpload.Bucket, multipartUpload.Key, multipartUpload.UploadId, partNumber, bytesWritten, upBytes)
+ return nil, s3err.ErrIncompleteBody
+ }
+ calculatedMd5 := hex.EncodeToString(md5Writer.Sum(nil))
+ if inputMd5 != "" && inputMd5 != calculatedMd5 {
+ shouldGc = true
+ log.Errorf("failed to UploadPart(%s, %s, %s, %d), input md5: %s, calculated: %s", multipartUpload.Bucket, multipartUpload.Key, multipartUpload.UploadId, partNumber, inputMd5, calculatedMd5)
+ return nil, s3err.ErrBadDigest
+ }
+ partInfo := &types.PartInfo{
+ UploadId: uploadId,
+ PartNum: partNumber,
+ ObjectId: oid,
+ Location: cephCluster.Name,
+ Pool: poolName,
+ Size: uint64(upBytes),
+ Etag: calculatedMd5,
+ Flag: types.MULTIPART_UPLOAD_IN_PROCESS,
+ }
+ err = yig.MetaStorage.PutPart(partInfo)
+ if err != nil {
+ shouldGc = true
+ log.Errorf("failed to put part for %s, %s, %s, %d, err: %v", multipartUpload.Bucket, multipartUpload.Key, multipartUpload.UploadId, partNumber, err)
+ return nil, err
+ }
+ result := &model.UploadPartResult{
+ PartNumber: partNumber,
+ ETag: calculatedMd5,
+ }
+ return result, nil
+}
+
+func (yig *YigStorage) CompleteMultipartUpload(ctx context.Context, multipartUpload *pb.MultipartUpload,
+ completeUpload *model.CompleteMultipartUpload) (*model.CompleteMultipartUploadResult, error) {
+ uploadId, err := str2UploadId(multipartUpload.UploadId)
+ if err != nil {
+ log.Errorf("failed to convert uploadId(%s) to int64, err: %v", multipartUpload.UploadId, err)
+ return nil, err
+ }
+ parts, err := yig.MetaStorage.ListParts(uploadId)
+ if err != nil {
+ log.Errorf("failed to list parts for uploadId(%d), err: %v", uploadId, err)
+ return nil, err
+ }
+
+ // check whether the given parts are already recorded in db.
+ if len(parts) != len(completeUpload.Parts) {
+ log.Errorf("input len(parts): %d, while we recorded len(parts): %d", len(completeUpload.Parts), len(parts))
+ return nil, s3err.ErrInvalidPartOrder
+ }
+
+ md5Writer := md5.New()
+ totalSize := uint64(0)
+ for k, part := range completeUpload.Parts {
+ // chech whether the part number starts at 1 and increases one by one.
+ if part.PartNumber != int64(k+1) {
+ log.Errorf("got invalid part[%d, %s] for uploadId %d with idx %d", part.PartNumber, part.ETag, uploadId, k)
+ return nil, s3err.ErrInvalidPart
+ }
+ // check whether the given parts are already recorded in db.
+ i := sort.Search(len(parts), func(i int) bool { return parts[i].PartNum >= part.PartNumber })
+ if i >= len(parts) || parts[i].PartNum != part.PartNumber {
+ log.Errorf("we cannot find the part[%d, %s] for uploadId %d", part.PartNumber, part.ETag, uploadId)
+ return nil, s3err.ErrInvalidPart
+ }
+ // check whether etag is matched.
+ if part.ETag != parts[i].Etag {
+ log.Errorf("got invalid part(%d, %s), the etag recorded is %s", part.PartNumber, part.ETag, parts[i].Etag)
+ return nil, s3err.ErrInvalidPart
+ }
+ // caclulate the md5 of etag for the whole object.
+ var etagBytes []byte
+ etagBytes, err = hex.DecodeString(part.ETag)
+ if err != nil {
+ log.Errorf("failed to decode etag of part(%d, %s) of uploadId(%d)", part.PartNumber, part.ETag, uploadId)
+ return nil, s3err.ErrInvalidPart
+ }
+ md5Writer.Write(etagBytes)
+ // check whether the size of each part except the last one is >= 5M and <= 5G.
+ if parts[i].Size < MIN_PART_SIZE && i < len(parts)-1 {
+ log.Errorf("got invalid size %d for part(%d, %s) of uploadId %d", parts[i].Size, parts[i].PartNum, parts[i].Etag, uploadId)
+ return nil, s3err.ErrInvalidPart
+ }
+ // complete an already completed uploadId, 404 will be returned.
+ if parts[i].Flag == types.MULTIPART_UPLOAD_COMPLETE {
+ log.Errorf("got completed part(%d) for uploadId %d", parts[i].PartNum, uploadId)
+ return nil, s3err.ErrNoSuchUpload
+ }
+
+ parts[i].Flag = types.MULTIPART_UPLOAD_COMPLETE
+ parts[i].Offset = totalSize
+ totalSize += parts[i].Size
+ }
+ // completes the parts.
+ err = yig.MetaStorage.CompleteParts(uploadId, parts)
+ if err != nil {
+ log.Errorf("failed to complete parts for uploadId(%d), err: %v", uploadId, err)
+ return nil, err
+ }
+ result := &model.CompleteMultipartUploadResult{
+ Bucket: multipartUpload.Bucket,
+ Key: multipartUpload.Key,
+ }
+ result.ETag = hex.EncodeToString(md5Writer.Sum(nil))
+ result.ETag += "-" + strconv.Itoa(len(parts))
+ result.Size = int64(totalSize)
+ return result, nil
+}
+
+func (yig *YigStorage) AbortMultipartUpload(ctx context.Context, multipartUpload *pb.MultipartUpload) error {
+ uploadId, err := str2UploadId(multipartUpload.UploadId)
+ if err != nil {
+ log.Errorf("failed to AbortMultipartUpload for %s, it was fail to parse uploadId, err: %v", multipartUpload.UploadId, err)
+ return err
+ }
+ parts, err := yig.MetaStorage.ListParts(uploadId)
+ if err != nil {
+ log.Errorf("failed to get parts for uploadId(%d), err: %v", uploadId, err)
+ return err
+ }
+
+ // remove the parts info from meta.
+ err = yig.MetaStorage.DeleteParts(uploadId)
+ if err != nil {
+ log.Errorf("failed to delete parts from meta for uploadId(%d), err: %v", uploadId, err)
+ return err
+ }
+
+ // remove all the parts from ceph cluster.
+ for _, p := range parts {
+ err = yig.putObjToGc(p.Location, p.Pool, p.ObjectId)
+ if err != nil {
+ log.Errorf("failed to put part(%s, %s, %s) to gc, err: %v", p.Location, p.Pool, p.ObjectId, err)
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (yig *YigStorage) ListParts(ctx context.Context, multipartUpload *pb.ListParts) (*model.ListPartsOutput, error) {
+ uploadId, err := str2UploadId(multipartUpload.UploadId)
+ if err != nil {
+ log.Errorf("failed to ListParts for %s, it failed to parse uploadId, err: %v", multipartUpload.UploadId, err)
+ return nil, err
+ }
+ parts, err := yig.MetaStorage.ListParts(uploadId)
+ if err != nil {
+ log.Errorf("ListParts failed, failed to get parts for uploadId(%d), err: %v", uploadId, err)
+ return nil, err
+ }
+
+ var partList []model.Part
+
+ for i, part := range parts {
+ if multipartUpload.PartNumberMarker > 0 && int64(i) <= multipartUpload.PartNumberMarker {
+ continue
+ }
+ if multipartUpload.MaxParts > 0 && int64(i) > multipartUpload.MaxParts {
+ break
+ }
+ p := model.Part{
+ PartNumber: part.PartNum,
+ ETag: part.Etag,
+ Size: int64(part.Size),
+ LastModifyTime: part.UpdateTime.Unix(),
+ }
+ partList = append(partList, p)
+ }
+
+ output := &model.ListPartsOutput{
+ Bucket: multipartUpload.Bucket,
+ Key: multipartUpload.Key,
+ UploadId: multipartUpload.UploadId,
+ MaxParts: int(multipartUpload.MaxParts),
+ IsTruncated: false,
+ Parts: partList,
+ }
+
+ return output, nil
+}
+
+func uploadId2Str(id int64) string {
+ return strconv.FormatUint(uint64(id), 16)
+}
+
+func str2UploadId(id string) (uint64, error) {
+ return strconv.ParseUint(id, 16, 64)
+}
+
+
+
package storage
+
+import (
+ "io"
+
+ s3err "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/types"
+ log "github.com/sirupsen/logrus"
+)
+
+type MultipartReader struct {
+ // uploadId for this multipart uploaded object.
+ uploadId uint64
+ // where to read from
+ start int64
+ // how much data to read
+ len int64
+ // parts for this uploadId, must be sorted by part number ascendly.
+ parts []*types.PartInfo
+ // YigStorage handle.
+ yig *YigStorage
+}
+
+func (mr *MultipartReader) Read(p []byte) (int, error) {
+ if len(mr.parts) == 0 || mr.len <= 0 {
+ return 0, io.EOF
+ }
+ if mr.parts[len(mr.parts)-1].Offset+mr.parts[len(mr.parts)-1].Size <= uint64(mr.start) {
+ return 0, io.EOF
+ }
+ // total length of input buffer
+ total := len(p)
+ if int64(total) > mr.len {
+ total = int(mr.len)
+ }
+ // where to start read from in the buffer.
+ // the data whose position < begin is already read.
+ begin := 0
+ for _, part := range mr.parts {
+ if part.Offset+part.Size < uint64(mr.start) {
+ continue
+ }
+ cephCluster, ok := mr.yig.DataStorage[part.Location]
+ if !ok {
+ log.Errorf("failed to get cephCluster for part(%d, %d, %s)", part.UploadId, part.PartNum, part.Location)
+ return 0, s3err.ErrInvalidPart
+ }
+ reader, err := cephCluster.getReader(part.Pool, part.ObjectId, mr.start-int64(part.Offset), int64(total-begin))
+ if err != nil {
+ log.Errorf("failed to get reader from ceph cluter for part(%d, %d, %s), err: %v", part.UploadId, part.PartNum, part.Location)
+ return 0, s3err.ErrInvalidPart
+ }
+ n, err := reader.Read(p[begin:])
+ // one read will try to read the whole data for the input buffer on each reader.
+ reader.Close()
+ if err != nil {
+ if err == io.EOF {
+ mr.start += int64(n)
+ begin += n
+ mr.len -= int64(n)
+ if begin < total {
+ continue
+ }
+ return 0, err
+ }
+ log.Infof("read data from uploadId(%d) with start(%d), len(%d) failed, err: %v", mr.uploadId, mr.start, mr.len, err)
+ return n, err
+ }
+ mr.start += int64(n)
+ begin += n
+ mr.len -= int64(n)
+ if begin >= total {
+ break
+ }
+ }
+
+ return begin, nil
+}
+
+func (mr *MultipartReader) Close() error {
+ return nil
+}
+
+func NewMultipartReader(yig *YigStorage, uploadIdStr string, start int64, end int64) (*MultipartReader, error) {
+ uploadId, err := str2UploadId(uploadIdStr)
+ if err != nil {
+ log.Errorf("failed to create MultipartReader, got invalid uploadId(%s), err: %v", uploadIdStr, err)
+ return nil, err
+ }
+ totalparts, err := yig.MetaStorage.ListParts(uploadId)
+ if err != nil {
+ log.Errorf("failed to create MultipartReader, failed to list parts for uploadId(%s), err: %v", uploadIdStr, err)
+ return nil, err
+ }
+ var parts []*types.PartInfo
+ for _, part := range totalparts {
+ if part.Offset+part.Size >= uint64(start) && part.Offset <= uint64(end) {
+ parts = append(parts, part)
+ }
+ }
+ return &MultipartReader{
+ uploadId: uploadId,
+ start: start,
+ len: end - start + 1,
+ parts: parts,
+ yig: yig,
+ }, nil
+}
+
+
+
package storage
+
+import (
+ "context"
+ "crypto/md5"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "io"
+ "math/rand"
+ "time"
+
+ . "github.com/opensds/multi-cloud/s3/error"
+ dscommon "github.com/opensds/multi-cloud/s3/pkg/datastore/common"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta/types"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+var latestQueryTime [2]time.Time // 0 is for SMALL_FILE_POOLNAME, 1 is for BIG_FILE_POOLNAME
+const CLUSTER_MAX_USED_SPACE_PERCENT = 85
+
+func (yig *YigStorage) PickOneClusterAndPool(bucket string, object string, size int64, isAppend bool) (cluster *CephStorage,
+ poolName string) {
+
+ var idx int
+ if isAppend {
+ poolName = BIG_FILE_POOLNAME
+ idx = 1
+ } else if size < 0 { // request.ContentLength is -1 if length is unknown
+ poolName = BIG_FILE_POOLNAME
+ idx = 1
+ } else if size < BIG_FILE_THRESHOLD {
+ poolName = SMALL_FILE_POOLNAME
+ idx = 0
+ } else {
+ poolName = BIG_FILE_POOLNAME
+ idx = 1
+ }
+ var needCheck bool
+ queryTime := latestQueryTime[idx]
+ if time.Since(queryTime).Hours() > 24 { // check used space every 24 hours
+ latestQueryTime[idx] = time.Now()
+ needCheck = true
+ }
+ var totalWeight int
+ clusterWeights := make(map[string]int, len(yig.DataStorage))
+ for fsid, _ := range yig.DataStorage {
+ cluster, err := yig.MetaStorage.GetCluster(fsid, poolName)
+ if err != nil {
+ log.Debug("Error getting cluster: ", err)
+ continue
+ }
+ if cluster.Weight == 0 {
+ continue
+ }
+ if needCheck {
+ pct, err := yig.DataStorage[fsid].GetUsedSpacePercent()
+ if err != nil {
+ log.Error("Error getting used space: ", err, "fsid: ", fsid)
+ continue
+ }
+ if pct > CLUSTER_MAX_USED_SPACE_PERCENT {
+ log.Error("Cluster used space exceed ", CLUSTER_MAX_USED_SPACE_PERCENT, fsid)
+ continue
+ }
+ }
+ totalWeight += cluster.Weight
+ clusterWeights[fsid] = cluster.Weight
+ }
+ if len(clusterWeights) == 0 || totalWeight == 0 {
+ log.Warn("Error picking cluster from table cluster in DB! Use first cluster in config to write.")
+ for _, c := range yig.DataStorage {
+ cluster = c
+ break
+ }
+ return
+ }
+ N := rand.Intn(totalWeight)
+ n := 0
+ for fsid, weight := range clusterWeights {
+ n += weight
+ if n > N {
+ cluster = yig.DataStorage[fsid]
+ break
+ }
+ }
+ return
+}
+
+func (yig *YigStorage) GetClusterByFsName(fsName string) (cluster *CephStorage, err error) {
+ if c, ok := yig.DataStorage[fsName]; ok {
+ cluster = c
+ } else {
+ err = errors.New("Cannot find specified ceph cluster: " + fsName)
+ }
+ return
+}
+
+// Write path:
+// +-----------+
+// PUT object/part | | Ceph
+// +---------+------------+----------+ Encryptor +----->
+// | | | |
+// | | +-----------+
+// v v
+// SHA256 MD5(ETag)
+//
+// SHA256 is calculated only for v4 signed authentication
+// Encryptor is enabled when user set SSE headers
+
+/* ctx should contain below elements:
+ * size: object size.
+ * encryptionKey:
+ * md5: the md5 put by user for the uploading object.
+ */
+func (yig *YigStorage) Put(ctx context.Context, stream io.Reader, obj *pb.Object) (result dscommon.PutResult,
+ err error) {
+ // get size from context.
+ val := ctx.Value(dscommon.CONTEXT_KEY_SIZE)
+ if val == nil {
+ return result, ErrIncompleteBody
+ }
+ size := val.(int64)
+ // md5 provided by user for uploading object.
+ userMd5 := ""
+ if val = ctx.Value(dscommon.CONTEXT_KEY_MD5); val != nil {
+ userMd5 = val.(string)
+ }
+
+ // check and remove the old object if exists.
+ if obj.StorageMeta != "" && obj.ObjectId != "" {
+ storageMeta, err := ParseObjectMeta(obj.StorageMeta)
+ if err != nil {
+ log.Errorf("Put(%s, %s, %s) failed, failed to parse storage meta(%s), err: %v", obj.BucketName,
+ obj.ObjectKey, obj.ObjectId, obj.StorageMeta, err)
+ return result, err
+ }
+ err = yig.putObjToGc(storageMeta.Cluster, storageMeta.Pool, obj.ObjectId)
+ if err != nil {
+ log.Errorf("Put(%s, %s, %s) failed, failed to put old obj(%s) to gc, err: %v", obj.BucketName,
+ obj.ObjectKey, obj.ObjectId, obj.ObjectId, err)
+ return result, err
+ }
+ }
+
+ md5Writer := md5.New()
+
+ // Limit the reader to its provided size if specified.
+ var limitedDataReader io.Reader
+ if size > 0 { // request.ContentLength is -1 if length is unknown
+ limitedDataReader = io.LimitReader(stream, size)
+ } else {
+ limitedDataReader = stream
+ }
+
+ cephCluster, poolName := yig.PickOneClusterAndPool(obj.BucketName, obj.ObjectKey, size, false)
+ if cephCluster == nil {
+ log.Errorf("failed to pick cluster and pool for(%s, %s), err: %v", obj.BucketName, obj.ObjectKey, err)
+ return result, ErrInternalError
+ }
+
+ objMeta := ObjectMetaInfo{
+ Cluster: cephCluster.Name,
+ Pool: poolName,
+ }
+
+ metaBytes, err := json.Marshal(objMeta)
+ if err != nil {
+ log.Errorf("failed to marshal %v for (%s, %s), err: %v", objMeta, obj.BucketName, obj.ObjectKey, err)
+ return result, ErrInternalError
+ }
+
+ // Mapping a shorter name for the object
+ oid := cephCluster.GetUniqUploadName()
+ dataReader := io.TeeReader(limitedDataReader, md5Writer)
+
+ bytesWritten, err := cephCluster.Put(poolName, oid, dataReader)
+ if err != nil {
+ log.Errorf("failed to put(%s, %s), err: %v", poolName, oid, err)
+ return
+ }
+ // Should metadata update failed, put the oid to gc,
+ // so the object in Ceph could be removed asynchronously
+ shouldGc := false
+ defer func() {
+ if shouldGc {
+ if err := yig.putObjToGc(cephCluster.Name, poolName, oid); err != nil {
+ log.Errorf("failed to put (%s, %s, %s) to gc, err: %v", cephCluster.Name, poolName, oid, err)
+ }
+ }
+ }()
+ if bytesWritten < size {
+ shouldGc = true
+ log.Errorf("failed to write objects, already written(%d), total size(%d)", bytesWritten, size)
+ return result, ErrIncompleteBody
+ }
+
+ calculatedMd5 := hex.EncodeToString(md5Writer.Sum(nil))
+ log.Info("### calculatedMd5:", calculatedMd5, "userMd5:", userMd5)
+ if userMd5 != "" && userMd5 != calculatedMd5 {
+ shouldGc = true
+ return result, ErrBadDigest
+ }
+
+ if err != nil {
+ shouldGc = true
+ return
+ }
+
+ // set the bytes written.
+ result.Written = bytesWritten
+ result.ObjectId = oid
+ result.Etag = calculatedMd5
+ result.UpdateTime = time.Now().Unix()
+ result.Meta = string(metaBytes)
+
+ return result, nil
+}
+
+func (yig *YigStorage) Get(ctx context.Context, object *pb.Object, start int64, end int64) (io.ReadCloser, error) {
+ // if object.StorageMeta is nil, it may be the multipart uploaded object.
+ if len(object.StorageMeta) == 0 {
+ reader, err := NewMultipartReader(yig, object.ObjectId, start, end)
+ if err != nil {
+ log.Errorf("failed to get object(%s, %s, %s) with start(%d), end(%d), err: %v", object.BucketName, object.ObjectKey, object.ObjectId, start, end, err)
+ return nil, err
+ }
+ return reader, nil
+ }
+ // get the cluster name and pool name from meta data of object
+ objMeta, err := ParseObjectMeta(object.StorageMeta)
+ if err != nil {
+ log.Errorf("failed to unmarshal storage meta (%s) for (%s, %s), err: %v", object.StorageMeta, object.BucketName, object.ObjectKey, err)
+ return nil, ErrUnmarshalFailed
+ }
+
+ cephCluster, ok := yig.DataStorage[objMeta.Cluster]
+ if !ok {
+ log.Errorf("cannot find ceph cluster(%s) for obj(%s, %s)", objMeta.Cluster, object.BucketName, object.ObjectKey)
+ return nil, ErrInvalidObjectName
+ }
+
+ len := end - start + 1
+ reader, err := cephCluster.getReader(objMeta.Pool, object.ObjectId, start, len)
+ if err != nil {
+ log.Errorf("failed to get ceph reader for pool(%s), obj(%s,%s) with err: %v", objMeta.Pool, object.BucketName, object.ObjectKey, err)
+ return nil, err
+ }
+
+ return reader, nil
+}
+
+/*
+* @objectId: input object id which will be deleted.
+* Below is the process logic:
+* 1. check whether objectId is multipart uploaded, if so
+* retrieve all the object ids from multiparts and put them into gc.
+* or else, put it into gc.
+*
+ */
+func (yig *YigStorage) Delete(ctx context.Context, object *pb.DeleteObjectInput) error {
+ // For multipart uploaded objects, no storage metas are returned to caller,
+ // so, when delete these objects, the meta will be empty.
+ // we need to perform check for multipart uploaded objects.
+ if object.StorageMeta == "" {
+ uploadId, err := str2UploadId(object.ObjectId)
+ if err != nil {
+ log.Errorf("Delete(%s, %s, %s) failed, failed to parse uploadId(%s), err: %v", object.Bucket,
+ object.Key, object.ObjectId, object.ObjectId, err)
+ return err
+ }
+ parts, err := yig.MetaStorage.ListParts(uploadId)
+ if err != nil {
+ log.Errorf("Delete(%s, %s, %s) failed, cannot listParts(%d), err: %v", object.Bucket,
+ object.Key, object.ObjectId, uploadId, err)
+ return err
+ }
+ err = yig.MetaStorage.PutPartsInGc(parts)
+ if err != nil {
+ log.Errorf("Delete(%s, %s, %s) failed, failed to put parts in gc, err: %v", object.Bucket,
+ object.Key, object.ObjectId, err)
+ return err
+ }
+
+ return nil
+ }
+
+ // put the normal object into gc.
+ objMeta, err := ParseObjectMeta(object.StorageMeta)
+ if err != nil {
+ log.Errorf("Delete(%s, %s, %s) failed, cannot parse meta(%s), err: %v", object.Bucket,
+ object.Key, object.ObjectId, object.StorageMeta, err)
+ return err
+ }
+ gcObj := &types.GcObject{
+ Location: objMeta.Cluster,
+ Pool: objMeta.Pool,
+ ObjectId: object.ObjectId,
+ }
+ err = yig.MetaStorage.PutGcObjects(gcObj)
+ if err != nil {
+ log.Errorf("Delete(%s, %s, %s) failed, failed to put gc object, err: %v", object.Bucket,
+ object.Key, object.ObjectId, err)
+ return err
+ }
+ return nil
+}
+
+func (yig *YigStorage) ChangeStorageClass(ctx context.Context, object *pb.Object, newClass *string) error {
+ return errors.New("not implemented.")
+}
+
+/*
+* target: should contain BucketName, ObjectKey, Size, Etag
+*
+ */
+
+func (yig *YigStorage) Copy(ctx context.Context, stream io.Reader, target *pb.Object) (result dscommon.PutResult, err error) {
+ var limitedDataReader io.Reader
+ limitedDataReader = io.LimitReader(stream, target.Size)
+ cephCluster, poolName := yig.PickOneClusterAndPool(target.BucketName, target.ObjectKey, target.Size, false)
+ md5Writer := md5.New()
+ oid := cephCluster.GetUniqUploadName()
+
+ objMeta := ObjectMetaInfo{
+ Cluster: cephCluster.Name,
+ Pool: poolName,
+ }
+
+ metaBytes, err := json.Marshal(objMeta)
+ if err != nil {
+ log.Errorf("failed to marshal %v for (%s, %s), err: %v", objMeta, target.BucketName, target.ObjectKey, err)
+ return result, ErrInternalError
+ }
+
+ dataReader := io.TeeReader(limitedDataReader, md5Writer)
+ var bytesWritten int64
+ bytesWritten, err = cephCluster.Put(poolName, oid, dataReader)
+ if err != nil {
+ log.Errorf("failed to write oid[%s] for obj[%s] in bucket[%s] with err: %v", oid, target.ObjectKey, target.BucketName, err)
+ return result, err
+ }
+ // Should metadata update failed, put the object to gc,
+ // so the object in Ceph could be removed asynchronously
+ shouldGc := false
+ defer func() {
+ if shouldGc {
+ if err := yig.putObjToGc(cephCluster.Name, poolName, oid); err != nil {
+ log.Errorf("failed to put(%s, %s, %s) to gc, err: %v", cephCluster.Name, poolName, oid, err)
+ }
+ }
+ }()
+
+ if bytesWritten < target.Size {
+ shouldGc = true
+ return result, ErrIncompleteBody
+ }
+
+ calculatedMd5 := hex.EncodeToString(md5Writer.Sum(nil))
+ if calculatedMd5 != target.Etag {
+ shouldGc = true
+ return result, ErrBadDigest
+ }
+
+ result.Etag = calculatedMd5
+ result.Written = bytesWritten
+ result.ObjectId = oid
+ result.UpdateTime = time.Now().Unix()
+ result.Meta = string(metaBytes)
+ target.ObjectId = oid
+
+ log.Debugf("succeeded to copy object[%s] in bucket[%s] with oid[%s]", target.ObjectKey, target.BucketName, oid)
+ return result, nil
+}
+
+func (yig *YigStorage) putObjToGc(location, pool, objectId string) error {
+ gcObj := &types.GcObject{
+ Location: location,
+ Pool: pool,
+ ObjectId: objectId,
+ }
+ err := yig.MetaStorage.PutGcObjects(gcObj)
+ if err != nil {
+ log.Errorf("Delete(%s, %s, %s) failed, failed to put gc object, err: %v", location,
+ pool, objectId, err)
+ return err
+ }
+ return nil
+}
+
+func ParseObjectMeta(meta string) (ObjectMetaInfo, error) {
+ objMeta := ObjectMetaInfo{}
+
+ err := json.Unmarshal([]byte(meta), &objMeta)
+ if err != nil {
+ return objMeta, err
+ }
+
+ return objMeta, nil
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package storage
+
+import (
+ "context"
+ "crypto/aes"
+ "crypto/cipher"
+ "crypto/rand"
+ "errors"
+ "io"
+ "path/filepath"
+
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/config"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/crypto"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/meta"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/utils"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ AES_BLOCK_SIZE = 16
+ ENCRYPTION_KEY_LENGTH = 32 // key size for AES-"256"
+ INITIALIZATION_VECTOR_LENGTH = 16 // block size of AES
+ DEFAULT_CEPHCONFIG_PATTERN = "conf/*.conf"
+)
+
+var (
+ RootContext = context.Background()
+)
+
+// YigStorage implements StorageDriver
+type YigStorage struct {
+ DataStorage map[string]*CephStorage
+ MetaStorage *meta.Meta
+ KMS crypto.KMS
+ Stopping bool
+ idGen *utils.GlobalIdGen
+ gcMgr *GcMgr
+}
+
+func New(cfg *config.Config) (*YigStorage, error) {
+ kms := crypto.NewKMS()
+ metaCfg := meta.MetaConfig{
+ Dbcfg: cfg.Database,
+ }
+ metaStorage, err := meta.New(metaCfg)
+ if err != nil {
+ log.Errorf("failed to new meta, err: %v", err)
+ return nil, err
+ }
+ idGen, err := utils.NewGlobalIdGen(int64(cfg.Endpoint.MachineId))
+ if err != nil {
+ log.Errorf("failed to new global id generator, err: %v", err)
+ return nil, err
+ }
+ yig := YigStorage{
+ DataStorage: make(map[string]*CephStorage),
+ MetaStorage: metaStorage,
+ KMS: kms,
+ Stopping: false,
+ idGen: idGen,
+ }
+ CephConfigPattern := cfg.StorageCfg.CephPath
+ if CephConfigPattern == "" {
+ CephConfigPattern = DEFAULT_CEPHCONFIG_PATTERN
+ }
+ cephConfs, err := filepath.Glob(CephConfigPattern)
+ log.Infof("Reading Ceph conf files from %+v\n", cephConfs)
+ if err != nil || len(cephConfs) == 0 {
+ log.Errorf("PANIC: No ceph conf found")
+ err = errors.New("no ceph conf found")
+ return nil, err
+ }
+
+ for _, conf := range cephConfs {
+ c := NewCephStorage(conf)
+ if c != nil {
+ yig.DataStorage[c.Name] = c
+ }
+ }
+
+ if len(yig.DataStorage) == 0 {
+ log.Errorf("PANIC: No data storage can be used!")
+ err = errors.New("no working data storage")
+ return nil, err
+ }
+
+ yig.gcMgr = NewGcMgr(RootContext, &yig, cfg.Endpoint.GcCheckTime)
+ // start gc
+ yig.gcMgr.Start()
+ return &yig, nil
+}
+
+func (y *YigStorage) Close() error {
+ y.Stopping = true
+ log.Info("Stopping storage...")
+ y.gcMgr.Stop()
+ log.Info("done")
+ log.Info("Stopping MetaStorage...")
+ y.MetaStorage.Close()
+
+ return nil
+}
+
+func newInitializationVector() (initializationVector []byte, err error) {
+
+ initializationVector = make([]byte, INITIALIZATION_VECTOR_LENGTH)
+ _, err = io.ReadFull(rand.Reader, initializationVector)
+ return
+}
+
+// Wraps reader with encryption if encryptionKey is not empty
+func wrapEncryptionReader(reader io.Reader, encryptionKey []byte,
+ initializationVector []byte) (wrappedReader io.Reader, err error) {
+
+ if len(encryptionKey) == 0 {
+ return reader, nil
+ }
+
+ var block cipher.Block
+ block, err = aes.NewCipher(encryptionKey)
+ if err != nil {
+ return
+ }
+ stream := cipher.NewCTR(block, initializationVector)
+ wrappedReader = cipher.StreamReader{
+ S: stream,
+ R: reader,
+ }
+ return
+}
+
+type alignedReader struct {
+ aligned bool // indicate whether alignment has already been done
+ offset int64
+ reader io.Reader
+}
+
+func (r *alignedReader) Read(p []byte) (n int, err error) {
+ if r.aligned {
+ return r.reader.Read(p)
+ }
+
+ r.aligned = true
+ buffer := make([]byte, len(p))
+ n, err = r.reader.Read(buffer)
+ if err != nil {
+ return
+ }
+
+ n = copy(p, buffer[r.offset:n])
+ return
+}
+
+// AES is a block cipher with block size of 16 bytes, i.e. the basic unit of encryption/decryption
+// is 16 bytes. As an HTTP range request could start from any byte, we need to read one more
+// block if necessary.
+// Also, our chosen mode of operation for YIG is CTR(counter), which features parallel
+// encryption/decryption and random read access. We need all these three features, this leaves
+// us only three choices: ECB, CTR, and GCM.
+// ECB is best known for its insecurity, meanwhile the GCM implementation of golang(as in 1.7) discourage
+// users to encrypt large files in one pass, which requires us to read the whole file into memory. So
+// the implement complexity is similar between GCM and CTR, we choose CTR because it's faster(but more
+// prone to man-in-the-middle modifications)
+//
+// See https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation
+// and http://stackoverflow.com/questions/39347206
+func wrapAlignedEncryptionReader(reader io.Reader, startOffset int64, encryptionKey []byte,
+ initializationVector []byte) (wrappedReader io.Reader, err error) {
+
+ if len(encryptionKey) == 0 {
+ return reader, nil
+ }
+
+ alignedOffset := startOffset / AES_BLOCK_SIZE * AES_BLOCK_SIZE
+ newReader, err := wrapEncryptionReader(reader, encryptionKey, initializationVector)
+ if err != nil {
+ return
+ }
+ if alignedOffset == startOffset {
+ return newReader, nil
+ }
+
+ wrappedReader = &alignedReader{
+ aligned: false,
+ offset: startOffset - alignedOffset,
+ reader: newReader,
+ }
+ return
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+ "errors"
+ "net"
+ "sync"
+ "time"
+)
+
+const (
+ START_EPOCH int64 = 1571385784722
+ MACHINE_LEN uint8 = 10
+ SEQ_LEN uint8 = 12
+ TIMESTAMP_SHIFS uint8 = 22
+ MAX_MACHINE_NUM int64 = 1023
+ MAX_SEQ_NUM int64 = 4095
+)
+
+type GlobalIdGen struct {
+ MachineId int64
+ startTime int64
+ seq int64
+ mux sync.Mutex
+}
+
+/*
+* machineId: a number which specify the id of the current node.
+* if machineId <=0, we will use lower bits of ip address for the node.
+*
+ */
+
+func NewGlobalIdGen(machineId int64) (*GlobalIdGen, error) {
+ gi := &GlobalIdGen{
+ seq: 0,
+ }
+ if machineId <= 0 {
+ id, err := gi.getMachineId()
+ if err != nil {
+ return nil, err
+ }
+ gi.MachineId = id
+ } else {
+ gi.MachineId = machineId
+ }
+ return gi, nil
+}
+
+/*
+* the global id is generated according to snowflake algo.
+* Please refer to https://github.com/twitter-archive/snowflake/tree/snowflake-2010.
+ */
+
+func (gi *GlobalIdGen) GetId() int64 {
+ gi.mux.Lock()
+ defer gi.mux.Unlock()
+ current := time.Now().UnixNano() / 1000000
+ if gi.startTime == current {
+ gi.seq++
+ if gi.seq > MAX_SEQ_NUM {
+ // process the overflow
+ for current <= gi.startTime {
+ current = time.Now().UnixNano() / 1000000
+ }
+ gi.seq = 0
+ }
+ }
+ gi.startTime = current
+ id := int64((current-START_EPOCH)<<TIMESTAMP_SHIFS | (gi.MachineId << MACHINE_LEN) | (gi.seq))
+ return id
+}
+
+func (gi *GlobalIdGen) getMachineId() (int64, error) {
+ addrs, err := net.InterfaceAddrs()
+ if err != nil {
+ return 0, err
+ }
+ for _, address := range addrs {
+ // check the address type and if it is not a loopback the display it
+ if ipNet, ok := address.(*net.IPNet); ok && !ipNet.IP.IsLoopback() {
+ ipAddr := ipNet.IP.To4()
+ if ipAddr != nil {
+ return (int64(ipAddr[2])<<8 + int64(ipAddr[3])) & 0x0fff, nil
+ }
+ }
+ }
+
+ return 0, errors.New("failed to find the private ip addr")
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package db
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/opensds/multi-cloud/s3/pkg/db/drivers/mongo"
+ . "github.com/opensds/multi-cloud/s3/pkg/exception"
+ . "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+// DbAdapter is a global variable that controls database module.
+var DbAdapter DBAdapter
+
+// Init function can perform some initialization work of different databases.
+func Init(db *Database) {
+ switch db.Driver {
+ case "etcd":
+ // C = etcd.Init(db.Driver, db.Crendential)
+ fmt.Printf("etcd is not implemented right now!")
+ return
+ case "mongodb":
+ //DbAdapter = mongo.Init(strings.Split(db.Endpoint, ","))
+ DbAdapter = mongo.Init(db.Endpoint)
+ return
+ default:
+ fmt.Printf("Can't find database driver %s!\n", db.Driver)
+ }
+}
+
+func Exit(db *Database) {
+ switch db.Driver {
+ case "etcd":
+ // C = etcd.Init(db.Driver, db.Crendential)
+ fmt.Printf("etcd is not implemented right now!")
+ return
+ case "mongodb":
+ mongo.Exit()
+ return
+ default:
+ fmt.Printf("Can't find database driver %s!\n", db.Driver)
+ }
+}
+
+type DBAdapter interface {
+ CreateBucket(ctx context.Context, bucket *pb.Bucket) S3Error
+ DeleteBucket(ctx context.Context, name string) S3Error
+ UpdateBucket(ctx context.Context, bucket *pb.Bucket) S3Error
+ GetBucketByName(ctx context.Context, name string, out *pb.Bucket) S3Error
+ ListBuckets(ctx context.Context, in *pb.BaseRequest, out *[]pb.Bucket) S3Error
+ CreateObject(ctx context.Context, in *pb.Object) S3Error
+ UpdateObject(ctx context.Context, in *pb.Object) S3Error
+ DeleteObject(ctx context.Context, in *pb.DeleteObjectInput) S3Error
+ GetObject(ctx context.Context, in *pb.GetObjectInput, out *pb.Object) S3Error
+ CountObjects(ctx context.Context, in *pb.ListObjectsRequest, out *ObjsCountInfo) S3Error
+ UpdateObjMeta(ctx context.Context, objKey *string, bucketName *string, lastmod int64, setting map[string]interface{}) S3Error
+ AddMultipartUpload(ctx context.Context, record *pb.MultipartUploadRecord) S3Error
+ DeleteMultipartUpload(ctx context.Context, record *pb.MultipartUploadRecord) S3Error
+ //ListUploadRecords(ctx context.Context, in *pb.ListMultipartUploadRequest, out *[]pb.MultipartUploadRecord) S3Error
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+ "context"
+ "strings"
+
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ log "github.com/sirupsen/logrus"
+ . "github.com/opensds/multi-cloud/s3/pkg/exception"
+ . "github.com/opensds/multi-cloud/s3/pkg/utils"
+)
+
+func (ad *adapter) DeleteBucket(ctx context.Context, bucketName string) S3Error {
+ //Check if the connctor exist or not
+ ss := ad.s.Copy()
+ defer ss.Close()
+
+ log.Infof("delete bucket, bucketName is %v:", bucketName)
+
+ m := bson.M{DBKEY_NAME: bucketName}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ //Delete it from database
+ err = ss.DB(DataBaseName).C(BucketMD).Remove(m)
+ log.Infof("err is %v:", err)
+ if err != nil {
+ if strings.Contains(err.Error(), "not found") {
+ log.Error("delete bucket from database failed, err: not found.")
+ return NoSuchBucket
+ } else {
+ log.Infof("delete bucket from database failed, err: %v.\n", err.Error())
+ return DBError
+ }
+ } else {
+ log.Infof("Delete bucket from database successfully")
+ return NoError
+ }
+
+ deleteErr := ss.DB(DataBaseName).C(bucketName).DropCollection()
+ if deleteErr != nil && deleteErr != mgo.ErrNotFound {
+ log.Errorf("delete bucket collection from database failed, err: %v.\n", deleteErr)
+ return InternalError
+ }
+
+ return NoError
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+ "context"
+
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ log "github.com/sirupsen/logrus"
+ . "github.com/opensds/multi-cloud/s3/pkg/exception"
+ . "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+func (ad *adapter) GetBucketByName(ctx context.Context, bucketName string, out *pb.Bucket) S3Error {
+ ss := ad.s.Copy()
+ defer ss.Close()
+
+ log.Infof("GetBucketByName: bucketName %s", bucketName)
+
+ m := bson.M{DBKEY_NAME: bucketName}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ err = ss.DB(DataBaseName).C(BucketMD).Find(m).One(out)
+ if err == mgo.ErrNotFound {
+ log.Error("bucket does not exist.")
+ return NoSuchBucket
+ } else if err != nil {
+ log.Errorf("get bucket from database failed, err: %v.\n", err)
+ return InternalError
+ }
+
+ return NoError
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+ "context"
+
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ log "github.com/sirupsen/logrus"
+ . "github.com/opensds/multi-cloud/s3/pkg/exception"
+ . "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+func (ad *adapter) CreateBucket(ctx context.Context, in *pb.Bucket) S3Error {
+ ss := ad.s.Copy()
+ defer ss.Close()
+
+ m := bson.M{DBKEY_NAME: in.Name}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ out := pb.Bucket{}
+ err = ss.DB(DataBaseName).C(BucketMD).Find(m).One(out)
+ if err == mgo.ErrNotFound {
+ err := ss.DB(DataBaseName).C(BucketMD).Insert(&in)
+ if err != nil {
+ log.Errorf("add bucket to database failed, err:%v\n", err)
+ return InternalError
+ }
+ } else {
+ log.Info("the bucket already exists")
+ return BucketAlreadyExists
+ }
+
+ return NoError
+}
+
+func (ad *adapter) UpdateBucket(ctx context.Context, bucket *pb.Bucket) S3Error {
+ //Check if the policy exist or not
+ ss := ad.s.Copy()
+ defer ss.Close()
+
+ log.Infof("update bucket, bucket name is %s\n", bucket.Name)
+
+ m := bson.M{DBKEY_NAME: bucket.Name}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ //Update database
+ err = ss.DB(DataBaseName).C(BucketMD).Update(m, bucket)
+ if err == mgo.ErrNotFound {
+ log.Error("update bucket failed: the specified bucket does not exist.")
+ return NoSuchBucket
+ } else if err != nil {
+ log.Errorf("update bucket in database failed, err: %v.\n", err)
+ return InternalError
+ }
+ return NoError
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+ "context"
+
+ "github.com/globalsign/mgo/bson"
+ log "github.com/sirupsen/logrus"
+ . "github.com/opensds/multi-cloud/s3/pkg/exception"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+func (ad *adapter) ListBuckets(ctx context.Context, in *pb.BaseRequest, out *[]pb.Bucket) S3Error {
+ ss := ad.s.Copy()
+ defer ss.Close()
+ c := ss.DB(DataBaseName).C(BucketMD)
+
+ log.Info("list buckets from database...... \n")
+
+ m := bson.M{}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ err = c.Find(m).All(out)
+ if err != nil {
+ log.Errorf("find buckets from database failed, err:%v\n", err)
+ return DBError
+ }
+
+ return NoError
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+ "context"
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ log "github.com/sirupsen/logrus"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/s3/pkg/exception"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+func (ad *adapter) CountObjects(ctx context.Context, in *pb.ListObjectsRequest, out *utils.ObjsCountInfo) S3Error {
+ ss := ad.s.Copy()
+ defer ss.Close()
+ c := ss.DB(DataBaseName).C(in.Bucket)
+
+ filt := ""
+ if in.Filter[common.KObjKey] != "" {
+ filt = in.Filter[common.KObjKey]
+ }
+
+ m := bson.M{
+ utils.DBKEY_OBJECTKEY: bson.M{"$regex": filt},
+ utils.DBKEY_INITFLAG: bson.M{"$ne": "0"},
+ utils.DBKEY_DELETEMARKER: bson.M{"$ne": "1"},
+ }
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ q1 := bson.M{
+ "$match": m,
+ }
+
+ q2 := bson.M{
+ "$group": bson.M{
+ "_id": nil,
+ "size": bson.M{"$sum": "$size"},
+ "count": bson.M{"$sum": 1},
+ },
+ }
+
+ operations := []bson.M{q1, q2}
+ pipe := c.Pipe(operations)
+ var ret utils.ObjsCountInfo
+ err = pipe.One(&ret)
+ if err == nil {
+ out.Count = ret.Count
+ out.Size = ret.Size
+ log.Infof("count objects of bucket[%s] successfully, count=%d, size=%d\n", in.Bucket, out.Count, out.Size)
+ } else if err == mgo.ErrNotFound {
+ out.Count = 0
+ out.Size = 0
+ log.Infof("count objects of bucket[%s] successfully, count=0, size=0\n", in.Bucket)
+ } else {
+ log.Errorf("count objects of bucket[%s] failed, err:%v\n", in.Bucket, err)
+ return InternalError
+ }
+
+ return NoError
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+ "context"
+ "errors"
+
+ log "github.com/sirupsen/logrus"
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ "github.com/micro/go-micro/metadata"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+)
+
+var adap = &adapter{}
+var DataBaseName = "metadatastore"
+var BucketMD = "metadatabucket"
+
+func Init(host string) *adapter {
+ //fmt.Println("edps:", deps)
+ session, err := mgo.Dial(host)
+ if err != nil {
+ panic(err)
+ }
+ //defer session.Close()
+
+ session.SetMode(mgo.Monotonic, true)
+ adap.s = session
+ adap.userID = "unknown"
+
+ return adap
+}
+
+func Exit() {
+ adap.s.Close()
+}
+
+type adapter struct {
+ s *mgo.Session
+ userID string
+}
+
+func UpdateContextFilter(ctx context.Context, m bson.M) error {
+ // if context is admin, no need filter by tenantId.
+ md, ok := metadata.FromContext(ctx)
+ if !ok {
+ log.Error("get context failed")
+ return errors.New("get context failed")
+ }
+
+ isAdmin, _ := md[common.CTX_KEY_IS_ADMIN]
+ if isAdmin != common.CTX_VAL_TRUE {
+ tenantId, ok := md[common.CTX_KEY_TENANT_ID]
+ if !ok {
+ log.Error("get tenantid failed")
+ return errors.New("get tenantid failed")
+ }
+ m["tenantid"] = tenantId
+ }
+
+ return nil
+}
+
+
+
package mongo
+
+import (
+ "context"
+ //"time"
+
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ log "github.com/sirupsen/logrus"
+ . "github.com/opensds/multi-cloud/s3/pkg/exception"
+ . "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+var CollMultipartUploadRecord = "multipartUploadRecords"
+
+func (ad *adapter) AddMultipartUpload(ctx context.Context, record *pb.MultipartUploadRecord) S3Error {
+ log.Infof("Add multipart upload: %+v\n", *record)
+ session := ad.s.Copy()
+ defer session.Close()
+
+ err := session.DB(DataBaseName).C(CollMultipartUploadRecord).Insert(record)
+ if err != nil {
+ log.Errorf("add multipart upload record[uploadid=%s] to database failed: %v\n", record.UploadId, err)
+ return DBError
+ }
+
+ log.Infof("add multipart upload record[uploadid=%s] successfully\n", record.UploadId)
+ return NoError
+}
+
+func (ad *adapter) DeleteMultipartUpload(ctx context.Context, record *pb.MultipartUploadRecord) S3Error {
+ log.Infof("Delete multipart upload: %+v\n", *record)
+ session := ad.s.Copy()
+ defer session.Close()
+
+ m := bson.M{DBKEY_OBJECTKEY: record.ObjectKey, DBKEY_UPLOADID: record.UploadId}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ // objectkey is unique in OpenSDS, uploadid is unique for a specific physical bucket
+ err = session.DB(DataBaseName).C(CollMultipartUploadRecord).Remove(m)
+ if err != nil && err != mgo.ErrNotFound {
+ log.Errorf("delete multipart upload record[uploadid=%s] from database failed: %v\n", record.UploadId, err)
+ return DBError
+ }
+
+ log.Infof("delete multipart upload record[uploadid=%s] from database sucessfully\n", record.UploadId)
+ return NoError
+}
+
+/*func (ad *adapter) ListUploadRecords(ctx context.Context, in *pb.ListMultipartUploadRequest,
+ out *[]pb.MultipartUploadRecord) S3Error {
+ ss := ad.s.Copy()
+ defer ss.Close()
+
+ secs := time.Now().Unix() - int64(in.Days*24*60*60)
+ log.Infof("list upload records here: bucket=%s, prefix=%s, daysAfterInitiation=%d, limit=%d, offset=%d, secs=%d\n",
+ in.Bucket, in.Prefix, in.Days, in.Limit, in.Offset, secs)
+
+ m := bson.M{DBKEY_BUCKET: in.Bucket, DBKEY_INITTIME: bson.M{"$lte": secs}, DBKEY_OBJECTKEY: bson.M{"$regex": "^" +
+ in.Prefix}}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ err = ss.DB(DataBaseName).C(CollMultipartUploadRecord).Find(m).Skip(int(in.Offset)).Limit(int(in.Limit)).All(out)
+ if err != nil && err != mgo.ErrNotFound {
+ log.Errorf("list upload records failed:%v\n", err)
+ return DBError
+ }
+
+ return NoError
+}*/
+
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+ "context"
+
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ log "github.com/sirupsen/logrus"
+ . "github.com/opensds/multi-cloud/s3/pkg/exception"
+ . "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+func (ad *adapter) DeleteObject(ctx context.Context, in *pb.DeleteObjectInput) S3Error {
+ //Check if the connctor exist or not
+ ss := ad.s.Copy()
+ defer ss.Close()
+
+ m := bson.M{DBKEY_OBJECTKEY: in.Key}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ //Delete it from database
+ _, err = ss.DB(DataBaseName).C(in.Bucket).RemoveAll(m)
+ if err == mgo.ErrNotFound {
+ log.Errorf("delete object %s failed, err: the specified object does not exist", in.Key)
+ return NoSuchObject
+ } else if err != nil {
+ log.Errorf("delete object %s from database failed,err:%v.\n", in.Key, err)
+ return InternalError
+ }
+ return NoError
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+ "context"
+
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ log "github.com/sirupsen/logrus"
+ . "github.com/opensds/multi-cloud/s3/pkg/exception"
+ . "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+func (ad *adapter) GetObject(ctx context.Context, in *pb.GetObjectInput, out *pb.Object) S3Error {
+ ss := ad.s.Copy()
+ defer ss.Close()
+ log.Info("Find object from database...... \n")
+
+ m := bson.M{DBKEY_OBJECTKEY: in.Key}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ err = ss.DB(DataBaseName).C(in.Bucket).Find(m).One(&out)
+ if err == mgo.ErrNotFound {
+ log.Error("object does not exist.")
+ return NoSuchObject
+ } else if err != nil {
+ log.Errorf("find object from database failed, err:%v\n", err)
+ return InternalError
+ }
+
+ return NoError
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+ "context"
+
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ log "github.com/sirupsen/logrus"
+ . "github.com/opensds/multi-cloud/s3/pkg/exception"
+ . "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+)
+
+func (ad *adapter) CreateObject(ctx context.Context, in *pb.Object) S3Error {
+ ss := ad.s.Copy()
+ defer ss.Close()
+
+ m := bson.M{DBKEY_OBJECTKEY: in.ObjectKey}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ out := pb.Object{}
+ err = ss.DB(DataBaseName).C(in.BucketName).Find(m).One(out)
+ if err == mgo.ErrNotFound {
+ err := ss.DB(DataBaseName).C(in.BucketName).Insert(&in)
+ if err != nil {
+ log.Info("add object to database failed, err:%v\n", err)
+ return InternalError
+ }
+ } else if err != nil {
+ return InternalError
+ }
+
+ return NoError
+}
+
+func (ad *adapter) UpdateObject(ctx context.Context, in *pb.Object) S3Error {
+ ss := ad.s.Copy()
+ defer ss.Close()
+
+ m := bson.M{DBKEY_OBJECTKEY: in.ObjectKey}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ log.Infof("update object:%+v\n", *in)
+ err = ss.DB(DataBaseName).C(in.BucketName).Update(m, in)
+ if err == mgo.ErrNotFound {
+ log.Info("update object to database failed, err:%v\n", err)
+ return NoSuchObject
+ } else if err != nil {
+ log.Info("update object to database failed, err:%v\n", err)
+ return InternalError
+ }
+
+ return NoError
+}
+
+func (ad *adapter) UpdateObjMeta(ctx context.Context, objKey *string, bucketName *string, lastmod int64,
+ setting map[string]interface{}) S3Error {
+ ss := ad.s.Copy()
+ defer ss.Close()
+
+ log.Infof("update object metadata: key=%s, bucket=%s, lastmodified=%d\n", *objKey, *bucketName, lastmod)
+
+ m := bson.M{DBKEY_OBJECTKEY: *objKey, DBKEY_LASTMODIFIED: lastmod}
+ err := UpdateContextFilter(ctx, m)
+ if err != nil {
+ return InternalError
+ }
+
+ data := bson.M{"$set": setting}
+ err = ss.DB(DataBaseName).C(*bucketName).Update(m, data)
+ if err != nil {
+ log.Errorf("update object[key=%s] metadata failed:%v.\n", *objKey, err)
+ return DBError
+ }
+
+ return NoError
+}
+
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package _exception
+
+import (
+ "errors"
+ "fmt"
+)
+
+type S3Error struct {
+ Code int
+ Description string
+}
+
+func (err *S3Error) Error() error {
+ s := fmt.Sprintf("{\"code\":\"%d\",\"message\":\"%s\"}", err.Code, err.Description)
+ return errors.New(s)
+}
+
+var ERR_OK = 200
+var NoError = S3Error{Code: ERR_OK}
+var InternalError = S3Error{Code: 500, Description: "Internal error. Please retry"}
+var NoSuchBucket = S3Error{Code: 404, Description: "The specified bucket does not exist."}
+var DBError = S3Error{Code: 500, Description: "DB occured exception."}
+var NoSuchObject = S3Error{Code: 404, Description: "The specified object does not exist."}
+var BucketAlreadyExists = S3Error{Code: 409, Description: "The requested bucket name already exist. Bucket namespace is shared by all users in the system. Select a different name and retry."}
+
+var NoSuchBackend = S3Error{Code: 404, Description: "The specified backend does not exists."}
+var NoSuchType = S3Error{Code: 404, Description: "The specified backend type does not exists."}
+var BucketDeleteError = S3Error{Code: 500, Description: "The bucket can not be deleted. please delete object first"}
+var BackendDeleteError = S3Error{Code: 500, Description: "The backend can not be deleted. please delete bucket first"}
+
+var InvalidQueryParameter = S3Error{Code:400, Description:"invalid query parameter"}
+var InvalidContentLength = S3Error{Code:400, Description:"invalid content length"}
+var InvalidStorageClass = S3Error{Code: 400, Description: "the storage class you specified is not valid"}
+var BadRequest = S3Error{Code:400, Description:"request is invalid"}
+
+
+
package gc
+
+import (
+ "context"
+ "strconv"
+ "time"
+
+ "github.com/micro/go-micro/client"
+ bkd "github.com/opensds/multi-cloud/backend/proto"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+
+ "github.com/micro/go-micro/metadata"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/driver"
+ "github.com/opensds/multi-cloud/s3/pkg/meta"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+var CTX context.Context
+var CancleFunc context.CancelFunc
+
+const (
+ LIST_LIMIT = 1000
+)
+
+func Init(ctx context.Context, cancelFunc context.CancelFunc, meta *meta.Meta) {
+ mt := meta
+ CTX = ctx
+ CancleFunc = cancelFunc
+ backend := bkd.NewBackendService("backend", client.DefaultClient)
+ go Run(mt, backend)
+}
+
+func Stop() {
+ CancleFunc()
+}
+
+func Run(mt *meta.Meta, bkservice bkd.BackendService) {
+ for {
+ select {
+ case <-time.After(60 * time.Second):
+ case <-CTX.Done():
+ log.Infoln("gc exit...")
+ return
+ }
+
+ offset := 0
+ for {
+ // get gc objects
+ log.Debugln("list gc objects ...")
+ objs, err := mt.ListGcObjs(CTX, offset, LIST_LIMIT)
+ if err != nil {
+ log.Warnf("list gc objects failed, err:%v\n", err)
+ // try in next round
+ break
+ }
+
+ total := len(objs)
+ deleted := 0
+ // for each obj, do clean
+ for _, o := range objs {
+ err = CleanFromBackend(o, bkservice)
+ if err == nil {
+ err = mt.DeleteGcobjRecord(CTX, o)
+ if err != nil {
+ // if delete failed, it will be deleted in the next round
+ log.Warnf("delete gc object[key=%s,version=%s] metadata failed, err:%v\n", o.ObjectKey, o.VersionId, err)
+ } else {
+ deleted++
+ }
+ }
+ }
+ // if some obj deleted failed, do not try to delete it again in this round, but do it in next round
+ offset += total - deleted
+ log.Debugf("total=%d, deleted=%d, offset=%d\n", total, deleted, offset)
+
+ if total < LIST_LIMIT {
+ log.Debugln("break this round of gc")
+ break
+ }
+ }
+ }
+}
+
+func CleanFromBackend(obj *types.Object, bkservice bkd.BackendService) error {
+ ctx := metadata.NewContext(context.Background(), map[string]string{
+ common.CTX_KEY_IS_ADMIN: strconv.FormatBool(true),
+ })
+ backend, err := utils.GetBackend(ctx, bkservice, obj.Location)
+ if err != nil {
+ log.Errorf("get backend faild, err:%v\n", err)
+ return err
+ }
+
+ sd, err := driver.CreateStorageDriver(backend.Type, backend)
+ if err != nil {
+ log.Errorf("failed to create storage driver for %s, err:%v\n", backend.Type, err)
+ return err
+ }
+
+ // delete object data in backend
+ log.Debugf("delete object, key=%s, verionid=%s, objectid=%s, storageMeta:%+v\n", obj.ObjectKey, obj.VersionId, obj.ObjectId, obj.StorageMeta)
+ err = sd.Delete(ctx, &pb.DeleteObjectInput{Bucket: obj.BucketName, Key: obj.ObjectKey, VersioId: obj.VersionId,
+ StorageMeta: obj.StorageMeta, ObjectId: obj.ObjectId})
+ if err != nil {
+ log.Errorf("failed to delete obejct[%s] from backend storage, err:", obj.ObjectKey, err)
+ } else {
+ log.Infof("delete obejct[%s] from backend storage successfully.", obj.ObjectKey)
+ }
+
+ return err
+}
+
+
+
package helper
+
+import (
+ "fmt"
+ "io/ioutil"
+ "time"
+
+ "github.com/BurntSushi/toml"
+)
+
+const (
+ S3_CONF_PATH = "/etc/multi-cloud/s3.toml"
+ MIN_DOWNLOAD_BUFPOOL_SIZE = 512 << 10 // 512k
+ MAX_DOWNLOAD_BUFPOOL_SIZE = 8 << 20 // 8M
+)
+
+type Config struct {
+ S3Domain []string `toml:"s3domain"` // Domain name of YIG
+ Region string `toml:"region"` // Region name this instance belongs to, e.g cn-bj-1
+ Plugins map[string]PluginConfig `toml:"plugins"`
+ LogPath string `toml:"log_path"`
+ AccessLogPath string `toml:"access_log_path"`
+ AccessLogFormat string `toml:"access_log_format"`
+ PanicLogPath string `toml:"panic_log_path"`
+ PidFile string `toml:"pid_file"`
+ BindApiAddress string `toml:"api_listener"`
+ BindAdminAddress string `toml:"admin_listener"`
+ SSLKeyPath string `toml:"ssl_key_path"`
+ SSLCertPath string `toml:"ssl_cert_path"`
+ ZookeeperAddress string `toml:"zk_address"`
+
+ InstanceId string // if empty, generated one at server startup
+ ConcurrentRequestLimit int
+ HbaseZnodeParent string // won't change default("/hbase") if leave this option empty
+ HbaseTimeout time.Duration // in seconds
+ DebugMode bool `toml:"debug_mode"`
+ AdminKey string `toml:"admin_key"` //used for tools/admin to communicate with yig
+ GcThread int `toml:"gc_thread"`
+ LcThread int //used for tools/lc only, set worker numbers to do lc
+ LcDebug bool //used for tools/lc only, if this was set true, will treat days as seconds
+ LogLevel int `toml:"log_level"` //1-20
+ CephConfigPattern string `toml:"ceph_config_pattern"`
+ ReservedOrigins string `toml:"reserved_origins"` // www.ccc.com,www.bbb.com,127.0.0.1
+ MetaStore string `toml:"meta_store"`
+ TidbInfo string `toml:"tidb_info"`
+ KeepAlive bool `toml:"keepalive"`
+
+ //About cache
+ RedisAddress string `toml:"redis_address"` // redis connection string, e.g localhost:1234
+ RedisConnectionNumber int `toml:"redis_connection_number"` // number of connections to redis(i.e max concurrent request number)
+ RedisPassword string `toml:"redis_password"` // redis auth password
+ MetaCacheType int `toml:"meta_cache_type"`
+ EnableDataCache bool `toml:"enable_data_cache"`
+ RedisMode int `toml:"redis_mode"`
+ RedisNodes []string `toml:"redis_nodes"`
+ RedisSentinelMasterName string `toml:"redis_sentinel_master_name"`
+ RedisConnectTimeout int `toml:"redis_connect_timeout"`
+ RedisReadTimeout int `toml:"redis_read_timeout"`
+ RedisWriteTimeout int `toml:"redis_write_timeout"`
+ RedisKeepAlive int `toml:"redis_keepalive"`
+ RedisPoolMaxIdle int `toml:"redis_pool_max_idle"`
+ RedisPoolIdleTimeout int `toml:"redis_pool_idle_timeout"`
+
+ // If the value is not 0, the cached ping detection will be turned on, and the interval is the number of seconds.
+ CacheCircuitCheckInterval int `toml:"cache_circuit_check_interval"`
+ // This property sets the amount of seconds, after tripping the circuit,
+ // to reject requests before allowing attempts again to determine if the circuit should again be closed.
+ CacheCircuitCloseSleepWindow int `toml:"cache_circuit_close_sleep_window"`
+ // This value is how may consecutive passing requests are required before the circuit is closed
+ CacheCircuitCloseRequiredCount int `toml:"cache_circuit_close_required_count"`
+ // This property sets the minimum number of requests in a rolling window that will trip the circuit.
+ CacheCircuitOpenThreshold int `toml:"cache_circuit_open_threshold"`
+
+ DownLoadBufPoolSize int `toml:"download_buf_pool_size"`
+
+ KMS KMSConfig `toml:"kms"`
+
+ // Message Bus
+ MsgBus MsgBusConfig `toml:"msg_bus"`
+}
+
+type PluginConfig struct {
+ Path string `toml:"path"`
+ Enable bool `toml:"enable"`
+ Args map[string]interface{} `toml:"args"`
+}
+
+type KMSConfig struct {
+ Type string
+ Endpoint string
+ Id string `toml:"kms_id"`
+ Secret string `toml:"kms_secret"`
+ Version int
+ Keyname string
+}
+
+type MsgBusConfig struct {
+ // Controls whether to enable message bus when receive the request.
+ Enabled bool `toml:"msg_bus_enable"`
+ // Controls the under implementation of message bus: 1 for kafka.
+ Type int `toml:"msg_bus_type"`
+ // Controls the message topic used by message bus.
+ Topic string `toml:"msg_bus_topic"`
+ // Controls the request timeout for sending a req through message bus.
+ RequestTimeoutMs int `toml:"msg_bus_request_timeout_ms"`
+ // Controls the total timeout for sending a req through message bus.
+ // It will timeout if min(MessageTimeoutMs, SendMaxRetries * RequestTimeoutMs) meets.
+ MessageTimeoutMs int `toml:"msg_bus_message_timeout_ms"`
+ // Controls the retry time used by message bus if it fails to send a req.
+ SendMaxRetries int `toml:"msg_bus_send_max_retries"`
+ // Controls the settings for the implementation of message bus.
+ // For kafka, the 'broker_list' must be set, like 'broker_list = "kafka:29092"'
+ Server map[string]interface{} `toml:"msg_bus_server"`
+}
+
+var CONFIG Config
+
+func SetupConfig() {
+ MarshalTOMLConfig()
+}
+
+func MarshalTOMLConfig() error {
+ data, err := ioutil.ReadFile(S3_CONF_PATH)
+ if err != nil {
+ if err != nil {
+ panic(fmt.Sprintf("failed to open %s, err: %v", S3_CONF_PATH, err))
+ }
+ }
+
+ fmt.Printf("config data: \n%s\n", string(data))
+ var c Config
+ _, err = toml.Decode(string(data), &c)
+ if err != nil {
+ panic("load yig.toml error: " + err.Error())
+ }
+ fmt.Printf("c: %+v\n", c)
+ // setup CONFIG with defaults
+ CONFIG.S3Domain = c.S3Domain
+ CONFIG.Region = c.Region
+ CONFIG.Plugins = c.Plugins
+ CONFIG.LogPath = c.LogPath
+ CONFIG.AccessLogPath = c.AccessLogPath
+ CONFIG.AccessLogFormat = c.AccessLogFormat
+ CONFIG.PanicLogPath = c.PanicLogPath
+ CONFIG.PidFile = c.PidFile
+ CONFIG.BindApiAddress = c.BindApiAddress
+ CONFIG.BindAdminAddress = c.BindAdminAddress
+ CONFIG.SSLKeyPath = c.SSLKeyPath
+ CONFIG.SSLCertPath = c.SSLCertPath
+ CONFIG.ZookeeperAddress = c.ZookeeperAddress
+ CONFIG.DebugMode = c.DebugMode
+ CONFIG.AdminKey = c.AdminKey
+ CONFIG.LcDebug = c.LcDebug
+ CONFIG.CephConfigPattern = c.CephConfigPattern
+ CONFIG.ReservedOrigins = c.ReservedOrigins
+ CONFIG.TidbInfo = c.TidbInfo
+ CONFIG.KeepAlive = c.KeepAlive
+ CONFIG.InstanceId = Ternary(c.InstanceId == "",
+ string(GenerateRandomId()), c.InstanceId).(string)
+ CONFIG.ConcurrentRequestLimit = Ternary(c.ConcurrentRequestLimit == 0,
+ 10000, c.ConcurrentRequestLimit).(int)
+ CONFIG.HbaseZnodeParent = Ternary(c.HbaseZnodeParent == "",
+ "/hbase", c.HbaseZnodeParent).(string)
+ CONFIG.HbaseTimeout = Ternary(c.HbaseTimeout == 0, 30*time.Second,
+ c.HbaseTimeout).(time.Duration)
+ CONFIG.GcThread = Ternary(c.GcThread == 0,
+ 1, c.GcThread).(int)
+ CONFIG.LcThread = Ternary(c.LcThread == 0,
+ 1, c.LcThread).(int)
+ CONFIG.LogLevel = Ternary(c.LogLevel == 0, 5, c.LogLevel).(int)
+ CONFIG.MetaStore = Ternary(c.MetaStore == "", "tidb", c.MetaStore).(string)
+
+ CONFIG.RedisAddress = c.RedisAddress
+ CONFIG.RedisPassword = c.RedisPassword
+ CONFIG.RedisMode = c.RedisMode
+ CONFIG.RedisNodes = c.RedisNodes
+ CONFIG.RedisSentinelMasterName = c.RedisSentinelMasterName
+ CONFIG.RedisConnectionNumber = Ternary(c.RedisConnectionNumber == 0,
+ 10, c.RedisConnectionNumber).(int)
+ CONFIG.EnableDataCache = c.EnableDataCache
+ CONFIG.MetaCacheType = c.MetaCacheType
+ CONFIG.RedisConnectTimeout = Ternary(c.RedisConnectTimeout < 0, 0, c.RedisConnectTimeout).(int)
+ CONFIG.RedisReadTimeout = Ternary(c.RedisReadTimeout < 0, 0, c.RedisReadTimeout).(int)
+ CONFIG.RedisWriteTimeout = Ternary(c.RedisWriteTimeout < 0, 0, c.RedisWriteTimeout).(int)
+ CONFIG.RedisKeepAlive = Ternary(c.RedisKeepAlive < 0, 0, c.RedisKeepAlive).(int)
+ CONFIG.RedisPoolMaxIdle = Ternary(c.RedisPoolMaxIdle < 0, 0, c.RedisPoolMaxIdle).(int)
+ CONFIG.RedisPoolIdleTimeout = Ternary(c.RedisPoolIdleTimeout < 0, 0, c.RedisPoolIdleTimeout).(int)
+
+ CONFIG.CacheCircuitCheckInterval = Ternary(c.CacheCircuitCheckInterval < 0, 0, c.CacheCircuitCheckInterval).(int)
+ CONFIG.CacheCircuitCloseSleepWindow = Ternary(c.CacheCircuitCloseSleepWindow < 0, 0, c.CacheCircuitCloseSleepWindow).(int)
+ CONFIG.CacheCircuitCloseRequiredCount = Ternary(c.CacheCircuitCloseRequiredCount < 0, 0, c.CacheCircuitCloseRequiredCount).(int)
+ CONFIG.CacheCircuitOpenThreshold = Ternary(c.CacheCircuitOpenThreshold < 0, 0, c.CacheCircuitOpenThreshold).(int)
+
+ CONFIG.DownLoadBufPoolSize = Ternary(c.DownLoadBufPoolSize < MIN_DOWNLOAD_BUFPOOL_SIZE || c.DownLoadBufPoolSize > MAX_DOWNLOAD_BUFPOOL_SIZE, MIN_DOWNLOAD_BUFPOOL_SIZE, c.DownLoadBufPoolSize).(int)
+
+ CONFIG.KMS = c.KMS
+
+ CONFIG.MsgBus = c.MsgBus
+ CONFIG.MsgBus.RequestTimeoutMs = Ternary(c.MsgBus.RequestTimeoutMs == 0, 3000, c.MsgBus.RequestTimeoutMs).(int)
+ CONFIG.MsgBus.MessageTimeoutMs = Ternary(c.MsgBus.MessageTimeoutMs == 0, 5000, c.MsgBus.MessageTimeoutMs).(int)
+ CONFIG.MsgBus.SendMaxRetries = Ternary(c.MsgBus.SendMaxRetries == 0, 2, c.MsgBus.SendMaxRetries).(int)
+
+ return nil
+}
+
+
+
package helper
+
+import (
+ "bytes"
+)
+
+func EscapeColon(s string) string {
+ //Byte loop is OK for utf8
+ buf := new(bytes.Buffer)
+ l := len(s)
+ for i := 0; i < l; i++ {
+ if s[i] == '%' {
+ buf.WriteString("%%")
+ } else if s[i] == ':' {
+ buf.WriteString("%n")
+ } else {
+ buf.WriteByte(s[i])
+ }
+ }
+ return buf.String()
+}
+
+func UnescapeColon(s string) string {
+ buf := new(bytes.Buffer)
+ l := len(s)
+ for i := 0; i < l; i++ {
+ if s[i] == '%' {
+ i++
+ if i == l {
+ panic("never be here")
+ }
+ if s[i] == '%' {
+ buf.WriteString("%")
+ } else if s[i] == 'n' {
+ buf.WriteString(":")
+ } else {
+ panic("never be here")
+ }
+ } else {
+ buf.WriteByte(s[i])
+ }
+ }
+ return buf.String()
+}
+
+
+
package helper
+
+import (
+ "os"
+)
+
+func FileExists(path string) bool {
+ st, e := os.Stat(path)
+ // If file exists and is regular return true.
+ if e == nil && st.Mode().IsRegular() {
+ return true
+ }
+ return false
+}
+
+
+
package helper
+
+func Filter(xs []string, f func(string) bool) []string {
+ ans := make([]string, 0, len(xs))
+ for _, x := range xs {
+ if f(x) {
+ ans = append(ans, x)
+ }
+ }
+ return ans
+}
+
+func Map(xs []string, f func(string) string) []string {
+ ans := make([]string, 0, len(xs))
+ for _, x := range xs {
+ ans = append(ans, f(x))
+ }
+ return ans
+}
+
+
+
package helper
+
+import (
+ "math/rand"
+ "reflect"
+)
+
+// mimic `?:` operator
+// Need type assertion to convert output to expected type
+func Ternary(IF bool, THEN interface{}, ELSE interface{}) interface{} {
+ if IF {
+ return THEN
+ } else {
+ return ELSE
+ }
+}
+
+// Get keys of a map, i.e.
+// map[string]interface{} -> []string
+// Note that some type checks are omitted for efficiency, you need to ensure them yourself,
+// otherwise your program should panic
+func Keys(v interface{}) []string {
+ rv := reflect.ValueOf(v)
+ result := make([]string, 0, rv.Len())
+ for _, kv := range rv.MapKeys() {
+ result = append(result, kv.String())
+ }
+ return result
+}
+
+// Static alphaNumeric table used for generating unique request ids
+var alphaNumericTable = []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+func GenerateRandomId() []byte {
+ alpha := make([]byte, 16, 16)
+ for i := 0; i < 16; i++ {
+ n := rand.Intn(len(alphaNumericTable))
+ alpha[i] = alphaNumericTable[n]
+ }
+ return alpha
+}
+
+
+
package helper
+
+import (
+ "encoding/json"
+ "io"
+ "io/ioutil"
+)
+
+// read from ReadCloser and unmarshal to out;
+// `out` should be of POINTER type
+func ReadJsonBody(body io.ReadCloser, out interface{}) (err error) {
+ defer func() {
+ _ = body.Close()
+ }()
+ jsonBytes, err := ioutil.ReadAll(body)
+ if err != nil {
+ return err
+ }
+ err = json.Unmarshal(jsonBytes, out)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+
+
+
package helper
+
+import (
+ "bytes"
+
+ "github.com/ugorji/go/codec"
+)
+
+func MsgPackMarshal(v interface{}) ([]byte, error) {
+ var buf = new(bytes.Buffer)
+ enc := codec.NewEncoder(buf, new(codec.MsgpackHandle))
+ err := enc.Encode(v)
+ return buf.Bytes(), err
+}
+func MsgPackUnMarshal(data []byte, v interface{}) error {
+ var buf = bytes.NewBuffer(data)
+ dec := codec.NewDecoder(buf, new(codec.MsgpackHandle))
+ return dec.Decode(v)
+}
+
+
+
package helper
+
+import "strings"
+
+func HasBucketInDomain(host string, prefix string, domains []string) (ok bool, bucket string) {
+ for _, d := range domains {
+ if strings.HasSuffix(host, prefix+d) {
+ return true, strings.TrimSuffix(host, prefix+d)
+ }
+ }
+ return false, ""
+}
+
+func StringInSlice(s string, ss []string) bool {
+ for _, x := range ss {
+ if s == x {
+ return true
+ }
+ }
+ return false
+}
+
+func CopiedBytes(source []byte) (destination []byte) {
+ destination = make([]byte, len(source), len(source))
+ copy(destination, source)
+ return destination
+}
+
+func UnicodeIndex(str, substr string) int {
+ result := strings.Index(str, substr)
+ if result >= 0 {
+ prefix := []byte(str)[0:result]
+ rs := []rune(string(prefix))
+ result = len(rs)
+ }
+ return result
+}
+
+func SubString(str string, begin, length int) (substr string) {
+ rs := []rune(str)
+ lth := len(rs)
+ if begin < 0 {
+ begin = 0
+ }
+ if begin >= lth {
+ begin = lth
+ }
+ var end int
+ if length == -1 {
+ end = lth
+ } else {
+ end = begin + length
+ }
+ if end > lth {
+ end = lth
+ }
+ return string(rs[begin:end])
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package meta
+
+import (
+ "context"
+ "fmt"
+
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/redis"
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ BUCKET_CACHE_PREFIX = "bucket:"
+ USER_CACHE_PREFIX = "user:"
+)
+
+// Note the usage info got from this method is possibly not accurate because we don't
+// invalid cache when updating usage. For accurate usage info, use `GetUsage()`
+func (m *Meta) GetBucket(ctx context.Context, bucketName string, willNeed bool) (bucket *Bucket, err error) {
+ getBucket := func() (b helper.Serializable, err error) {
+ bt, err := m.Db.GetBucket(ctx, bucketName)
+ log.Info("GetBucket CacheMiss. bucket:", bucketName)
+ return bt, err
+ }
+
+ toBucket := func(fields map[string]string) (interface{}, error) {
+ b := &Bucket{Bucket: &pb.Bucket{}}
+ return b.Deserialize(fields)
+ }
+
+ b, err := m.Cache.Get(redis.BucketTable, BUCKET_CACHE_PREFIX, bucketName, getBucket, toBucket, willNeed)
+ if err != nil {
+ if err == ErrNoSuchKey {
+ err = ErrNoSuchBucket
+ } else if err != ErrDBError {
+ err = ErrInternalError
+ }
+ log.Errorf("get bucket failed:%v\n", err)
+ return
+ }
+ bucket, ok := b.(*Bucket)
+ if !ok {
+ log.Error("Cast b failed:", b)
+ err = ErrInternalError
+ return
+ }
+ return bucket, nil
+}
+
+/*
+* init bucket usage cache when meta is newed.
+*
+ */
+func (m *Meta) InitBucketUsageCache() error {
+ // the map contains the bucket usage which are not in cache.
+ bucketUsageMap := make(map[string]*Bucket)
+ // the map contains the bucket usage which are in cache and will be synced into database.
+ bucketUsageCacheMap := make(map[string]int64)
+ // the usage in buckets table is accurate now.
+
+ buckets, err := m.Db.GetBuckets(context.Background())
+ if err != nil {
+ log.Error("failed to get buckets from db. err: ", err)
+ return err
+ }
+
+ // init the bucket usage key in cache.
+ for _, bucket := range buckets {
+ bucketUsageMap[bucket.Name] = bucket
+ }
+
+ // try to get all bucket usage keys from cache.
+ pattern := fmt.Sprintf("%s*", BUCKET_CACHE_PREFIX)
+ bucketsInCache, err := m.Cache.Keys(redis.BucketTable, pattern)
+ if err != nil {
+ log.Error("failed to get bucket usage from cache, err: ", err)
+ return err
+ }
+
+ if len(bucketsInCache) > 0 {
+ // query all usages from cache.
+ for _, bic := range bucketsInCache {
+ usage, err := m.Cache.HGetInt64(redis.BucketTable, BUCKET_CACHE_PREFIX, bic, FIELD_NAME_USAGE)
+ if err != nil {
+ log.Error("failed to get usage for bucket: ", bic, " with err: ", err)
+ continue
+ }
+ // add the to be synced usage.
+ bucketUsageCacheMap[bic] = usage
+ if _, ok := bucketUsageMap[bic]; ok {
+ // if the key already exists in cache, then delete it from map
+ delete(bucketUsageMap, bic)
+ }
+ }
+
+ }
+
+ // init the bucket usage in cache.
+ if len(bucketUsageMap) > 0 {
+ for _, bk := range bucketUsageMap {
+ fields, err := bk.Serialize()
+ if err != nil {
+ log.Error("failed to serialize for bucket: ", bk.Name, " with err: ", err)
+ return err
+ }
+ _, err = m.Cache.HMSet(redis.BucketTable, BUCKET_CACHE_PREFIX, bk.Name, fields)
+ if err != nil {
+ log.Error("failed to set bucket to cache: ", bk.Name, " with err: ", err)
+ return err
+ }
+ }
+
+ }
+ // sync the buckets usage in cache into database.
+ if len(bucketUsageCacheMap) > 0 {
+ err = m.Db.UpdateUsages(context.Background(), bucketUsageCacheMap, nil)
+ if err != nil {
+ log.Error("failed to sync usages to database, err: ", err)
+ return err
+ }
+ }
+ return nil
+}
+
+/*func (m *Meta) bucketUsageSync(event SyncEvent) error {
+ bu := &BucketUsageEvent{}
+ err := helper.MsgPackUnMarshal(event.Data.([]byte), bu)
+ if err != nil {
+ log.Error("failed to unpack from event data to BucketUsageEvent, err: %v", err)
+ return err
+ }
+
+ err = m.Db.UpdateUsage(bu.BucketName, bu.Usage, nil)
+ if err != nil {
+ log.Error("failed to update bucket usage ", bu.Usage, " to bucket: ", bu.BucketName,
+ " err: ", err)
+ return err
+ }
+
+ log.Infof("succeed to update bucket usage ", bu.Usage, " for bucket: ", bu.BucketName)
+ return nil
+}*/
+
+func AddBucketUsageSyncEvent(bucketName string, usage int64) {
+ bu := &BucketUsageEvent{
+ Usage: usage,
+ BucketName: bucketName,
+ }
+ data, err := helper.MsgPackMarshal(bu)
+ if err != nil {
+ log.Errorf("failed to package bucket usage event for bucket %s with usage %d, err: %v",
+ bucketName, usage, err)
+ return
+ }
+ if MetaSyncQueue != nil {
+ event := SyncEvent{
+ Type: SYNC_EVENT_TYPE_BUCKET_USAGE,
+ Data: data,
+ }
+ MetaSyncQueue <- event
+ }
+}
+
+
+
package meta
+
+import (
+ "errors"
+
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/redis"
+ log "github.com/sirupsen/logrus"
+)
+
+type CacheType int
+
+const (
+ NoCache CacheType = iota
+ EnableCache
+ SimpleCache
+)
+
+const (
+ MSG_NOT_IMPL = "not implemented."
+)
+
+var cacheNames = [...]string{"NOCACHE", "EnableCache", "SimpleCache"}
+
+type MetaCache interface {
+ Close()
+ Get(table redis.RedisDatabase, prefix, key string,
+ onCacheMiss func() (helper.Serializable, error),
+ onDeserialize func(map[string]string) (interface{}, error),
+ willNeed bool) (value interface{}, err error)
+ Remove(table redis.RedisDatabase, prefix, key string)
+ GetCacheHitRatio() float64
+ Keys(table redis.RedisDatabase, pattern string) ([]string, error)
+ HGetInt64(table redis.RedisDatabase, prefix, key, field string) (int64, error)
+ HMSet(table redis.RedisDatabase, prefix, key string, fields map[string]interface{}) (string, error)
+ HIncrBy(table redis.RedisDatabase, prefix, key, field string, value int64) (int64, error)
+}
+
+type disabledMetaCache struct{}
+
+type entry struct {
+ table redis.RedisDatabase
+ key string
+ value interface{}
+}
+
+func newMetaCache(myType CacheType) (m MetaCache) {
+
+ log.Infof("Setting Up Metadata Cache: %s\n", cacheNames[int(myType)])
+ if myType == SimpleCache {
+ m := new(enabledSimpleMetaCache)
+ m.Hit = 0
+ m.Miss = 0
+ return m
+ }
+ return &disabledMetaCache{}
+}
+
+func (m *disabledMetaCache) Get(table redis.RedisDatabase, prefix, key string,
+ onCacheMiss func() (helper.Serializable, error),
+ onDeserialize func(map[string]string) (interface{}, error),
+ willNeed bool) (value interface{}, err error) {
+ return onCacheMiss()
+}
+
+func (m *disabledMetaCache) Remove(table redis.RedisDatabase, prefix, key string) {
+ return
+}
+
+func (m *disabledMetaCache) GetCacheHitRatio() float64 {
+ return -1
+}
+
+func (m *disabledMetaCache) Keys(table redis.RedisDatabase, pattern string) ([]string, error) {
+ return nil, errors.New(MSG_NOT_IMPL)
+}
+
+func (m *disabledMetaCache) HGetInt64(table redis.RedisDatabase, prefix, key, field string) (int64, error) {
+ return 0, errors.New(MSG_NOT_IMPL)
+}
+
+func (m *disabledMetaCache) HMSet(table redis.RedisDatabase, prefix, key string, fields map[string]interface{}) (string, error) {
+ return "", errors.New(MSG_NOT_IMPL)
+}
+
+func (m *disabledMetaCache) HIncrBy(table redis.RedisDatabase, prefix, key, field string, value int64) (int64, error) {
+ return 0, errors.New(MSG_NOT_IMPL)
+}
+
+func (m *disabledMetaCache) Close() {
+}
+
+type enabledSimpleMetaCache struct {
+ Hit int64
+ Miss int64
+}
+
+func (m *enabledSimpleMetaCache) Get(
+ table redis.RedisDatabase,
+ prefix, key string,
+ onCacheMiss func() (helper.Serializable, error),
+ onDeserialize func(map[string]string) (interface{}, error),
+ willNeed bool) (value interface{}, err error) {
+
+ log.Info("enabledSimpleMetaCache Get. table:", table, "key:", key)
+
+ fields, err := redis.HGetAll(table, prefix, key)
+ if err != nil {
+ log.Error("enabledSimpleMetaCache Get err:", err, "table:", table, "key:", key)
+ }
+ if err == nil && fields != nil && len(fields) > 0 {
+ value, err = onDeserialize(fields)
+ m.Hit = m.Hit + 1
+ return value, err
+ }
+
+ //if redis doesn't have the entry
+ if onCacheMiss != nil {
+ obj, err := onCacheMiss()
+ if err != nil {
+ return nil, err
+ }
+
+ if willNeed == true {
+ values, err := obj.Serialize()
+ if err != nil {
+ log.Error("failed to serialize from %v", obj, " with err: ", err)
+ return nil, err
+ }
+ _, err = redis.HMSet(table, prefix, key, values)
+ if err != nil {
+ log.Error("failed to set key: ", key, " with err: ", err)
+ //do nothing, even if redis is down.
+ }
+ }
+ m.Miss = m.Miss + 1
+ return obj, nil
+ }
+ return nil, nil
+}
+
+func (m *enabledSimpleMetaCache) Remove(table redis.RedisDatabase, prefix, key string) {
+ redis.Remove(table, prefix, key)
+}
+
+func (m *enabledSimpleMetaCache) GetCacheHitRatio() float64 {
+ return float64(m.Hit) / float64(m.Hit+m.Miss)
+}
+
+func (m *enabledSimpleMetaCache) Keys(table redis.RedisDatabase, pattern string) ([]string, error) {
+ return redis.Keys(table, pattern)
+}
+
+func (m *enabledSimpleMetaCache) HGetInt64(table redis.RedisDatabase, prefix, key, field string) (int64, error) {
+ return redis.HGetInt64(table, prefix, key, field)
+}
+
+func (m *enabledSimpleMetaCache) HMSet(table redis.RedisDatabase, prefix, key string, fields map[string]interface{}) (string, error) {
+ return redis.HMSet(table, prefix, key, fields)
+}
+
+func (m *enabledSimpleMetaCache) HIncrBy(table redis.RedisDatabase, prefix, key, field string, value int64) (int64, error) {
+ return redis.HIncrBy(table, prefix, key, field, value)
+}
+
+func (m *enabledSimpleMetaCache) Close() {
+ redis.Close()
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package tidbclient
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "strconv"
+ "strings"
+ "time"
+
+ _ "github.com/go-sql-driver/mysql"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/util"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+const TimeDur = 5000 // milisecond
+
+func (t *TidbClient) GetBucket(ctx context.Context, bucketName string) (bucket *Bucket, err error) {
+ log.Infof("get bucket[%s] from tidb ...\n", bucketName)
+ var acl, cors, lc, policy, replication, createTime string
+ var updateTime sql.NullString
+ var row *sql.Row
+
+ sqltext := "select bucketname,tenantid,createtime,usages,location,acl,cors,lc,policy,versioning,replication," +
+ "update_time from buckets where bucketname=?;"
+ row = t.Client.QueryRow(sqltext, bucketName)
+
+ tmp := &Bucket{Bucket: &pb.Bucket{}}
+ tmp.Versioning = &pb.BucketVersioning{}
+ err = row.Scan(
+ &tmp.Name,
+ &tmp.TenantId,
+ &createTime,
+ &tmp.Usages,
+ &tmp.DefaultLocation,
+ &acl,
+ &cors,
+ &lc,
+ &policy,
+ &tmp.Versioning.Status,
+ &replication,
+ &updateTime,
+ )
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+
+ ct, err := time.Parse(TIME_LAYOUT_TIDB, createTime)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ tmp.CreateTime = ct.Unix()
+
+ pbAcl := pb.Acl{}
+ err = json.Unmarshal([]byte(acl), &pbAcl)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ tmp.Acl = &pbAcl
+
+ err = json.Unmarshal([]byte(cors), &tmp.Cors)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ err = json.Unmarshal([]byte(lc), &tmp.LifecycleConfiguration)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ err = json.Unmarshal([]byte(policy), &tmp.BucketPolicy)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ err = json.Unmarshal([]byte(replication), &tmp.ReplicationConfiguration)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ //get versioning for the bucket
+ versionOpts, versionErr := t.GetBucketVersioning(ctx, tmp.Name)
+ if versionErr != nil {
+ log.Error("error in getting versioning information, err:%v\n", versionErr)
+ err = handleDBError(versionErr)
+ return
+ }
+ tmp.Versioning = &pb.BucketVersioning{}
+ if versionOpts != nil {
+ tmp.Versioning.Status = versionOpts.Status
+ }
+
+ // get SSE info for this bucket
+ tmp.ServerSideEncryption = &pb.ServerSideEncryption{}
+ sseOpts, sseErr := t.GetBucketSSE(ctx, tmp.Name)
+ if sseErr != nil {
+ return
+ }
+ sseType := "NONE"
+ if sseOpts != nil {
+ sseType = sseOpts.SseType
+ tmp.ServerSideEncryption.EncryptionKey = sseOpts.EncryptionKey
+ }
+ tmp.ServerSideEncryption.SseType = sseType
+ bucket = tmp
+ return
+}
+
+// For the request that list buckets, better to filter according to tenant.
+func (t *TidbClient) GetBuckets(ctx context.Context) (buckets []*Bucket, err error) {
+ log.Info("list buckets from tidb ...")
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil {
+ return nil, ErrInternalError
+ }
+
+ var rows *sql.Rows
+ sqltext := "select bucketname,tenantid,userid,createtime,usages,location,deleted,acl,cors,lc,policy," +
+ "versioning,replication,update_time from buckets;"
+
+ if !isAdmin {
+ sqltext = "select bucketname,tenantid,userid,createtime,usages,location,deleted,acl,cors,lc,policy," +
+ "versioning,replication,update_time from buckets where tenantid=?;"
+ rows, err = t.Client.Query(sqltext, tenantId)
+ } else {
+ rows, err = t.Client.Query(sqltext)
+ }
+
+ if err == sql.ErrNoRows {
+ err = nil
+ return
+ } else if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ tmp := Bucket{Bucket: &pb.Bucket{}}
+ bucketVer := pb.BucketVersioning{}
+ tmp.Versioning = &bucketVer
+ var acl, cors, lc, policy, createTime, replication string
+ var updateTime sql.NullString
+ err = rows.Scan(
+ &tmp.Name,
+ &tmp.TenantId,
+ &tmp.UserId,
+ &createTime,
+ &tmp.Usages,
+ &tmp.DefaultLocation,
+ &tmp.Deleted,
+ &acl,
+ &cors,
+ &lc,
+ &policy,
+ &tmp.Versioning.Status,
+ &replication,
+ &updateTime)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+
+ //get versioning for the bucket
+ versionOpts, versionErr := t.GetBucketVersioning(ctx, tmp.Name)
+ if versionErr != nil {
+ log.Error("error in getting versioning information, err:%v\n", versionErr)
+ err = handleDBError(versionErr)
+ return
+ }
+ tmp.Versioning = &pb.BucketVersioning{}
+ if versionOpts != nil {
+ tmp.Versioning.Status = versionOpts.Status
+ }
+
+ // get SSE info for this bucket
+ sseOpts, sseErr := t.GetBucketSSE(ctx, tmp.Name)
+ if sseErr != nil {
+ return
+ }
+ sseType := "NONE"
+ if sseOpts != nil {
+ sseType = sseOpts.SseType
+ }
+ tmp.ServerSideEncryption = &pb.ServerSideEncryption{}
+ tmp.ServerSideEncryption.SseType = sseType
+
+ var ctime time.Time
+ ctime, err = time.ParseInLocation(TIME_LAYOUT_TIDB, createTime, time.Local)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ tmp.CreateTime = ctime.Unix()
+ err = json.Unmarshal([]byte(acl), &tmp.Acl)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ err = json.Unmarshal([]byte(cors), &tmp.Cors)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ err = json.Unmarshal([]byte(lc), &tmp.LifecycleConfiguration)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ err = json.Unmarshal([]byte(policy), &tmp.BucketPolicy)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ err = json.Unmarshal([]byte(replication), &tmp.ReplicationConfiguration)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ buckets = append(buckets, &tmp)
+ }
+ return
+}
+
+//Actually this method is used to update bucket
+func (t *TidbClient) PutBucket(ctx context.Context, bucket *Bucket) error {
+ log.Infof("put bucket[%s] into tidb ...\n", bucket.Name)
+ acl, _ := json.Marshal(bucket.Acl)
+ cors, _ := json.Marshal(bucket.Cors)
+ lc, _ := json.Marshal(bucket.LifecycleConfiguration)
+ bucket_policy, _ := json.Marshal(bucket.BucketPolicy)
+ sql := "update buckets set bucketname=?,acl=?,policy=?,cors=?,lc=?,tenantid=?,versioning=? where bucketname=?"
+ args := []interface{}{bucket.Name, acl, bucket_policy, cors, lc, bucket.TenantId, bucket.Versioning.Status, bucket.Name}
+
+ _, err := t.Client.Exec(sql, args...)
+ if err != nil {
+ return handleDBError(err)
+ }
+
+ return nil
+}
+
+func (t *TidbClient) CheckAndPutBucket(ctx context.Context, bucket *Bucket) (bool, error) {
+ var processed bool
+ _, err := t.GetBucket(ctx, bucket.Name)
+ if err == nil {
+ processed = false
+ return processed, err
+ } else if err != ErrNoSuchKey {
+ processed = false
+ return processed, err
+ } else {
+ processed = true
+ }
+ log.Infof("insert bucket[%s] into database.\n", bucket.Name)
+ sql, args := bucket.GetCreateSql()
+ _, err = t.Client.Exec(sql, args...)
+ if err != nil {
+ err = handleDBError(err)
+
+ }
+ return processed, err
+}
+
+func (t *TidbClient) ListObjects(ctx context.Context, bucketName string, versioned bool, maxKeys int,
+ filter map[string]string) (retObjects []*Object, appendInfo utils.ListObjsAppendInfo, err error) {
+ const MaxObjectList = 10000
+ // TODO: support versioning
+ if versioned {
+ log.Errorf("not supported.")
+ err = ErrInternalError
+ return
+ }
+ defer func() {
+ err = handleDBError(err)
+ }()
+ var count int
+ var exit bool
+ objectMap := make(map[string]struct{})
+ objectNum := make(map[string]int)
+ commonPrefixes := make(map[string]struct{})
+ if filter == nil {
+ filter = make(map[string]string)
+ }
+ omarker := filter[common.KMarker]
+ delimiter := filter[common.KDelimiter]
+ prefix := filter[common.KPrefix]
+ for {
+ var loopcount int
+ var sqltext string
+ var rows *sql.Rows
+ args := make([]interface{}, 0)
+ // only select index column here to avoid slow query
+ sqltext = "select bucketname,name,version from objects where bucketName=? and deletemarker=0"
+ args = append(args, bucketName)
+ var inargs []interface{}
+ sqltext, inargs, err = buildSql(ctx, filter, sqltext)
+ for _, v := range inargs {
+ args = append(args, v)
+ }
+ log.Infof("sqltext:%s, args:%v\n", sqltext, args)
+ tstart := time.Now()
+ rows, err = t.Client.Query(sqltext, args...)
+ if err != nil {
+ return
+ }
+ tqueryend := time.Now()
+ tdur := tqueryend.Sub(tstart).Nanoseconds()
+ if tdur/1000000 > TimeDur {
+ log.Debugf("slow query when list objects, sqltext:%s, args:%v, nanoseconds:%d\n", sqltext, args, tdur)
+ }
+ defer rows.Close()
+ for rows.Next() {
+ loopcount += 1
+ //var name, lastModified string
+ var bname, name string
+ var version uint64
+ err = rows.Scan(
+ &bname,
+ &name,
+ &version,
+ )
+ if err != nil {
+ log.Errorf("err:%v\n", err)
+ return
+ }
+ //prepare next marker, marker is last bucketname got from the last query,
+ //TODU: be sure how tidb/mysql compare strings
+ if _, ok := objectNum[name]; !ok {
+ objectNum[name] = 0
+ }
+ objectNum[name] += 1
+ filter[common.KMarker] = name
+
+ if _, ok := objectMap[name]; !ok {
+ objectMap[name] = struct{}{}
+ } else {
+ continue
+ }
+
+ // TODO: filter by deletemarker if versioning enabled
+
+ if name == omarker {
+ log.Infof("%s euqals to omarker, continue\n", name)
+ continue
+ }
+
+ // Group by delimiter, delimiter is used to group object keys. Object keys that contain the same string
+ // between the prefix and the first occurrence of the delimiter to be rolled up into a single result element
+ // in the CommonPrefixes collection. These rolled-up keys are not returned elsewhere in the response. Each
+ // rolled-up result counts as only one return against the MaxKeys value. Those are compatible with AWS S3,
+ // please see https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_ListObjectsV2.html for details.
+ if len(delimiter) != 0 {
+ subStr := strings.TrimPrefix(name, prefix)
+ n := strings.Index(subStr, delimiter)
+ if n != -1 {
+ prefixKey := prefix + string([]byte(subStr)[0:(n+1)])
+ filter[common.KMarker] = prefixKey[0:(len(prefixKey)-1)] + string(delimiter[len(delimiter)-1]+1)
+ if prefixKey == omarker {
+ continue
+ }
+ if _, ok := commonPrefixes[prefixKey]; !ok {
+ if count == maxKeys {
+ appendInfo.Truncated = true
+ exit = true
+ break
+ }
+ commonPrefixes[prefixKey] = struct{}{}
+ // When response is truncated (the IsTruncated element value in the response is true), you can
+ // use the key name in this field as marker in the subsequent request to get next set of objects,
+ // the same as AWS S3. See https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_ListObjects.html
+ // for details.
+ appendInfo.NextMarker = prefixKey
+ count += 1
+ }
+ continue
+ }
+ }
+
+ var o *Object
+ strVer := strconv.FormatUint(version, 10)
+ o, err = t.GetObject(ctx, bname, name, strVer)
+ if err != nil {
+ log.Errorf("err:%v\n", err)
+ return
+ }
+
+ count += 1
+ if count == maxKeys {
+ appendInfo.NextMarker = name
+ }
+
+ if count > maxKeys {
+ appendInfo.Truncated = true
+ exit = true
+ break
+ }
+
+ retObjects = append(retObjects, o)
+ }
+
+ tend := time.Now()
+ tdur = tend.Sub(tqueryend).Nanoseconds()
+ if tdur/1000000 > TimeDur {
+ log.Debugf("slow get when list objects, time:%d\n", tdur)
+ }
+
+ if loopcount < MaxObjectList {
+ exit = true
+ }
+ if exit {
+ break
+ }
+
+ err = rows.Err()
+ if err != nil {
+ log.Errorf("err:%v\n", err)
+ return
+ }
+ }
+
+ appendInfo.Prefixes = helper.Keys(commonPrefixes)
+
+ return
+}
+
+func (t *TidbClient) CountObjects(ctx context.Context, bucketName, prefix string) (*utils.ObjsCountInfo, error) {
+ var sqltext string
+ rsp := utils.ObjsCountInfo{}
+ var err error
+ var sizeStr sql.NullString
+ if prefix == "" {
+ sqltext = "select count(*),sum(size) from objects where bucketname=? and tier<?;"
+ err = t.Client.QueryRow(sqltext, bucketName, utils.Tier999).Scan(&rsp.Count, &sizeStr)
+ } else {
+ filt := prefix + "%"
+ sqltext = "select count(*),sum(size) from objects where bucketname=? and tier<? and name like ?;"
+ err = t.Client.QueryRow(sqltext, bucketName, utils.Tier999, filt).Scan(&rsp.Count, &sizeStr)
+ }
+
+ if err != nil {
+ log.Errorf("db error:%v\n", err)
+ return nil, err
+ }
+
+ if sizeStr.Valid {
+ size, err := strconv.ParseInt(sizeStr.String, 10, 64)
+ if err != nil {
+ log.Errorf("error:%v\n", err)
+ } else {
+ rsp.Size = size
+ }
+ }
+
+ return &rsp, err
+}
+
+func (t *TidbClient) DeleteBucket(ctx context.Context, bucket *Bucket) error {
+ sqltext := "delete from buckets where bucketname=?;"
+ _, err := t.Client.Exec(sqltext, bucket.Name)
+ if err != nil {
+ return handleDBError(err)
+ }
+ return nil
+}
+
+func (t *TidbClient) UpdateUsage(ctx context.Context, bucketName string, size int64, tx interface{}) (err error) {
+ var sqlTx *sql.Tx
+ if tx == nil {
+ tx, err = t.Client.Begin()
+
+ defer func() {
+ if err == nil {
+ err = sqlTx.Commit()
+ }
+ if err != nil {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+
+ sql := "update buckets set usages=? where bucketname=?;"
+ _, err = sqlTx.Exec(sql, size, bucketName)
+ if err != nil {
+ err = handleDBError(err)
+ }
+ return
+}
+
+func (t *TidbClient) UpdateUsages(ctx context.Context, usages map[string]int64, tx interface{}) error {
+ var sqlTx *sql.Tx
+ var err error
+ if nil == tx {
+ tx, err = t.Client.Begin()
+ defer func() {
+ if nil == err {
+ err = sqlTx.Commit()
+ } else {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+ sqlStr := "update buckets set usages = ? where bucketname = ?;"
+ st, err := sqlTx.Prepare(sqlStr)
+ if err != nil {
+ log.Error("failed to prepare statment with sql: ", sqlStr, ", err: ", err)
+ return ErrDBError
+ }
+ defer st.Close()
+
+ for bucket, usage := range usages {
+ _, err = st.Exec(usage, bucket)
+ if err != nil {
+ log.Error("failed to update usage for bucket: ", bucket, " with usage: ", usage, ", err: ", err)
+ return ErrDBError
+ }
+ }
+ return nil
+}
+
+func (t *TidbClient) ListBucketLifecycle(ctx context.Context) (buckets []*Bucket, err error) {
+ log.Infoln("list bucket lifecycle from tidb ...\n")
+ defer func() {
+ err = handleDBError(err)
+ }()
+
+ var rows *sql.Rows
+ sqltext := "select bucketname,lc from buckets where lc!=CAST('null' AS JSON);"
+ rows, err = t.Client.Query(sqltext)
+ if err == sql.ErrNoRows {
+ err = nil
+ return
+ } else if err != nil {
+ log.Errorf("db err:%v\n", err)
+ return
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ tmp := Bucket{Bucket: &pb.Bucket{}}
+ var lc string
+ err = rows.Scan(
+ &tmp.Name,
+ &lc)
+ if err != nil {
+ log.Errorf("db err:%v\n", err)
+ return
+ }
+
+ err = json.Unmarshal([]byte(lc), &tmp.LifecycleConfiguration)
+ if err != nil {
+ log.Errorf("db err:%v\n", err)
+ return
+ }
+
+ buckets = append(buckets, &tmp)
+ }
+
+ err = rows.Err()
+ if err != nil {
+ log.Errorf("db err:%v\n", err)
+ }
+
+ return
+}
+
+func (t *TidbClient) UpdateBucketVersioning(ctx context.Context, bucketName string, versionStatus string) error {
+ log.Infof("put bucket[%s] Version info[%s] into tidb ...\n", bucketName, versionStatus)
+
+ sql := "update bucket_versionopts set versionstatus=? where bucketname=?"
+ args := []interface{}{versionStatus, bucketName}
+
+ _, err := t.Client.Exec(sql, args...)
+ if err != nil {
+ log.Error("error in updating versioning information, err:%v\n", err)
+ return handleDBError(err)
+ }
+
+ return nil
+}
+
+func (t *TidbClient) CreateBucketVersioning(ctx context.Context, bucketName string, versionStatus string) error {
+ log.Infof("create bucket[%s] Version info[%s] into tidb ...\n", bucketName, versionStatus)
+
+ sql := "insert into bucket_versionopts(bucketname, versionstatus) values(?,?);"
+ args := []interface{}{bucketName, versionStatus}
+
+ _, err := t.Client.Exec(sql, args...)
+ if err != nil {
+ log.Error("error in creating versioning information, err:%v\n", err)
+ return handleDBError(err)
+ }
+
+ return nil
+}
+
+func (t *TidbClient) GetBucketVersioning(ctx context.Context, bucketName string) (versionOptsPtr *pb.BucketVersioning, err error) {
+ log.Info("list bucket Versions info from tidb ...")
+ /*m := bson.M{}
+ err = UpdateContextFilter(ctx, m)
+ if err != nil {
+ return nil, ErrInternalError
+ }*/
+
+ var rows *sql.Rows
+ sqltext := "select versionstatus from bucket_versionopts where bucketname=?;"
+
+ rows, err = t.Client.Query(sqltext, bucketName)
+
+ if err == sql.ErrNoRows {
+ err = nil
+ return
+ } else if err != nil {
+ log.Error("error in getting bucket versioning configuration, err:%v\n", err)
+ err = handleDBError(err)
+ return
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ tmp := &pb.BucketVersioning{}
+ err = rows.Scan(
+ &tmp.Status)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ return tmp, nil
+ }
+ return
+}
+
+func (t *TidbClient) CreateBucketSSE(ctx context.Context, bucketName string, sseType string, sseKey []byte) error {
+ log.Infof("create bucket[%s] SSE info[%s] into tidb ...\n", bucketName, sseType)
+
+ sql := "insert into bucket_sseopts(bucketname, sse, sseserverkey) values(?,?,?);"
+ args := []interface{}{bucketName, sseType, sseKey}
+
+ _, err := t.Client.Exec(sql, args...)
+ if err != nil {
+ return handleDBError(err)
+ }
+
+ return nil
+}
+
+func (t *TidbClient) GetBucketSSE(ctx context.Context, bucketName string) (sseOptsPtr *pb.ServerSideEncryption, err error) {
+ log.Info("list bucket SSE info from tidb ...")
+
+ var rows *sql.Rows
+ sqltext := "select sse,sseserverkey from bucket_sseopts where bucketname=?;"
+
+ rows, err = t.Client.Query(sqltext, bucketName)
+
+ if err == sql.ErrNoRows {
+ err = nil
+ return
+ } else if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ tmp := &pb.ServerSideEncryption{}
+
+ err = rows.Scan(
+ &tmp.SseType,
+ &tmp.EncryptionKey)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ rErr := rows.Err()
+ if rErr != nil {
+ rErr = handleDBError(rErr)
+ return
+ }
+ return tmp, nil
+ }
+ return
+}
+
+func (t *TidbClient) UpdateBucketSSE(ctx context.Context, bucketName string, sseType string, sseKey []byte) error {
+ log.Infof("put bucket[%s] SSE info[%s] into tidb ...\n", bucketName, sseType)
+
+ sql := "update bucket_sseopts set sse=?,sseserverkey=? where bucketname=?"
+ args := []interface{}{sseType, sseKey, bucketName}
+
+ _, err := t.Client.Exec(sql, args...)
+ if err != nil {
+ return handleDBError(err)
+ }
+
+ return nil
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package tidbclient
+
+import (
+ "context"
+ "database/sql"
+ "encoding/hex"
+ "encoding/json"
+ "math"
+ "os"
+ "strconv"
+ "strings"
+
+ _ "github.com/go-sql-driver/mysql"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/s3/error"
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ log "github.com/sirupsen/logrus"
+ "github.com/xxtea/xxtea-go/xxtea"
+)
+
+const MAX_OPEN_CONNS = 1024
+
+type TidbClient struct {
+ Client *sql.DB
+}
+
+func NewTidbClient(dbInfo string) *TidbClient {
+ cli := &TidbClient{}
+ conn, err := sql.Open("mysql", dbInfo)
+ if err != nil {
+ log.Errorf("connect to tidb failed, err:%v\n", err)
+ os.Exit(1)
+ }
+ log.Info("connected to tidb ...")
+ conn.SetMaxIdleConns(0)
+ conn.SetMaxOpenConns(MAX_OPEN_CONNS)
+ cli.Client = conn
+ return cli
+}
+
+func handleDBError(in error) (out error) {
+ if in == nil {
+ return nil
+ }
+
+ log.Errorf("db error:%v\n", in)
+ if in == sql.ErrNoRows {
+ out = ErrNoSuchKey
+ } else {
+ out = ErrDBError
+ }
+ return
+}
+
+func buildSql(ctx context.Context, filter map[string]string, sqltxt string) (string, []interface{}, error) {
+ const MaxObjectList = 10000
+ args := make([]interface{}, 0)
+
+ prefix := filter[common.KPrefix]
+ if prefix != "" {
+ sqltxt += " and name like ?"
+ args = append(args, prefix+"%")
+ log.Debug("query prefix:", prefix)
+ }
+ if filter[common.KMarker] != "" {
+ sqltxt += " and name >= ?"
+ args = append(args, filter[common.KMarker])
+ log.Debug("query marker:", filter[common.KMarker])
+ }
+
+ // lifecycle management may need to filter by LastModified
+ if filter[common.KLastModified] != "" {
+ var tmFilter map[string]string
+ err := json.Unmarshal([]byte(filter[common.KLastModified]), &tmFilter)
+ if err != nil {
+ log.Errorf("unmarshal lastmodified value failed:%s\n", err)
+ return sqltxt, args, ErrInternalError
+ }
+ log.Debugf("tmpFilter:%+v\n", tmFilter)
+
+ for k, v := range tmFilter {
+ var op string
+ switch k {
+ case "lt":
+ op = "<"
+ case "gt":
+ op = ">"
+ case "lte":
+ op = "<="
+ case "gte":
+ op = ">="
+ default:
+ log.Infof("unsupport filter action:%s\n", k)
+ return sqltxt, args, ErrInternalError
+ }
+
+ sqltxt += " and lastmodifiedtime " + op + " ?"
+ args = append(args, v)
+ }
+ }
+
+ // lifecycle management may need to filter by StorageTier
+ if filter[common.KStorageTier] != "" {
+ tier, err := strconv.Atoi(filter[common.KStorageTier])
+ if err != nil {
+ log.Errorf("invalid storage tier:%s\n", filter[common.KStorageTier])
+ return sqltxt, args, ErrInternalError
+ }
+
+ sqltxt += " and tier <= ?"
+ args = append(args, tier)
+ }
+
+ delimiter := filter[common.KDelimiter]
+ if delimiter == "" {
+ sqltxt += " order by bucketname,name,version limit ?"
+ args = append(args, MaxObjectList)
+ } else {
+ num := len(strings.Split(prefix, delimiter))
+ if prefix == "" {
+ num += 1
+ }
+
+ sqltxt += " group by SUBSTRING_INDEX(name, ?, ?) order by bucketname, name,version limit ?"
+ args = append(args, delimiter, num, MaxObjectList)
+ }
+
+ return sqltxt, args, nil
+}
+
+func VersionStr2UInt64(vers string) uint64 {
+ vidByte, _ := hex.DecodeString(vers)
+ decrByte := xxtea.Decrypt(vidByte, XXTEA_KEY)
+ reVersion, _ := strconv.ParseUint(string(decrByte), 10, 64)
+ version := math.MaxUint64 - reVersion
+
+ return version
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package tidbclient
+
+import (
+ "context"
+ "database/sql"
+ "math"
+ "strconv"
+ "time"
+
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (t *TidbClient) PutGcobjRecord(ctx context.Context, o *Object, tx interface{}) (err error) {
+ var sqlTx *sql.Tx
+ if tx == nil {
+ tx, err = t.Client.Begin()
+ defer func() {
+ if err == nil {
+ err = sqlTx.Commit()
+ }
+ if err != nil {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+
+ version := math.MaxUint64 - uint64(o.LastModified)
+ lastModifiedTime := time.Unix(o.LastModified, 0).Format(TIME_LAYOUT_TIDB)
+ sqltext := "insert into gcobjs (bucketname, name, version, location, tenantid, userid, size, objectid, " +
+ " lastmodifiedtime, storageMeta) values(?,?,?,?,?,?,?,?,?,?)"
+ args := []interface{}{o.BucketName, o.ObjectKey, version, o.Location, o.TenantId, o.UserId, o.Size, o.ObjectId,
+ lastModifiedTime, o.StorageMeta}
+ log.Debugf("sqltext:%s, args:%v\n", sqltext, args)
+ _, err = sqlTx.Exec(sqltext, args...)
+ log.Debugf("err:%v\n", err)
+
+ return err
+}
+
+func (t *TidbClient) DeleteGcobjRecord(ctx context.Context, o *Object, tx interface{}) (err error) {
+ var sqlTx *sql.Tx
+ if tx == nil {
+ tx, err = t.Client.Begin()
+ defer func() {
+ if err == nil {
+ err = sqlTx.Commit()
+ }
+ if err != nil {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+
+ sqltext := "delete from gcobjs where bucketname=? and name=? and version=?"
+ version, err := strconv.ParseUint(o.VersionId, 10, 64)
+ if err != nil {
+ log.Error("delete gc failed, err: ", err)
+ return err
+ }
+ args := []interface{}{o.BucketName, o.ObjectKey, version}
+ log.Debugf("sqltext:%s, args:%v\n", sqltext, args)
+ _, err = sqlTx.Exec(sqltext, args...)
+ log.Debugf("err:%v\n", err)
+
+ return err
+}
+
+func (t *TidbClient) ListGcObjs(ctx context.Context, offset, limit int) (objs []*Object, err error) {
+ sqltext := "select bucketname,name,version,location,objectid,storageMeta from gcobjs order by bucketname,name," +
+ "version limit ?,?;"
+ args := []interface{}{offset, limit}
+ log.Debugf("sqltext:%s, args:%v\n", sqltext, args)
+
+ rows, err := t.Client.Query(sqltext, args...)
+ if err != nil {
+ log.Errorf("err:%v\n", err)
+ return
+ }
+ defer rows.Close()
+
+ var iversion uint64
+ for rows.Next() {
+ obj := &Object{Object: &pb.Object{}}
+
+ err = rows.Scan(
+ &obj.BucketName,
+ &obj.ObjectKey,
+ &iversion,
+ &obj.Location,
+ &obj.ObjectId,
+ &obj.StorageMeta,
+ )
+ obj.VersionId = strconv.FormatUint(iversion, 10)
+ objs = append(objs, obj)
+ }
+
+ err = rows.Err()
+ if err != nil {
+ log.Errorf("err:%v\n", err)
+ }
+
+ return
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package tidbclient
+
+import (
+ "database/sql"
+ "encoding/json"
+ "strings"
+ "time"
+
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (t *TidbClient) CreateMultipart(multipart Multipart) (err error) {
+ m := multipart.Metadata
+ uploadTime := multipart.InitialTime.Format(TIME_LAYOUT_TIDB)
+ acl, _ := json.Marshal(m.Acl)
+ attrs, _ := json.Marshal(m.Attrs)
+ sqltext := "insert into multiparts(bucketname,objectname,uploadid,uploadtime,initiatorid,tenantid,userid,contenttype, " +
+ "location,acl,attrs,objectid,storageMeta,storageclass) " +
+ "values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+ _, err = t.Client.Exec(sqltext, multipart.BucketName, multipart.ObjectKey, multipart.UploadId, uploadTime,
+ m.InitiatorId, m.TenantId, m.UserId, m.ContentType,
+ m.Location, acl, attrs, multipart.ObjectId, multipart.StorageMeta, m.Tier)
+ return
+}
+
+func (t *TidbClient) GetMultipart(bucketName, objectName, uploadId string) (multipart Multipart, err error) {
+ sqltext := "select bucketname,objectname,uploadid,uploadtime,initiatorid,tenantid,userid,contenttype," +
+ "location,acl,attrs,objectid,storageMeta,storageclass from multiparts " +
+ "where bucketname=? and objectname=? and uploadid=?;"
+ var acl, attrs, uploadTime string
+ err = t.Client.QueryRow(sqltext, bucketName, objectName, uploadId).Scan(
+ &multipart.BucketName,
+ &multipart.ObjectKey,
+ &multipart.UploadId,
+ &uploadTime,
+ &multipart.Metadata.InitiatorId,
+ &multipart.Metadata.TenantId,
+ &multipart.Metadata.UserId,
+ &multipart.Metadata.ContentType,
+ &multipart.Metadata.Location,
+ &acl,
+ &attrs,
+ &multipart.ObjectId,
+ &multipart.StorageMeta,
+ &multipart.Metadata.Tier,
+ )
+ if err != nil && err == sql.ErrNoRows {
+ err = ErrNoSuchUpload
+ return
+ } else if err != nil {
+ return
+ }
+
+ ut, err := time.Parse(TIME_LAYOUT_TIDB, uploadTime)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ multipart.InitialTime = ut
+
+ err = json.Unmarshal([]byte(acl), &multipart.Metadata.Acl)
+ if err != nil {
+ return
+ }
+
+ err = json.Unmarshal([]byte(attrs), &multipart.Metadata.Attrs)
+ if err != nil {
+ return
+ }
+
+ sqltext = "select partnumber,size,objectid,offset,etag,lastmodified from objectparts " +
+ "where bucketname=? and objectname=? and uploadid=?;"
+ rows, err := t.Client.Query(sqltext, bucketName, objectName, uploadId)
+ if err != nil {
+ return
+ }
+ defer rows.Close()
+ multipart.Parts = make(map[int]*Part)
+ for rows.Next() {
+ p := &Part{}
+ err = rows.Scan(
+ &p.PartNumber,
+ &p.Size,
+ &p.ObjectId,
+ &p.Offset,
+ &p.Etag,
+ &p.LastModified,
+ )
+ ts, e := time.Parse(TIME_LAYOUT_TIDB, p.LastModified)
+ if e != nil {
+ return
+ }
+ p.LastModified = ts.Format(CREATE_TIME_LAYOUT)
+ multipart.Parts[p.PartNumber] = p
+ if err != nil {
+ return
+ }
+ }
+
+ return
+}
+
+func (t *TidbClient) DeleteMultipart(multipart *Multipart, tx interface{}) (err error) {
+ var sqlTx *sql.Tx
+ if tx == nil {
+ tx, err = t.Client.Begin()
+ defer func() {
+ if err == nil {
+ err = sqlTx.Commit()
+ }
+ if err != nil {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+
+ sqltext := "delete from multiparts where bucketname=? and objectname=? and uploadid=?;"
+ _, err = sqlTx.Exec(sqltext, multipart.BucketName, multipart.ObjectKey, multipart.UploadId)
+ if err != nil {
+ return
+ }
+
+ sqltext = "delete from objectparts where bucketname=? and objectname=? and uploadid=?;"
+ _, err = sqlTx.Exec(sqltext, multipart.BucketName, multipart.ObjectKey, multipart.UploadId)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (t *TidbClient) ListMultipartUploads(input *pb.ListBucketUploadRequest) (output *pb.ListBucketUploadResult, err error) {
+ var count int
+ var exit bool
+ bucketName := input.BucketName
+ keyMarker := input.KeyMarker
+ uploadIdMarker := input.UploadIdMarker
+ prefix := input.Prefix
+ delimiter := input.Delimiter
+ maxUploads := int(input.MaxUploads)
+
+ output = &pb.ListBucketUploadResult{}
+
+ log.Infof("bucketName:%v, keyMarker:%v, uploadIdMarker:%v,prefix:%v,delimiter:%v,maxUploads:%v",
+ bucketName, keyMarker, uploadIdMarker, prefix, delimiter, maxUploads)
+
+ commonPrefixes := make(map[string]struct{})
+ objectMap := make(map[string]struct{})
+ uploadIdMap := make(map[string]struct{})
+ currentMarker := keyMarker
+ currentUploadIdMarker := uploadIdMarker
+ for !exit {
+ var loopnum int
+ var sqltext string
+ var rows *sql.Rows
+ args := make([]interface{}, 0)
+ sqltext = "select objectname,uploadid,uploadtime,initiatorid,tenantid,userid from multiparts where bucketName=?"
+ args = append(args, bucketName)
+ if prefix != "" {
+ sqltext += " and objectname like ?"
+ args = append(args, prefix+"%")
+ log.Infof("query prefix: %s", prefix)
+ }
+ if currentMarker != "" {
+ sqltext += " and objectname >= ?"
+ args = append(args, currentMarker)
+ log.Infof("query object name marker: %s", currentMarker)
+ }
+ if uploadIdMarker != "" {
+ sqltext += " and uploadid >= ?"
+ args = append(args, uploadIdMarker)
+ log.Infof("query uplaodid marker: %s", uploadIdMarker)
+ }
+ if delimiter == "" {
+ sqltext += " order by bucketname,objectname,uploadid limit ?"
+ args = append(args, maxUploads)
+ } else {
+ num := len(strings.Split(prefix, delimiter))
+ args = append(args, delimiter, num, maxUploads)
+ sqltext += " group by SUBSTRING_INDEX(name, ?, ?) order by bucketname,objectname,uploadid limit ?"
+ }
+ rows, err = t.Client.Query(sqltext, args...)
+ if err != nil {
+ err = handleDBError(err)
+ return
+ }
+ defer rows.Close()
+ for rows.Next() {
+ // loopnum is used to calculate the total number of the query result
+ loopnum += 1
+ var objName, uploadId, initiatorId, tenantId, uploadTime, userId string
+ err = rows.Scan(
+ &objName,
+ &uploadId,
+ &uploadTime,
+ &initiatorId,
+ &tenantId,
+ &userId,
+ )
+ if err != nil {
+ err = handleDBError(err)
+ log.Errorln("sql err:", err)
+ return
+ }
+ log.Infoln("objectName:", objName, " uploadId:", uploadId)
+ // if update marker for next query
+ currentMarker = objName
+ currentUploadIdMarker = uploadId
+
+ // if the object name and upload id have not been processed, we use objectMap and uploadIdMap to flag them
+ var objExist, uploadIdExist bool
+ if _, objExist := objectMap[objName]; !objExist {
+ objectMap[objName] = struct{}{}
+ }
+ if _, uploadIdExist := uploadIdMap[uploadId]; !uploadIdExist {
+ uploadIdMap[uploadId] = struct{}{}
+ }
+ // if the same record row is processed before, we should skip to next row
+ if objExist && uploadIdExist {
+ continue
+ }
+
+ if currentMarker == keyMarker && currentUploadIdMarker == uploadIdMarker {
+ // because we search result from maker and the result should not include origin marker from client.
+ continue
+ }
+
+ // filter by delimiter
+ if len(delimiter) != 0 {
+ subStr := strings.TrimPrefix(objName, prefix)
+ n := strings.Index(subStr, delimiter)
+ if n != -1 {
+ prefixKey := prefix + string([]byte(subStr)[0:(n+1)])
+ // the example is from https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html
+ // some multipart object key as follows:
+ // photos/2006/January/sample.jpg
+ // photos/2006/February/sample.jpg
+ // photos/2006/March/sample.jpg
+ // videos/2006/March/sample.wmv
+ // sample.jpg
+ // we get "photos" as prefix, and "photos0" as marker for next query. The value of character '0' is
+ // next to the value of "/", and next query we should use it as marker
+ currentMarker = prefixKey[0:(len(prefixKey)-1)] + string(delimiter[len(delimiter)-1]+1)
+ if prefixKey == keyMarker {
+ // because we search result from maker and the result should not include origin marker from client.
+ continue
+ }
+ if _, ok := commonPrefixes[prefixKey]; !ok {
+ if count == maxUploads {
+ output.IsTruncated = true
+ exit = true
+ break
+ }
+ commonPrefixes[prefixKey] = struct{}{}
+ output.NextKeyMarker = objName
+ output.NextUploadIdMarker = uploadId
+ count += 1
+ }
+ continue
+ }
+ }
+ // now start to get out multipart upload records, and update nextmarker if necessary
+ count += 1
+ if count == maxUploads {
+ output.NextKeyMarker = objName
+ output.NextUploadIdMarker = uploadId
+ }
+ // if count are more than maxUploads, it means that we should exit the loop and set IsTruncated true
+ if count > maxUploads {
+ output.IsTruncated = true
+ exit = true
+ break
+ }
+
+ var ct time.Time
+ ct, err = time.Parse(TIME_LAYOUT_TIDB, uploadTime)
+ if err != nil {
+ return
+ }
+ output.Uploads = append(output.Uploads, &pb.Upload{
+ Key: objName,
+ UploadId: uploadId,
+ Initiator: &pb.Owner{
+ Id: tenantId,
+ DisplayName: tenantId,
+ },
+ Owner: &pb.Owner{
+ Id: initiatorId,
+ DisplayName: initiatorId,
+ },
+ Initiated: ct.Format(CREATE_TIME_LAYOUT),
+ StorageClass: "STANDARD",
+ })
+ }
+
+ // if the number of query result are less maxUploads, it must be no more result, and we can exit the loop
+ if loopnum < maxUploads {
+ exit = true
+ }
+ }
+ output.CommonPrefix = helper.Keys(commonPrefixes)
+ return
+}
+
+func (t *TidbClient) PutObjectPart(multipart *Multipart, part *Part, tx interface{}) (err error) {
+ var sqlTx *sql.Tx
+ if tx == nil {
+ tx, err = t.Client.Begin()
+ defer func() {
+ if err == nil {
+ err = sqlTx.Commit()
+ }
+ if err != nil {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+
+ lastt, err := time.Parse(CREATE_TIME_LAYOUT, part.LastModified)
+ if err != nil {
+ return
+ }
+ lastModified := lastt.Format(TIME_LAYOUT_TIDB)
+ sqltext := "insert into objectparts(bucketname,objectname,uploadid,partnumber,size,objectid,offset,etag,lastmodified) " +
+ "values(?,?,?,?,?,?,?,?,?)"
+ _, err = sqlTx.Exec(sqltext, multipart.BucketName, multipart.ObjectKey, multipart.UploadId, part.PartNumber, part.Size,
+ part.ObjectId, part.Offset, part.Etag, lastModified)
+ return
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package tidbclient
+
+import (
+ "context"
+ "database/sql"
+ "encoding/hex"
+ "encoding/json"
+ "math"
+ "strconv"
+ "time"
+
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+ "github.com/xxtea/xxtea-go/xxtea"
+)
+
+func (t *TidbClient) GetObject(ctx context.Context, bucketName, objectName, version string) (object *Object, err error) {
+ var sqltext, ibucketname, iname, customattributes, acl, lastModified string
+ var iversion uint64
+ var row *sql.Row
+ if version == "" {
+ sqltext = "select bucketname,name,version,location,tenantid,userid,size,objectid,lastmodifiedtime,etag," +
+ "contenttype,customattributes,acl,nullversion,deletemarker,ssetype,encryptionkey,initializationvector,type,tier,storageMeta,encsize" +
+ " from objects where bucketname=? and name=? order by bucketname,name,version limit 1;"
+ row = t.Client.QueryRow(sqltext, bucketName, objectName)
+ } else {
+ sqltext = "select bucketname,name,version,location,tenantid,userid,size,objectid,lastmodifiedtime,etag," +
+ "contenttype,customattributes,acl,nullversion,deletemarker,ssetype,encryptionkey,initializationvector,type,tier,storageMeta,encsize" +
+ " from objects where bucketname=? and name=? and version=?;"
+ row = t.Client.QueryRow(sqltext, bucketName, objectName, version)
+ }
+ log.Infof("sqltext:%s, bucketName=%s, objectName=%s, version:%s\n", sqltext, bucketName, objectName, version)
+ object = &Object{Object: &pb.Object{ServerSideEncryption: &pb.ServerSideEncryption{}}}
+ err = row.Scan(
+ &ibucketname,
+ &iname,
+ &iversion,
+ &object.Location,
+ &object.TenantId,
+ &object.UserId,
+ &object.Size,
+ &object.ObjectId,
+ &lastModified,
+ &object.Etag,
+ &object.ContentType,
+ &customattributes,
+ &acl,
+ &object.NullVersion,
+ &object.DeleteMarker,
+ &object.ServerSideEncryption.SseType,
+ &object.ServerSideEncryption.EncryptionKey,
+ &object.ServerSideEncryption.InitilizationVector,
+ &object.Type,
+ &object.Tier,
+ &object.StorageMeta,
+ &object.EncSize,
+ )
+ if err != nil {
+ log.Errorf("err: %v\n", err)
+ err = handleDBError(err)
+ return
+ }
+
+ object.ObjectKey = objectName
+ object.BucketName = bucketName
+ lastModifiedTime, _ := time.ParseInLocation(TIME_LAYOUT_TIDB, lastModified, time.Local)
+ object.LastModified = lastModifiedTime.Unix()
+
+ err = json.Unmarshal([]byte(acl), &object.Acl)
+ if err != nil {
+ return
+ }
+ err = json.Unmarshal([]byte(customattributes), &object.CustomAttributes)
+ if err != nil {
+ return
+ }
+ // TODO: getting multi-parts
+ timestamp := math.MaxUint64 - iversion
+ timeData := []byte(strconv.FormatUint(timestamp, 10))
+ object.VersionId = hex.EncodeToString(xxtea.Encrypt(timeData, XXTEA_KEY))
+ return
+}
+
+func (t *TidbClient) UpdateObjectMeta(object *Object) error {
+ sql, args := object.GetUpdateMetaSql()
+ _, err := t.Client.Exec(sql, args...)
+ return err
+}
+
+func (t *TidbClient) PutObject(ctx context.Context, object *Object, tx interface{}) (err error) {
+ var sqlTx *sql.Tx
+ if tx == nil {
+ tx, err = t.Client.Begin()
+ defer func() {
+ if err == nil {
+ err = sqlTx.Commit()
+ }
+ if err != nil {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+ sql, args := object.GetCreateSql()
+ _, err = sqlTx.Exec(sql, args...)
+ // TODO: multi-part handle, see issue https://github.com/opensds/multi-cloud/issues/690
+
+ return err
+}
+
+func (t *TidbClient) DeleteObject(ctx context.Context, object *Object, tx interface{}) (err error) {
+ var sqlTx *sql.Tx
+ if tx == nil {
+ tx, err = t.Client.Begin()
+ defer func() {
+ if err == nil {
+ err = sqlTx.Commit()
+ }
+ if err != nil {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+
+ version := VersionStr2UInt64(object.VersionId)
+ log.Infof("delete from objects where name=%s and bucketname=%s and version=%d;\n",
+ object.ObjectKey, object.BucketName, version)
+
+ sqltext := "delete from objects where name=? and bucketname=? and version=?;"
+ _, err = sqlTx.Exec(sqltext, object.ObjectKey, object.BucketName, version)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (t *TidbClient) SetObjectDeleteMarker(ctx context.Context, object *Object, deleteMarker bool) error {
+ version := VersionStr2UInt64(object.VersionId)
+
+ sqltext := "update objects set deletemarker=? where bucketname=? and name=? and version=?;"
+ _, err := t.Client.Exec(sqltext, deleteMarker, object.BucketName, object.ObjectKey, version)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (t *TidbClient) UpdateObject4Lifecycle(ctx context.Context, old, new *Object, tx interface{}) (err error) {
+ var sqlTx *sql.Tx
+ if tx == nil {
+ tx, err = t.Client.Begin()
+ defer func() {
+ if err == nil {
+ err = sqlTx.Commit()
+ }
+ if err != nil {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+
+ oldversion := VersionStr2UInt64(old.VersionId)
+
+ sqltext := "update objects set location=?,objectid=?,tier=?,storageMeta=? where bucketname=? and name=? and version=?"
+ args := []interface{}{new.Location, new.ObjectId, new.Tier, new.StorageMeta, old.BucketName, old.ObjectKey, oldversion}
+
+ log.Debugf("sqltext:%s, args:%+v\n", sqltext, args)
+ _, err = sqlTx.Exec(sqltext, args...)
+
+ log.Debugf("err:%v\n", err)
+ return err
+}
+
+
+
package tidbclient
+
+import (
+ "database/sql"
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ "strconv"
+)
+
+//objmap
+func (t *TidbClient) GetObjectMap(bucketName, objectName string) (objMap *ObjMap, err error) {
+ objMap = &ObjMap{}
+ sqltext := "select bucketname,objectname,nullvernum from objmap where bucketname=? and objectName=?;"
+ err = t.Client.QueryRow(sqltext, bucketName, objectName).Scan(
+ &objMap.BucketName,
+ &objMap.Name,
+ &objMap.NullVerNum,
+ )
+ if err != nil {
+ return
+ }
+ objMap.NullVerId = strconv.FormatUint(objMap.NullVerNum, 10)
+ return
+}
+
+func (t *TidbClient) PutObjectMap(objMap *ObjMap, tx interface{}) (err error) {
+ var sqlTx *sql.Tx
+ if tx == nil {
+ tx, err = t.Client.Begin()
+ defer func() {
+ if err == nil {
+ err = sqlTx.Commit()
+ }
+ if err != nil {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+
+ sqltext := "insert into objmap(bucketname,objectname,nullvernum) values(?,?,?);"
+ _, err = sqlTx.Exec(sqltext, objMap.BucketName, objMap.Name, objMap.NullVerNum)
+ return err
+}
+
+func (t *TidbClient) DeleteObjectMap(objMap *ObjMap, tx interface{}) (err error) {
+ var sqlTx *sql.Tx
+ if tx == nil {
+ tx, err = t.Client.Begin()
+ defer func() {
+ if err == nil {
+ err = sqlTx.Commit()
+ }
+ if err != nil {
+ sqlTx.Rollback()
+ }
+ }()
+ }
+ sqlTx, _ = tx.(*sql.Tx)
+ sqltext := "delete from objmap where bucketname=? and objectname=?;"
+ _, err = sqlTx.Exec(sqltext, objMap.BucketName, objMap.Name)
+ return err
+}
+
+
+
package tidbclient
+
+import "database/sql"
+
+func (t *TidbClient) NewTrans()(tx interface{}, err error) {
+ tx, err = t.Client.Begin()
+ return
+}
+
+func (t *TidbClient) AbortTrans(tx interface{}) (err error) {
+ err = tx.(* sql.Tx).Rollback()
+ return
+}
+
+func (t *TidbClient) CommitTrans(tx interface{}) (err error) {
+ err = tx.(* sql.Tx).Commit()
+ return
+}
+
+
package meta
+
+import (
+ "context"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+)
+
+func (m *Meta) AddGcobjRecord(ctx context.Context, obj *types.Object) error {
+ return m.Db.PutGcobjRecord(ctx, obj, nil)
+}
+
+func (m *Meta) DeleteGcobjRecord(ctx context.Context, obj *types.Object) error {
+ return m.Db.DeleteGcobjRecord(ctx, obj, nil)
+}
+
+func (m *Meta) ListGcObjs(ctx context.Context, offset, limit int) ([]*types.Object, error) {
+ return m.Db.ListGcObjs(ctx, offset, limit)
+}
+
+
+
package meta
+
+import (
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/log"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/db"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/db/drivers/tidb"
+)
+
+const (
+ ENCRYPTION_KEY_LENGTH = 32 // 32 bytes for AES-"256"
+)
+
+type MetaConfig struct {
+ CacheType CacheType
+ TidbInfo string
+}
+
+type Meta struct {
+ Db db.DBAdapter
+ Logger *log.Logger
+ Cache MetaCache
+}
+
+func (m *Meta) Stop() {
+ if m.Cache != nil {
+ m.Cache.Close()
+ }
+}
+
+func New(cfg MetaConfig) *Meta {
+ meta := Meta{
+ Cache: newMetaCache(cfg.CacheType),
+ }
+ meta.Db = tidbclient.NewTidbClient(cfg.TidbInfo)
+ return &meta
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package meta
+
+import (
+ "context"
+
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+)
+
+func (m *Meta) GetMultipart(bucketName, objectName, uploadId string) (Multipart, error) {
+ return m.Db.GetMultipart(bucketName, objectName, uploadId)
+}
+
+func (m *Meta) DeleteMultipart(ctx context.Context, multipart Multipart) (err error) {
+ tx, err := m.Db.NewTrans()
+ defer func() {
+ if err != nil {
+ m.Db.AbortTrans(tx)
+ }
+ }()
+ err = m.Db.DeleteMultipart(&multipart, tx)
+ if err != nil {
+ return
+ }
+
+ // TODO: usage need to be updated for charging, it depends on redis, and the mechanism is:
+ // 1. Update usage in redis when each delete happens.
+ // 2. Update usage in database periodically based on redis.
+ // see https://github.com/opensds/multi-cloud/issues/698 for redis related issue.
+
+ err = m.Db.CommitTrans(tx)
+ return
+}
+
+func (m *Meta) PutObjectPart(ctx context.Context, multipart Multipart, part Part) (err error) {
+ tx, err := m.Db.NewTrans()
+ defer func() {
+ if err != nil {
+ m.Db.AbortTrans(tx)
+ }
+ }()
+ err = m.Db.PutObjectPart(&multipart, &part, tx)
+ if err != nil {
+ return
+ }
+ //TODO: update bucket size
+ err = m.Db.CommitTrans(tx)
+ return
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package meta
+
+import (
+ "context"
+
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/db/drivers/tidb"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/redis"
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ log "github.com/sirupsen/logrus"
+ "strconv"
+)
+
+const (
+ OBJECT_CACHE_PREFIX = "object:"
+)
+
+// Object will be updated to cache while willNeed is true
+func (m *Meta) GetObject(ctx context.Context, bucketName string, objectName string, versionId string, willNeed bool) (object *Object, err error) {
+ getObject := func() (o helper.Serializable, err error) {
+ log.Info("GetObject CacheMiss. bucket:", bucketName, ", object:", objectName)
+ version := ""
+ if versionId != "" {
+ version = strconv.FormatUint(tidbclient.VersionStr2UInt64(versionId), 10)
+ }
+ object, err := m.Db.GetObject(ctx, bucketName, objectName, version)
+ if err != nil {
+ log.Errorln("get object failed, err:", err)
+ return
+ }
+ log.Infoln("GetObject object.Name:", objectName)
+ if object.ObjectKey != objectName {
+ err = ErrNoSuchKey
+ return
+ }
+ return object, nil
+ }
+
+ toObject := func(fields map[string]string) (interface{}, error) {
+ o := &Object{}
+ return o.Deserialize(fields)
+ }
+
+ o, err := m.Cache.Get(redis.ObjectTable, OBJECT_CACHE_PREFIX, bucketName+":"+objectName+":",
+ getObject, toObject, willNeed)
+ if err != nil {
+ return
+ }
+ object, ok := o.(*Object)
+ if !ok {
+ err = ErrInternalError
+ return
+ }
+ return object, nil
+}
+
+func (m *Meta) PutObject(ctx context.Context, object, deleteObj *Object, multipart *Multipart, objMap *ObjMap, updateUsage bool) error {
+ log.Debugf("PutObject begin, object=%+v, deleteObj:%+v\n", object, deleteObj)
+ tx, err := m.Db.NewTrans()
+ defer func() {
+ if err != nil {
+ m.Db.AbortTrans(tx)
+ }
+ }()
+
+ // if target object exist and it's location is different from new location, need to clean it
+ if deleteObj != nil {
+ if deleteObj.Location != object.Location {
+ log.Infoln("put gc, deleteObj:", deleteObj)
+ err = m.Db.PutGcobjRecord(ctx, deleteObj, tx)
+ if err != nil {
+ return err
+ }
+ }
+
+ log.Infoln("delete object metadata, deleteObj:", deleteObj)
+ err = m.Db.DeleteObject(ctx, deleteObj, tx)
+ if err != nil {
+ return err
+ }
+ }
+
+ err = m.Db.PutObject(ctx, object, tx)
+ if err != nil {
+ return err
+ }
+
+ if multipart != nil {
+ err = m.Db.DeleteMultipart(multipart, tx)
+ if err != nil {
+ return err
+ }
+ }
+
+ // TODO: usage need to be updated for charging, and it depends on redis, and the mechanism is:
+ // 1. Update usage in redis when each put happens.
+ // 2. Update usage in database periodically based on redis.
+ // see https://github.com/opensds/multi-cloud/issues/698 for redis related issue.
+
+ err = m.Db.CommitTrans(tx)
+ return nil
+}
+
+func (m *Meta) UpdateObjectMeta(object *Object) error {
+ err := m.Db.UpdateObjectMeta(object)
+ return err
+}
+
+func (m *Meta) DeleteObject(ctx context.Context, object *Object) error {
+ tx, err := m.Db.NewTrans()
+ defer func() {
+ if err != nil {
+ m.Db.AbortTrans(tx)
+ }
+ }()
+
+ err = m.Db.DeleteObject(ctx, object, tx)
+ if err != nil {
+ return err
+ }
+
+ // TODO: usage need to be updated for charging, it depends on redis, and the mechanism is:
+ // 1. Update usage in redis when each delete happens.
+ // 2. Update usage in database periodically based on redis.
+ // see https://github.com/opensds/multi-cloud/issues/698 for redis related issue.
+
+ err = m.Db.CommitTrans(tx)
+
+ return err
+}
+
+func (m *Meta) MarkObjectAsDeleted(ctx context.Context, object *Object) error {
+ return m.Db.SetObjectDeleteMarker(ctx, object, true)
+}
+
+func (m *Meta) UpdateObject4Lifecycle(ctx context.Context, old, new *Object, multipart *Multipart) (err error) {
+ log.Infof("update object from %v to %v\n", *old, *new)
+ tx, err := m.Db.NewTrans()
+ defer func() {
+ if err != nil {
+ m.Db.AbortTrans(tx)
+ }
+ }()
+
+ err = m.Db.UpdateObject4Lifecycle(ctx, old, new, tx)
+ if err != nil {
+ return err
+ }
+
+ if multipart != nil {
+ err = m.Db.DeleteMultipart(multipart, tx)
+ if err != nil {
+ return err
+ }
+ }
+
+ err = m.Db.CommitTrans(tx)
+ return err
+}
+
+
+
package redis
+
+import (
+ "strconv"
+
+ log "github.com/sirupsen/logrus"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/config"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+)
+
+var (
+ redisClient *RedisCli
+)
+
+const InvalidQueueName = "InvalidQueue"
+
+const keyvalue = "000102030405060708090A0B0C0D0E0FF0E0D0C0B0A090807060504030201000" // This is the key for hash sum !
+
+type RedisDatabase int
+
+func (r RedisDatabase) String() string {
+ return strconv.Itoa(int(r))
+}
+
+func (r RedisDatabase) InvalidQueue() string {
+ return InvalidQueueName + r.String()
+}
+
+const (
+ UserTable RedisDatabase = iota
+ BucketTable
+ ObjectTable
+ FileTable
+ ClusterTable
+)
+
+var MetadataTables = []RedisDatabase{UserTable, BucketTable, ObjectTable, ClusterTable}
+var DataTables = []RedisDatabase{FileTable}
+
+func Initialize(cfg *config.CacheConfig) {
+ redisClient = NewRedisCli()
+ redisClient.Init(cfg)
+}
+
+func Close() {
+ if redisClient != nil && redisClient.IsValid() {
+ err := redisClient.Close()
+ if err != nil {
+ log.Errorf("Cannot close redis pool, err: %v", err)
+ }
+ }
+}
+
+func GetClient() (*RedisCli, error) {
+ return redisClient, nil
+}
+
+func HasRedisClient() bool {
+ if redisClient != nil && redisClient.IsValid() {
+ return true
+ }
+
+ return false
+}
+
+func Remove(table RedisDatabase, prefix, key string) (err error) {
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return err
+ }
+
+ // Use table.String() + hashkey as Redis key
+ _, err = c.Del(table.String() + prefix + helper.EscapeColon(key))
+ if err != nil {
+ log.Errorf("failed to call redis del for (%s), err: %v", table.String()+prefix+helper.EscapeColon(key), err)
+ return err
+ }
+
+ log.Infof("Cmd: %s. Key: %s.", "DEL", table.String()+key)
+ return nil
+}
+
+func Set(table RedisDatabase, prefix, key string, value interface{}) (err error) {
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return err
+ }
+ encodedValue, err := helper.MsgPackMarshal(value)
+ if err != nil {
+ log.Errorf("failed to make pack for(%s, %s, %v), err: %v", prefix, key, value, err)
+ return err
+ }
+
+ // Use table.String() + hashkey as Redis key. Set expire time to 30s.
+ r, err := c.Set(table.String()+prefix+helper.EscapeColon(key), encodedValue, 30000)
+ if err != nil {
+ log.Errorf("failed to call redis Set(%s, %v), err: %v", table.String()+prefix+helper.EscapeColon(key), string(encodedValue), err)
+ return err
+ }
+ log.Infof("Cmd: %s. Key: %s. Value: %s. Reply: %s.", "SET", table.String()+key, string(encodedValue), r)
+ return nil
+}
+
+func Get(table RedisDatabase, prefix, key string) (value interface{}, err error) {
+ var encodedValue []byte
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return nil, err
+ }
+
+ // Use table.String() + hashkey as Redis key
+ encodedValue, err = c.Get(table.String() + prefix + helper.EscapeColon(key))
+ if err != nil {
+ log.Errorf("failed to call redis Get(%s), err: %v", table.String()+prefix+helper.EscapeColon(key), err)
+ return nil, err
+ }
+ if len(encodedValue) == 0 {
+ return nil, nil
+ }
+ err = helper.MsgPackUnMarshal(encodedValue, value)
+ return value, err
+}
+
+// don't use the escapecolon in keys command.
+func Keys(table RedisDatabase, pattern string) ([]string, error) {
+ var keys []string
+ query := table.String() + pattern
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return nil, err
+ }
+
+ keys, err = c.Keys(query)
+ if err != nil {
+ log.Errorf("failed to call redis Keys(%s), err: %s", query, err)
+ return nil, err
+ }
+
+ return keys, nil
+
+}
+
+func MGet(table RedisDatabase, prefix string, keys []string) ([]interface{}, error) {
+ var results []interface{}
+ var queryKeys []string
+ for _, key := range keys {
+ queryKeys = append(queryKeys, table.String()+prefix+helper.EscapeColon(key))
+ }
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return nil, err
+ }
+
+ results, err = c.MGet(queryKeys)
+ if err != nil {
+ log.Errorf("failed to call redis MGet(%v), err: %v", queryKeys, err)
+ return nil, err
+ }
+ return results, nil
+}
+
+func MSet(table RedisDatabase, prefix string, pairs map[string]interface{}) (string, error) {
+ var result string
+ tmpPairs := make(map[interface{}]interface{})
+ for k, v := range pairs {
+ tmpPairs[table.String()+prefix+helper.EscapeColon(k)] = v
+ }
+
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return "", err
+ }
+
+ result, err = c.MSet(tmpPairs)
+ if err != nil {
+ log.Errorf("failed to call redis MSet(%s, %v), err: %v", prefix, pairs, err)
+ return "", err
+ }
+
+ return result, nil
+
+}
+
+func IncrBy(table RedisDatabase, prefix, key string, value int64) (int64, error) {
+ var result int64
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return 0, err
+ }
+
+ result, err = c.IncrBy(prefix+helper.EscapeColon(key), value)
+ if err != nil {
+ log.Errorf("failed to call redis IncrBy(%s, %d), err: %v", prefix+helper.EscapeColon(key), value, err)
+ return 0, err
+ }
+ return result, nil
+}
+
+// Get file bytes
+// `start` and `end` are inclusive
+// FIXME: this API causes an extra memory copy, need to patch radix to fix it
+func GetBytes(key string, start int64, end int64) ([]byte, error) {
+ var value []byte
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return nil, err
+ }
+
+ // Use table.String() + hashkey as Redis key
+ value, err = c.GetRange(FileTable.String()+helper.EscapeColon(key), start, end)
+ if err != nil {
+ log.Errorf("failed to call redis GetRange(%s, %d, %d), err: %v", FileTable.String()+helper.EscapeColon(key), start, end, err)
+ return nil, err
+ }
+ return value, nil
+}
+
+// Set file bytes
+func SetBytes(key string, value []byte) (err error) {
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return err
+ }
+
+ // Use table.String() + hashkey as Redis key
+ _, err = c.Set(FileTable.String()+helper.EscapeColon(key), value, 0)
+ if err != nil {
+ log.Errorf("failed to call redis set(%s), err: %d", FileTable.String()+helper.EscapeColon(key), err)
+ return err
+ }
+ return nil
+}
+
+func HSet(table RedisDatabase, prefix, key, field string, value interface{}) (bool, error) {
+ var r bool
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return false, err
+ }
+
+ r, err = c.HSet(table.String()+prefix+helper.EscapeColon(key), field, value)
+ if err != nil {
+ log.Errorf("failed to call redis HSet(%s, %s), err: %v", table.String()+prefix+helper.EscapeColon(key), field, err)
+ return false, err
+ }
+ return r, nil
+}
+
+func HGet(table RedisDatabase, prefix, key, field string) (string, error) {
+ var r string
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return "", err
+ }
+
+ r, err = c.HGet(table.String()+prefix+helper.EscapeColon(key), field)
+ if err != nil {
+ log.Errorf("failed to call redis HGet(%s, %s), err: %v", table.String()+prefix+helper.EscapeColon(key), field)
+ return "", err
+ }
+ return r, nil
+}
+
+func HDel(table RedisDatabase, prefix, key string, fields []string) (int64, error) {
+ var r int64
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return 0, err
+ }
+
+ r, err = c.HDel(table.String()+prefix+helper.EscapeColon(key), fields)
+ if err != nil {
+ log.Errorf("failed to call redis HDel(%s, %v), err: %v", table.String()+prefix+helper.EscapeColon(key), fields, err)
+ return 0, err
+ }
+ return r, nil
+}
+
+func HGetInt64(table RedisDatabase, prefix, key, field string) (int64, error) {
+ var r int64
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return 0, err
+ }
+
+ theKey := table.String() + prefix + helper.EscapeColon(key)
+ r, err = c.HGetInt64(theKey, field)
+ if err != nil {
+ log.Errorf("failed to call redis HGetInt64(%s, %s), err: %v", theKey, field, err)
+ return 0, err
+ }
+ return r, nil
+}
+
+func HGetAll(table RedisDatabase, prefix, key string) (map[string]string, error) {
+ var r map[string]string
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return nil, err
+ }
+
+ r, err = c.HGetAll(table.String() + prefix + helper.EscapeColon(key))
+ if err != nil {
+ log.Errorf("failed to call redis HGetAll(%s), err: %v", table.String()+prefix+helper.EscapeColon(key), err)
+ return nil, err
+ }
+ return r, nil
+}
+
+func HIncrBy(table RedisDatabase, prefix, key, field string, incr int64) (int64, error) {
+ var r int64
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return 0, err
+ }
+
+ r, err = c.HIncrBy(table.String()+prefix+helper.EscapeColon(key), field, incr)
+ if err != nil {
+ log.Errorf("failed to call redis HIncrBy(%s, %s, %d), err: %v", table.String()+prefix+helper.EscapeColon(key), field, incr)
+ return 0, err
+ }
+ return r, nil
+}
+
+func HMSet(table RedisDatabase, prefix, key string, fields map[string]interface{}) (string, error) {
+ var r string
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return "", err
+ }
+
+ r, err = c.HMSet(table.String()+prefix+helper.EscapeColon(key), fields)
+ if err != nil {
+ log.Errorf("failed to call redis HMSet(%s, %v), err: %v", table.String()+prefix+helper.EscapeColon(key), fields, err)
+ return "", err
+ }
+ return r, nil
+}
+
+func HMGet(table RedisDatabase, prefix, key string, fields []string) (map[string]interface{}, error) {
+ var r map[string]interface{}
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return nil, err
+ }
+
+ r, err = c.HMGet(table.String()+prefix+helper.EscapeColon(key), fields)
+ if err != nil {
+ log.Errorf("failed to call redis HMGet(%s, %v), err: %v", table.String()+prefix+helper.EscapeColon(key), fields, err)
+ return nil, err
+ }
+ return r, nil
+}
+
+// Publish the invalid message to other YIG instances through Redis
+func Invalid(table RedisDatabase, key string) (err error) {
+ c, err := GetClient()
+ if err != nil {
+ log.Errorf("failed to get redis client, err: %v", err)
+ return err
+ }
+
+ // Use table.String() + hashkey as Redis key
+ _, err = c.Publish(table.InvalidQueue(), helper.EscapeColon(key))
+ if err != nil {
+ log.Errorf("failed to call redis Public(%s), err: %v", helper.EscapeColon(key), err)
+ return err
+ }
+ return nil
+}
+
+
+
package redis
+
+import (
+ "errors"
+ "time"
+
+ log "github.com/sirupsen/logrus"
+ "github.com/go-redis/redis"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/yig/config"
+)
+
+const (
+ REDIS_UNKNOWN_CLIENT = iota
+ REDIS_NORMAL_CLIENT
+ REDIS_CLUSTER_CLIENT
+ REDIS_SENTINEL_CLIENT
+)
+
+const ERR_NOT_INIT_MSG = "redis client is not initialized yet."
+
+type RedisCli struct {
+ clientType int
+ redisClient *redis.Client
+ redisClusterClient *redis.ClusterClient
+}
+
+func NewRedisCli() *RedisCli {
+ return &RedisCli{
+ clientType: REDIS_UNKNOWN_CLIENT,
+ }
+}
+
+func (cli *RedisCli) Init(cfg *config.CacheConfig) {
+ switch cfg.Mode {
+ case 1:
+ options := &redis.ClusterOptions{
+ Addrs: cfg.Nodes,
+ ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
+ DialTimeout: time.Duration(cfg.ConnectionTimeout) * time.Second,
+ WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
+ IdleTimeout: time.Duration(cfg.KeepAlive) * time.Second,
+ }
+ if cfg.PoolMaxIdle > 0 {
+ options.PoolSize = cfg.PoolMaxIdle
+ }
+ if cfg.Password != "" {
+ options.Password = cfg.Password
+ }
+ cli.redisClusterClient = redis.NewClusterClient(options)
+ cli.clientType = REDIS_CLUSTER_CLIENT
+ case 2:
+ options := &redis.FailoverOptions{
+ MasterName: cfg.Master,
+ SentinelAddrs: cfg.Nodes,
+ ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
+ DialTimeout: time.Duration(cfg.ConnectionTimeout) * time.Second,
+ WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
+ IdleTimeout: time.Duration(cfg.KeepAlive) * time.Second,
+ }
+ if cfg.PoolMaxIdle > 0 {
+ options.PoolSize = cfg.PoolMaxIdle
+ }
+ if cfg.Password != "" {
+ options.Password = cfg.Password
+ }
+ cli.redisClient = redis.NewFailoverClient(options)
+ cli.clientType = REDIS_SENTINEL_CLIENT
+ default:
+ options := &redis.Options{
+ Addr: cfg.Address,
+ ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
+ DialTimeout: time.Duration(cfg.ConnectionTimeout) * time.Second,
+ WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
+ IdleTimeout: time.Duration(cfg.KeepAlive) * time.Second,
+ }
+
+ if cfg.PoolMaxIdle > 0 {
+ options.PoolSize = cfg.PoolMaxIdle
+ }
+
+ if cfg.Password != "" {
+ options.Password = cfg.Password
+ }
+
+ log.Infoln("create redis for options: ", options)
+ cli.redisClient = redis.NewClient(options)
+ cli.clientType = REDIS_NORMAL_CLIENT
+ }
+}
+
+func (cli *RedisCli) IsValid() bool {
+ return cli.clientType != REDIS_UNKNOWN_CLIENT
+}
+
+func (cli *RedisCli) Close() error {
+ switch cli.clientType {
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.Close()
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.Close()
+ default:
+ return nil
+ }
+}
+
+func (cli *RedisCli) Del(key string) (int64, error) {
+ var err error
+ var val int64
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ val, err = cli.redisClient.Del(key).Result()
+ case REDIS_CLUSTER_CLIENT:
+ val, err = cli.redisClusterClient.Del(key).Result()
+ default:
+ return 0, errors.New(ERR_NOT_INIT_MSG)
+ }
+ if err == redis.Nil {
+ return val, nil
+ }
+ return val, err
+}
+
+/*
+* @key: input key
+* @value: input value
+* @expire: expiration for the key in milliseconds.
+ */
+
+func (cli *RedisCli) Set(key string, value interface{}, expire int64) (string, error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.Set(key, value, time.Duration(expire)*time.Millisecond).Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.Set(key, value, time.Duration(expire)*time.Millisecond).Result()
+ default:
+ return "", errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+func (cli *RedisCli) Get(key string) (val []byte, err error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT:
+ val, err = cli.redisClient.Get(key).Bytes()
+ case REDIS_CLUSTER_CLIENT:
+ val, err = cli.redisClusterClient.Get(key).Bytes()
+ default:
+ return nil, errors.New(ERR_NOT_INIT_MSG)
+ }
+
+ if err == redis.Nil {
+ return val, nil
+ }
+ return val, err
+}
+
+func (cli *RedisCli) GetRange(key string, start, end int64) (val []byte, err error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ val, err = cli.redisClient.GetRange(key, start, end).Bytes()
+ case REDIS_CLUSTER_CLIENT:
+ val, err = cli.redisClusterClient.GetRange(key, start, end).Bytes()
+ default:
+ return nil, errors.New(ERR_NOT_INIT_MSG)
+ }
+
+ if err == redis.Nil {
+ return val, nil
+ }
+ return val, err
+}
+
+func (cli *RedisCli) Publish(channel string, message interface{}) (int64, error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.Publish(channel, message).Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.Publish(channel, message).Result()
+ default:
+ return 0, errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+func (cli *RedisCli) Ping() (string, error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.Ping().Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.Ping().Result()
+ default:
+ return "", errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+func (cli *RedisCli) Keys(pattern string) ([]string, error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.Keys(pattern).Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.Keys(pattern).Result()
+ default:
+ return nil, errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+func (cli *RedisCli) MGet(keys []string) ([]interface{}, error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.MGet(keys...).Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.MGet(keys...).Result()
+ default:
+ return nil, errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+func (cli *RedisCli) MSet(pairs map[interface{}]interface{}) (string, error) {
+ var pairList []interface{}
+
+ for k, v := range pairs {
+ pairList = append(pairList, k, v)
+ }
+
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.MSet(pairList...).Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.MSet(pairList...).Result()
+ default:
+ return "", errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+func (cli *RedisCli) IncrBy(key string, value int64) (int64, error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.IncrBy(key, value).Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.IncrBy(key, value).Result()
+ default:
+ return 0, errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+func (cli *RedisCli) Expire(key string, expire int64) (bool, error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.Expire(key, time.Duration(expire)*time.Millisecond).Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.Expire(key, time.Duration(expire)*time.Millisecond).Result()
+ default:
+ return false, errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+/***************** below are hashes commands *************************/
+
+func (cli *RedisCli) HSet(key, field string, value interface{}) (bool, error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.HSet(key, field, value).Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.HSet(key, field, value).Result()
+ default:
+ return false, errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+func (cli *RedisCli) HGet(key, field string) (val string, err error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ val, err = cli.redisClient.HGet(key, field).Result()
+ case REDIS_CLUSTER_CLIENT:
+ val, err = cli.redisClusterClient.HGet(key, field).Result()
+ default:
+ return "", errors.New(ERR_NOT_INIT_MSG)
+ }
+
+ if err == redis.Nil {
+ return val, nil
+ }
+ return val, err
+}
+
+func (cli *RedisCli) HDel(key string, fields []string) (val int64, err error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ val, err = cli.redisClient.HDel(key, fields...).Result()
+ case REDIS_CLUSTER_CLIENT:
+ val, err = cli.redisClusterClient.HDel(key, fields...).Result()
+ default:
+ return 0, errors.New(ERR_NOT_INIT_MSG)
+ }
+
+ if err == redis.Nil {
+ return val, nil
+ }
+ return val, err
+}
+
+func (cli *RedisCli) HGetInt64(key, field string) (val int64, err error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ val, err = cli.redisClient.HGet(key, field).Int64()
+ case REDIS_CLUSTER_CLIENT:
+ val, err = cli.redisClusterClient.HGet(key, field).Int64()
+ default:
+ return 0, errors.New(ERR_NOT_INIT_MSG)
+ }
+
+ if err == redis.Nil {
+ return val, nil
+ }
+ return val, err
+}
+
+func (cli *RedisCli) HGetAll(key string) (val map[string]string, err error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ val, err = cli.redisClient.HGetAll(key).Result()
+ case REDIS_CLUSTER_CLIENT:
+ val, err = cli.redisClusterClient.HGetAll(key).Result()
+ default:
+ return nil, errors.New(ERR_NOT_INIT_MSG)
+ }
+
+ if err == redis.Nil {
+ return val, nil
+ }
+ return val, err
+}
+
+func (cli *RedisCli) HIncrBy(key, field string, incr int64) (int64, error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.HIncrBy(key, field, incr).Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.HIncrBy(key, field, incr).Result()
+ default:
+ return 0, errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+func (cli *RedisCli) HMSet(key string, fields map[string]interface{}) (string, error) {
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ return cli.redisClient.HMSet(key, fields).Result()
+ case REDIS_CLUSTER_CLIENT:
+ return cli.redisClusterClient.HMSet(key, fields).Result()
+ default:
+ return "", errors.New(ERR_NOT_INIT_MSG)
+ }
+}
+
+func (cli *RedisCli) HMGet(key string, fields []string) (map[string]interface{}, error) {
+ results := make(map[string]interface{})
+ var values []interface{}
+ var err error
+
+ switch cli.clientType {
+ case REDIS_NORMAL_CLIENT, REDIS_SENTINEL_CLIENT:
+ values, err = cli.redisClient.HMGet(key, fields...).Result()
+ case REDIS_CLUSTER_CLIENT:
+ values, err = cli.redisClusterClient.HMGet(key, fields...).Result()
+ default:
+ return nil, errors.New(ERR_NOT_INIT_MSG)
+ }
+
+ if err == redis.Nil {
+ return nil, nil
+ }
+
+ if err != nil {
+ log.Error("failed to HMGet for key ", key, " with err: ", err)
+ return nil, err
+ }
+
+ if len(fields) != len(values) {
+ log.Error("panic HMGet, input fields number: ", len(fields), " got values number: ",
+ len(values))
+ return nil, errors.New("HMGet fields number is not equal to values number.")
+ }
+
+ for i, key := range fields {
+ results[key] = values[i]
+ }
+ return results, nil
+}
+
+
+
package types
+
+import (
+ "bytes"
+ "encoding/binary"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strconv"
+ "time"
+
+ "github.com/dustin/go-humanize"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ FIELD_NAME_BODY = "body"
+ FIELD_NAME_USAGE = "usage"
+ FIELD_NAME_FILECOUNTS = "file_counts"
+)
+
+type Bucket struct {
+ *pb.Bucket
+}
+
+// implements the Serializable interface
+func (b *Bucket) Serialize() (map[string]interface{}, error) {
+ fields := make(map[string]interface{})
+ bytes, err := helper.MsgPackMarshal(b)
+ if err != nil {
+ return nil, err
+ }
+ fields[FIELD_NAME_BODY] = string(bytes)
+ fields[FIELD_NAME_USAGE] = b.Usages
+ return fields, nil
+}
+
+func (b *Bucket) Deserialize(fields map[string]string) (interface{}, error) {
+ body, ok := fields[FIELD_NAME_BODY]
+ if !ok {
+ return nil, errors.New(fmt.Sprintf("no field %s found", FIELD_NAME_BODY))
+ }
+
+ err := helper.MsgPackUnMarshal([]byte(body), b)
+ if err != nil {
+ return nil, err
+ }
+ if usageStr, ok := fields[FIELD_NAME_USAGE]; ok {
+ b.Usages, err = strconv.ParseInt(usageStr, 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return b, nil
+}
+
+func (b *Bucket) String() (s string) {
+ s += "Name: " + b.Name + "\n"
+ s += "CreateTime: " + time.Unix(b.CreateTime, 0).Format(CREATE_TIME_LAYOUT) + "\n"
+ s += "TenantId: " + b.TenantId + "\n"
+ s += "DefaultLocation: " + b.DefaultLocation + "\n"
+ s += "CORS: " + fmt.Sprintf("%+v", b.Cors) + "\n"
+ s += "ACL: " + fmt.Sprintf("%+v", b.Acl) + "\n"
+ s += "LifeCycle: " + fmt.Sprintf("%+v", b.LifecycleConfiguration) + "\n"
+ s += "Policy: " + fmt.Sprintf("%+v", b.BucketPolicy) + "\n"
+ s += "Versioning: " + fmt.Sprintf("%+v", b.Versioning) + "\n"
+ s += "Usage: " + humanize.Bytes(uint64(b.Usages)) + "\n"
+ return
+}
+
+/* Learn from this, http://stackoverflow.com/questions/33587227/golang-method-sets-pointer-vs-value-receiver */
+/* If you have a T and it is addressable you can call methods that have a receiver type of *T as well as methods that have a receiver type of T */
+func (b *Bucket) GetValues() (values map[string]map[string][]byte, err error) {
+ cors, err := json.Marshal(b.Cors)
+ if err != nil {
+ return
+ }
+ lc, err := json.Marshal(b.LifecycleConfiguration)
+ if err != nil {
+ return
+ }
+
+ var usage bytes.Buffer
+ err = binary.Write(&usage, binary.BigEndian, b.Usages)
+ if err != nil {
+ return
+ }
+ values = map[string]map[string][]byte{
+ BUCKET_COLUMN_FAMILY: map[string][]byte{
+ "UID": []byte(b.TenantId),
+ "ACL": []byte(b.Acl.CannedAcl),
+ "CORS": cors,
+ "LC": lc,
+ "createTime": []byte(time.Unix(b.CreateTime, 0).Format(CREATE_TIME_LAYOUT)),
+ "usage": usage.Bytes(),
+ },
+ // TODO fancy ACL
+ }
+ return
+}
+
+func (b Bucket) GetCreateSql() (string, []interface{}) {
+ acl, _ := json.Marshal(b.Acl)
+ cors, _ := json.Marshal(b.Cors)
+ lc, _ := json.Marshal(b.LifecycleConfiguration)
+ bucket_policy, _ := json.Marshal(b.BucketPolicy)
+ replia, _ := json.Marshal(b.ReplicationConfiguration)
+ //createTime := time.Unix(b.CreateTime, 0).Format(TIME_LAYOUT_TIDB)
+ createTime := time.Now().Format(TIME_LAYOUT_TIDB)
+ log.Infof("createTime=%v\n", createTime)
+
+ sql := "insert into buckets(bucketname,tenantid,userid,createtime,usages,location,acl,cors,lc,policy,versioning," +
+ "replication) values(?,?,?,?,?,?,?,?,?,?,?,?);"
+ args := []interface{}{b.Name, b.TenantId, b.UserId, createTime, b.Usages, b.DefaultLocation, acl, cors, lc,
+ bucket_policy, b.Versioning.Status, replia}
+ return sql, args
+}
+
+
+
package types
+
+import (
+ "errors"
+ "fmt"
+ "strconv"
+
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+)
+
+type Cluster struct {
+ Fsid string
+ Pool string
+ Weight int
+}
+
+func (c *Cluster) Serialize() (map[string]interface{}, error) {
+ fields := make(map[string]interface{})
+ bytes, err := helper.MsgPackMarshal(c)
+ if err != nil {
+ return nil, err
+ }
+ fields[FIELD_NAME_BODY] = string(bytes)
+ return fields, nil
+}
+
+func (c *Cluster) Deserialize(fields map[string]string) (interface{}, error) {
+ body, ok := fields[FIELD_NAME_BODY]
+ if !ok {
+ return nil, errors.New(fmt.Sprintf("no field %s found", FIELD_NAME_BODY))
+ }
+
+ err := helper.MsgPackUnMarshal([]byte(body), c)
+ if err != nil {
+ return nil, err
+ }
+ return c, nil
+}
+
+func (c *Cluster) GetValues() (values map[string]map[string][]byte, err error) {
+ values = map[string]map[string][]byte{
+ CLUSTER_COLUMN_FAMILY: map[string][]byte{
+ "weight": []byte(strconv.Itoa(c.Weight)),
+ },
+ }
+ return
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package types
+
+import (
+ "bytes"
+ "encoding/binary"
+ "strconv"
+ "time"
+)
+
+type GarbageCollection struct {
+ Rowkey string // rowkey cache
+ BucketName string
+ ObjectName string
+ Location string
+ ObjectId string
+ Status string // status of this entry, in Pending/Deleting
+ StorageMeta string // meta data used by storage driver, different storage driver may have different information
+ MTime time.Time // last modify time of status
+ Parts map[int]*Part
+ TriedTimes int
+}
+
+func (gc GarbageCollection) GetValues() (values map[string]map[string][]byte, err error) {
+ values = map[string]map[string][]byte{
+ GARBAGE_COLLECTION_COLUMN_FAMILY: map[string][]byte{
+ "location": []byte(gc.Location),
+ "storagemeta": []byte(gc.StorageMeta),
+ "oid": []byte(gc.ObjectId),
+ "status": []byte(gc.Status),
+ "mtime": []byte(gc.MTime.Format(CREATE_TIME_LAYOUT)),
+ "tried": []byte(strconv.Itoa(gc.TriedTimes)),
+ },
+ }
+ if len(gc.Parts) != 0 {
+ values[GARBAGE_COLLECTION_PART_COLUMN_FAMILY], err = valuesForParts(gc.Parts)
+ if err != nil {
+ return
+ }
+ }
+ return
+}
+
+func (gc GarbageCollection) GetValuesForDelete() map[string]map[string][]byte {
+ return map[string]map[string][]byte{
+ GARBAGE_COLLECTION_COLUMN_FAMILY: map[string][]byte{},
+ GARBAGE_COLLECTION_PART_COLUMN_FAMILY: map[string][]byte{},
+ }
+}
+
+// Rowkey format:
+// bigEndian(unixNanoTimestamp) + BucketName + ObjectName
+func (gc GarbageCollection) GetRowkey() (string, error) {
+ var rowkey bytes.Buffer
+ err := binary.Write(&rowkey, binary.BigEndian,
+ uint64(time.Now().UnixNano()))
+ if err != nil {
+ return "", err
+ }
+ rowkey.WriteString(gc.BucketName)
+ rowkey.WriteString(gc.ObjectName)
+ return rowkey.String(), nil
+}
+
+
+
package types
+
+import (
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "math"
+ "strconv"
+ "time"
+
+ "github.com/opensds/multi-cloud/api/pkg/s3/datatype"
+ "github.com/opensds/multi-cloud/s3/proto"
+ "github.com/xxtea/xxtea-go/xxtea"
+)
+
+type Part struct {
+ PartNumber int
+ Size int64
+ ObjectId string
+
+ // offset of this part in whole object, calculated when moving parts from
+ // `multiparts` table to `objects` table
+ Offset int64
+ Etag string
+ LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+ InitializationVector []byte
+}
+
+type MultipartMetadata struct {
+ InitiatorId string //TenantId
+ TenantId string
+ UserId string
+ ContentType string
+ Location string
+ Pool string
+ Acl s3.Acl
+ SseRequest datatype.SseRequest
+ EncryptionKey []byte
+ CipherKey []byte
+ Attrs map[string]string
+ Tier int32
+}
+
+type Multipart struct {
+ BucketName string
+ ObjectKey string
+ InitialTime time.Time
+ UploadId string // upload id cache
+ ObjectId string
+ StorageMeta string
+ Metadata MultipartMetadata
+ Parts map[int]*Part
+}
+
+func (m *Multipart) GetUploadId() (string, error) {
+ if m.UploadId != "" {
+ return m.UploadId, nil
+ }
+ if m.InitialTime.IsZero() {
+ return "", errors.New("Zero value InitialTime for Multipart")
+ }
+ m.UploadId = getMultipartUploadId(m.InitialTime)
+ return m.UploadId, nil
+}
+func getMultipartUploadId(t time.Time) string {
+ timeData := []byte(strconv.FormatUint(uint64(t.UnixNano()), 10))
+ return hex.EncodeToString(xxtea.Encrypt(timeData, XXTEA_KEY))
+}
+
+func GetMultipartUploadIdForTidb(uploadtime uint64) string {
+ realUploadTime := math.MaxUint64 - uploadtime
+ timeData := []byte(strconv.FormatUint(realUploadTime, 10))
+ return hex.EncodeToString(xxtea.Encrypt(timeData, XXTEA_KEY))
+}
+
+func (m *Multipart) GetValuesForDelete() map[string]map[string][]byte {
+ return map[string]map[string][]byte{
+ MULTIPART_COLUMN_FAMILY: map[string][]byte{},
+ }
+}
+
+func valuesForParts(parts map[int]*Part) (values map[string][]byte, err error) {
+ for partNumber, part := range parts {
+ var marshaled []byte
+ marshaled, err = json.Marshal(part)
+ if err != nil {
+ return
+ }
+ if values == nil {
+ values = make(map[string][]byte)
+ }
+ values[strconv.Itoa(partNumber)] = marshaled
+ }
+ return
+}
+
+
+
/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package types
+
+import (
+ "fmt"
+)
+
+// PartTooSmall - error if part size is less than 5MB.
+type PartTooSmall struct {
+ PartSize int64
+ PartNumber int
+ PartETag string
+}
+
+func (e PartTooSmall) Error() string {
+ return fmt.Sprintf("Part size for %d should be atleast 5MB. The part size is %d. Etag: %s",
+ e.PartNumber, e.PartNumber, e.PartETag)
+}
+
+
+
package types
+
+import (
+ "bytes"
+ "crypto/aes"
+ "crypto/cipher"
+ "crypto/rand"
+ "encoding/binary"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "strconv"
+ "time"
+
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/util"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ "github.com/xxtea/xxtea-go/xxtea"
+)
+
+type Object struct {
+ *pb.Object
+}
+
+type ObjectType string
+
+const (
+ ObjectTypeNormal = iota
+ ObjectTypeAppendable
+ ObjectTypeMultipart
+)
+
+func (o *Object) Serialize() (map[string]interface{}, error) {
+ fields := make(map[string]interface{})
+ body, err := helper.MsgPackMarshal(o)
+ if err != nil {
+ return nil, err
+ }
+ fields[FIELD_NAME_BODY] = string(body)
+ return fields, nil
+}
+
+func (o *Object) Deserialize(fields map[string]string) (interface{}, error) {
+ body, ok := fields[FIELD_NAME_BODY]
+ if !ok {
+ return nil, errors.New(fmt.Sprintf("no field %s", FIELD_NAME_BODY))
+ }
+
+ err := helper.MsgPackUnMarshal([]byte(body), o)
+ if err != nil {
+ return nil, err
+ }
+ return o, nil
+}
+
+func (o *Object) ObjectTypeToString() string {
+ switch o.Type {
+ case ObjectTypeNormal:
+ return "Normal"
+ case ObjectTypeAppendable:
+ return "Appendable"
+ case ObjectTypeMultipart:
+ return "Multipart"
+ default:
+ return "Unknown"
+ }
+}
+
+func (o *Object) String() (s string) {
+ s += "Name: " + o.ObjectKey + "\n"
+ s += "Bucket: " + o.BucketName + "\n"
+ s += "Location: " + o.Location + "\n"
+ //s += "Pool: " + o.Pool + "\n"
+ s += "Object ID: " + o.ObjectId + "\n"
+ s += "Last Modified Time: " + time.Unix(o.LastModified, 0).Format(CREATE_TIME_LAYOUT) + "\n"
+ s += "Version: " + o.VersionId + "\n"
+ s += "Type: " + o.ObjectTypeToString() + "\n"
+ s += "Tier: " + fmt.Sprintf("%d", o.Tier) + "\n"
+ // TODO: multi-part handle
+
+ return s
+}
+
+func (o *Object) GetVersionNumber() (uint64, error) {
+ decrypted, err := util.Decrypt(o.VersionId)
+ if err != nil {
+ return 0, err
+ }
+ version, err := strconv.ParseUint(decrypted, 10, 64)
+ if err != nil {
+ return 0, err
+ }
+ return version, nil
+}
+
+func (o *Object) GetValues() (values map[string]map[string][]byte, err error) {
+ var size, tier bytes.Buffer
+ err = binary.Write(&size, binary.BigEndian, o.Size)
+ if err != nil {
+ return
+ }
+ err = binary.Write(&tier, binary.BigEndian, o.Tier)
+ if err != nil {
+ return
+ }
+ err = o.encryptSseKey()
+ if err != nil {
+ return
+ }
+ if o.ServerSideEncryption.EncryptionKey == nil {
+ o.ServerSideEncryption.EncryptionKey = []byte{}
+ }
+ if o.ServerSideEncryption.InitilizationVector == nil {
+ o.ServerSideEncryption.InitilizationVector = []byte{}
+ }
+ var attrsData []byte
+ if o.CustomAttributes != nil {
+ attrsData, err = json.Marshal(o.CustomAttributes)
+ if err != nil {
+ return
+ }
+ }
+ values = map[string]map[string][]byte{
+ OBJECT_COLUMN_FAMILY: map[string][]byte{
+ "bucket": []byte(o.BucketName),
+ "location": []byte(o.Location),
+ "owner": []byte(o.UserId),
+ "oid": []byte(o.ObjectId),
+ "size": size.Bytes(),
+ "lastModified": []byte(time.Unix(o.LastModified, 0).Format(CREATE_TIME_LAYOUT)),
+ "etag": []byte(o.Etag),
+ "content-type": []byte(o.ContentType),
+ "attributes": attrsData, // TODO
+ "ACL": []byte(o.Acl.CannedAcl),
+ "nullVersion": []byte(helper.Ternary(o.NullVersion, "true", "false").(string)),
+ "deleteMarker": []byte(helper.Ternary(o.DeleteMarker, "true", "false").(string)),
+ "sseType": []byte(o.ServerSideEncryption.SseType),
+ "encryptionKey": o.ServerSideEncryption.EncryptionKey,
+ "IV": o.ServerSideEncryption.InitilizationVector,
+ "type": []byte(o.ObjectTypeToString()),
+ "tier": tier.Bytes(),
+ },
+ }
+ // TODO: multipart handle
+
+ return
+}
+
+func (o *Object) GetValuesForDelete() (values map[string]map[string][]byte) {
+ return map[string]map[string][]byte{
+ OBJECT_COLUMN_FAMILY: map[string][]byte{},
+ OBJECT_PART_COLUMN_FAMILY: map[string][]byte{},
+ }
+}
+
+func (o *Object) encryptSseKey() (err error) {
+ // Don't encrypt if `EncryptionKey` is not set
+ if len(o.ServerSideEncryption.EncryptionKey) == 0 {
+ return
+ }
+
+ if len(o.ServerSideEncryption.InitilizationVector) == 0 {
+ o.ServerSideEncryption.InitilizationVector = make([]byte, INITIALIZATION_VECTOR_LENGTH)
+ _, err = io.ReadFull(rand.Reader, o.ServerSideEncryption.InitilizationVector)
+ if err != nil {
+ return
+ }
+ }
+
+ block, err := aes.NewCipher(SSE_S3_MASTER_KEY)
+ if err != nil {
+ return err
+ }
+
+ aesGcm, err := cipher.NewGCM(block)
+ if err != nil {
+ return err
+ }
+
+ // InitializationVector is 16 bytes(because of CTR), but use only first 12 bytes in GCM
+ // for performance
+ o.ServerSideEncryption.EncryptionKey = aesGcm.Seal(nil, o.ServerSideEncryption.InitilizationVector[:12], o.ServerSideEncryption.EncryptionKey, nil)
+ return nil
+}
+
+func (o *Object) GetVersionId() string {
+ if o.NullVersion {
+ return "null"
+ }
+ if o.VersionId != "" {
+ return o.VersionId
+ }
+ timeData := []byte(strconv.FormatUint(uint64(o.LastModified), 10))
+ o.VersionId = hex.EncodeToString(xxtea.Encrypt(timeData, XXTEA_KEY))
+ return o.VersionId
+}
+
+//Tidb related function
+
+func (o *Object) GetCreateSql() (string, []interface{}) {
+ version := math.MaxUint64 - uint64(o.LastModified)
+ customAttributes, _ := json.Marshal(o.CustomAttributes)
+ acl, _ := json.Marshal(o.Acl)
+ var sseType string
+ var encryptionKey, initVector []byte
+ if o.ServerSideEncryption != nil {
+ sseType = o.ServerSideEncryption.SseType
+ encryptionKey = o.ServerSideEncryption.EncryptionKey
+ initVector = o.ServerSideEncryption.InitilizationVector
+ }
+
+ lastModifiedTime := time.Unix(o.LastModified, 0).Format(TIME_LAYOUT_TIDB)
+ sql := "insert into objects (bucketname, name, version, location, tenantid, userid, size, objectid, " +
+ " lastmodifiedtime, etag, contenttype, customattributes, acl, nullversion, deletemarker, ssetype, " +
+ " encryptionkey, initializationvector, type, tier, storageMeta, encsize) " +
+ "values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+ args := []interface{}{o.BucketName, o.ObjectKey, version, o.Location, o.TenantId, o.UserId, o.Size, o.ObjectId,
+ lastModifiedTime, o.Etag, o.ContentType, customAttributes, acl, o.NullVersion, o.DeleteMarker, sseType,
+ encryptionKey, initVector, o.Type, o.Tier, o.StorageMeta, o.EncSize}
+
+ return sql, args
+}
+
+func (o *Object) GetUpdateMetaSql() (string, []interface{}) {
+ version := math.MaxUint64 - uint64(o.LastModified)
+ attrs, _ := json.Marshal(o.CustomAttributes)
+ acl, _ := json.Marshal(o.Acl)
+ sql := "update objects set acl = ?, contenttype = ?, customattributes =? where bucketname=? and name=? and version=?"
+ args := []interface{}{acl, o.ContentType, attrs, o.BucketName, o.ObjectKey, version}
+ return sql, args
+
+}
+
+
+
+
package types
+
+import (
+ "bytes"
+ "encoding/binary"
+)
+
+type ObjMap struct {
+ Rowkey []byte // Rowkey cache
+ Name string
+ BucketName string
+ NullVerNum uint64
+ NullVerId string
+}
+
+func (om *ObjMap) GetRowKey() (string, error) {
+ if len(om.Rowkey) != 0 {
+ return string(om.Rowkey), nil
+ }
+ var rowkey bytes.Buffer
+ rowkey.WriteString(om.BucketName + ObjectNameSeparator)
+
+ rowkey.WriteString(om.Name + ObjectNameSeparator)
+
+ om.Rowkey = rowkey.Bytes()
+ return string(om.Rowkey), nil
+}
+
+func (om *ObjMap) GetValues() (values map[string]map[string][]byte, err error) {
+ var nullVerNum bytes.Buffer
+ err = binary.Write(&nullVerNum, binary.BigEndian, om.NullVerNum)
+ if err != nil {
+ return
+ }
+ values = map[string]map[string][]byte{
+ OBJMAP_COLUMN_FAMILY: map[string][]byte{
+ "nullVerNum": nullVerNum.Bytes(),
+ },
+ }
+ return
+}
+
+func (om *ObjMap) GetValuesForDelete() (values map[string]map[string][]byte) {
+ return map[string]map[string][]byte{
+ OBJMAP_COLUMN_FAMILY: map[string][]byte{},
+ }
+}
+
+
+
package types
+
+type SimpleIndex struct {
+ Index []int64
+}
+
+func (array *SimpleIndex) SearchLowerBound(key int64) int {
+ var low int = 0
+ var high int = len(array.Index) - 1
+ var mid = (low + high) / 2
+
+ if array.Index[low] > key {
+ return -1
+ }
+
+ for low <= high {
+
+ if array.Index[mid] == key {
+ break
+ }
+
+ if array.Index[mid] > key {
+ high = mid - 1
+ } else {
+ low = mid + 1
+ }
+ mid = (low + high) / 2
+
+ }
+
+ return mid
+
+}
+
+func (array *SimpleIndex) SearchUpperBound(key int64) int {
+ var low int = 0
+ var high int = len(array.Index) - 1
+ var mid = (low + high) / 2
+
+ if array.Index[high] <= key {
+ return -1
+ }
+
+ for low <= high {
+ if array.Index[mid] > key {
+ if mid-1 >= low && key >= array.Index[mid-1] {
+ return mid
+ } else {
+ high = mid - 1
+ }
+ } else {
+ if mid+1 <= high && key < array.Index[mid+1] {
+ return mid + 1
+ } else {
+ low = mid + 1
+ }
+ }
+ mid = (low + high) / 2
+ }
+
+ return mid
+}
+
+
+
package types
+
+import (
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+)
+
+type StorageClass int32
+
+var (
+ StorageClassIndexMap = map[StorageClass]string{
+ utils.Tier1: "STANDARD",
+ utils.Tier99: "STANDARD_IA",
+ utils.Tier999: "GLACIER",
+ }
+
+ StorageClassStringMap = map[string]StorageClass{
+ "STANDARD": utils.Tier1,
+ "STANDARD_IA": utils.Tier99,
+ "GLACIER": utils.Tier999,
+ }
+)
+
+func (s StorageClass) ToString() string {
+ return StorageClassIndexMap[s]
+}
+
+func MatchStorageClassIndex(storageClass string) (StorageClass, error) {
+ if index, ok := StorageClassStringMap[storageClass]; ok {
+ return index, nil
+ } else {
+ return 0, ErrInvalidStorageClass
+ }
+}
+
+
+
package util
+
+import (
+ "context"
+ "encoding/hex"
+ "github.com/micro/go-micro/metadata"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ . "github.com/opensds/multi-cloud/s3/error"
+ log "github.com/sirupsen/logrus"
+ "github.com/xxtea/xxtea-go/xxtea"
+)
+
+var XXTEA_KEY = []byte("hehehehe")
+
+func Decrypt(value string) (string, error) {
+ bytes, err := hex.DecodeString(value)
+ if err != nil {
+ return "", err
+ }
+ return string(xxtea.Decrypt(bytes, XXTEA_KEY)), nil
+}
+
+func Encrypt(value string) string {
+ return hex.EncodeToString(xxtea.Encrypt([]byte(value), XXTEA_KEY))
+}
+
+func GetCredentialFromCtx(ctx context.Context) (isAdmin bool, tenantId string, userId string, err error) {
+ var ok bool
+ var md map[string]string
+ md, ok = metadata.FromContext(ctx)
+ if !ok {
+ log.Error("get metadata from ctx failed.")
+ err = ErrInternalError
+ return
+ }
+
+ isAdmin = false
+ isAdminStr, _ := md[common.CTX_KEY_IS_ADMIN]
+ if isAdminStr == common.CTX_VAL_TRUE {
+ isAdmin = true
+ }
+
+ tenantId, ok = md[common.CTX_KEY_TENANT_ID]
+ userId, ok = md[common.CTX_KEY_USER_ID]
+
+ log.Debugf("isAdmin=%v, tenantId=%s, userId=%s, err=%v\n", isAdmin, tenantId, userId, err)
+ return
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package model
+
+import "encoding/xml"
+
+var Xmlns = "http://s3.amazonaws.com/doc/2006-03-01"
+
+type CreateBucketConfiguration struct {
+ Xmlns string `xml:"xmlns,attr"`
+ LocationConstraint string `xml:"LocationConstraint"`
+ SSEOpts SSEConfiguration
+}
+
+type Owner struct {
+ ID string `xml:"ID"`
+ DisplayName string `xml:"DisplayName"`
+}
+
+type Bucket struct {
+ Name string `xml:"Name"`
+ CreateTime string `xml:"CreateTime"`
+ LocationConstraint string `xml:"LocationConstraint"`
+ VersionOpts VersioningConfiguration
+ SSEOpts SSEConfiguration
+}
+
+type ListAllMyBucketsResult struct {
+ Xmlns string `xml:"xmlns,attr"`
+ Owner Owner `xml:"Owner"`
+ Buckets []Bucket `xml:"Buckets"`
+}
+
+type InitiateMultipartUploadResult struct {
+ Xmlns string `xml:"xmlns,attr"`
+ Bucket string `xml:"Bucket"`
+ Key string `xml:"Key"`
+ UploadId string `xml:"UploadId"`
+}
+
+//PartNumber should be between 1 and 10000.
+//Please refer to https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/qfacts.html
+type UploadPartResult struct {
+ Xmlns string `xml:"xmlns,attr"`
+ PartNumber int64 `xml:"PartNumber"`
+ ETag string `xml:"ETag"`
+}
+
+// completedParts - is a collection satisfying sort.Interface.
+type CompletedParts []Part
+
+func (a CompletedParts) Len() int { return len(a) }
+func (a CompletedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a CompletedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
+
+type Part struct {
+ PartNumber int64 `xml:"PartNumber"`
+ ETag string `xml:"ETag"`
+ Size int64 `xml:"Size"`
+ LastModifyTime int64 `xml:"LastModifyTime"`
+}
+
+type CompleteMultipartUpload struct {
+ Xmlns string `xml:"xmlns,attr"`
+ Parts []Part `xml:"Part"`
+}
+
+type CompleteMultipartUploadResult struct {
+ Xmlns string `xml:"xmlns,attr"`
+ Location string `xml:"Location"`
+ Bucket string `xml:"Bucket"`
+ Key string `xml:"Key"`
+ Size int64 `xml:"Size"`
+ ETag string `xml:"ETag"`
+}
+
+type ListPartsOutput struct {
+ Xmlns string `xml:"xmlns,attr"`
+ Bucket string `xml:"Bucket"`
+ Key string `xml:"Key"`
+ UploadId string `xml:"UploadId"`
+ MaxParts int `xml:"MaxParts"`
+ IsTruncated bool `xml:"IsTruncated"`
+ Owner Owner `xml:"Owner"`
+ Parts []Part `xml:"Part"`
+}
+
+type LifecycleConfiguration struct {
+ Rule []Rule `xml:"Rule"`
+}
+
+type SSEConfiguration struct {
+ XMLName xml.Name `xml:"SSEConfiguration"`
+ Text string `xml:",chardata"`
+ SSE struct {
+ Text string `xml:",chardata"`
+ Enabled string `xml:"enabled"`
+ } `xml:"SSE"`
+ SSEKMS struct {
+ Text string `xml:",chardata"`
+ Enabled string `xml:"enabled"`
+ DefaultKMSMasterKey string `xml:"DefaultKMSMasterKey"`
+ } `xml:"SSE-KMS"`
+}
+
+type Rule struct {
+ ID string `xml:"ID"`
+ Filter Filter `xml:"Filter"`
+ Status string `xml:"Status"`
+ Transition []Transition `xml:"Transition"`
+ Expiration []Expiration `xml:"Expiration"`
+ AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload"`
+}
+
+type Filter struct {
+ Prefix string `xml:"Prefix"`
+}
+
+type Transition struct {
+ Days int32 `xml:"Days"`
+ StorageClass string `xml:"StorageClass"`
+ Backend string `xml:"Backend"`
+}
+
+type Expiration struct {
+ Days int32 `xml:"Days"`
+ //Delete marker will be used in later release
+ //ExpiredObjectDeleteMarker string `xml:"ExpiredObjectDeleteMArker"`
+}
+
+type AbortIncompleteMultipartUpload struct {
+ DaysAfterInitiation int32 `xml:"DaysAfterInitiation"`
+}
+
+type StorageClass struct {
+ Name string `xml:"Name"`
+ Tier int32 `xml:"Tier"`
+}
+
+type ListStorageClasses struct {
+ Xmlns string `xml:"xmlns,attr"`
+ Classes []StorageClass `xml:"Class"`
+}
+
+type VersioningConfiguration struct {
+ XMLName xml.Name `xml:"VersioningConfiguration"`
+ Status string `xml:"Status"`
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+ "context"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+
+ "github.com/opensds/multi-cloud/api/pkg/s3"
+ . "github.com/opensds/multi-cloud/s3/error"
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/util"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+func (s *s3Service) ListBuckets(ctx context.Context, in *pb.BaseRequest, out *pb.ListBucketsResponse) error {
+ log.Info("ListBuckets is called in s3 service.")
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ buckets, err := s.MetaStorage.Db.GetBuckets(ctx)
+ if err != nil {
+ log.Errorf("list buckets failed, err:%v\n", err)
+ return nil
+ }
+
+ // TODO: paging list
+ for j := 0; j < len(buckets); j++ {
+ if buckets[j].Deleted != true {
+ out.Buckets = append(out.Buckets, &pb.Bucket{
+ Name: buckets[j].Name,
+ TenantId: buckets[j].TenantId,
+ CreateTime: buckets[j].CreateTime,
+ Usages: buckets[j].Usages,
+ Tier: buckets[j].Tier,
+ DefaultLocation: buckets[j].DefaultLocation,
+ Versioning: buckets[j].Versioning,
+ ServerSideEncryption: buckets[j].ServerSideEncryption,
+ })
+ }
+ }
+
+ log.Infof("out.Buckets:%+v\n", out.Buckets)
+ return nil
+}
+
+func (s *s3Service) CreateBucket(ctx context.Context, in *pb.Bucket, out *pb.BaseResponse) error {
+ log.Infof("CreateBucket is called in s3 service, in:%+v\n", in)
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucketName := in.Name
+ if err = s3.CheckValidBucketName(bucketName); err != nil {
+ log.Errorf("invalid bucket name:%v\n", err)
+ err = ErrInvalidBucketName
+ return nil
+ }
+
+ processed, err := s.MetaStorage.Db.CheckAndPutBucket(ctx, &Bucket{Bucket: in})
+ if err != nil {
+ log.Error("Error making checkandput: ", err)
+ return nil
+ }
+ log.Infof("create bucket[%s] in database succeed, processed=%v.\n", in.Name, processed)
+ if !processed { // bucket already exists, return accurate message
+ /*bucket*/ _, err := s.MetaStorage.GetBucket(ctx, bucketName, false)
+ if err == nil {
+ log.Error("Error get bucket: ", bucketName, ", with error", err)
+ err = ErrBucketAlreadyExists
+ }
+ }
+ if in.Versioning != nil {
+ err = s.MetaStorage.Db.CreateBucketVersioning(ctx, in.Name, in.Versioning.Status)
+ if err != nil {
+ // set default version to disabled
+ err = s.MetaStorage.Db.CreateBucketVersioning(ctx, in.Name, "Disabled")
+ log.Error("Error creating version entry: ", err)
+ return err
+ }
+ }
+
+ if in.ServerSideEncryption != nil{
+ byteArr, keyErr := utils.GetRandom32BitKey()
+ if keyErr != nil {
+ log.Error("Error generating SSE key", keyErr)
+ return keyErr
+ }
+ err = s.MetaStorage.Db.CreateBucketSSE(ctx, in.Name, in.ServerSideEncryption.SseType, byteArr)
+ if err != nil {
+ log.Error("Error creating SSE entry: ", err)
+ return err
+ }
+ } else{
+ // set default SSE option to none
+ err = s.MetaStorage.Db.CreateBucketSSE(ctx, in.Name, "NONE", []byte("NONE"))
+ if err != nil {
+ log.Error("Error creating SSE entry: ", err)
+ return err
+ }
+ }
+ return err
+}
+
+func (s *s3Service) GetBucket(ctx context.Context, in *pb.Bucket, out *pb.GetBucketResponse) error {
+ log.Infof("GetBucket %s is called in s3 service.", in.Id)
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, in.Name, false)
+ if err != nil {
+ log.Errorf("get bucket[%s] failed, err:%v\n", in.Name, err)
+ // return nil, otherwise api cannot get error code
+ return nil
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil {
+ log.Errorf("get credential faied, err:%v\n", err)
+ return err
+ }
+ if !isAdmin {
+ if tenantId != bucket.TenantId {
+ switch bucket.Acl.CannedAcl {
+ case "public-read", "public-read-write":
+ break
+ default:
+ err = ErrBucketAccessForbidden
+ return err
+ }
+ }
+ }
+
+ out.BucketMeta = &pb.Bucket{
+ Id: bucket.Id,
+ Name: bucket.Name,
+ TenantId: bucket.TenantId,
+ UserId: bucket.UserId,
+ Acl: bucket.Acl,
+ CreateTime: bucket.CreateTime,
+ Deleted: bucket.Deleted,
+ DefaultLocation: bucket.DefaultLocation,
+ Tier: bucket.Tier,
+ Usages: bucket.Usages,
+ Versioning: bucket.Versioning,
+ ServerSideEncryption: bucket.ServerSideEncryption,
+ }
+
+ return nil
+}
+
+func (s *s3Service) DeleteBucket(ctx context.Context, in *pb.Bucket, out *pb.BaseResponse) error {
+ bucketName := in.Name
+ log.Infof("DeleteBucket is called in s3 service, bucketName is %s.\n", bucketName)
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, bucketName, false)
+ if err != nil {
+ log.Errorf("get bucket failed, err:%+v\n", err)
+ return nil
+ }
+ _, _, _, err = CheckRights(ctx, bucket.TenantId)
+ if err != nil {
+ log.Errorln("failed to check rights, err:", err)
+ return nil
+ }
+
+ // Check if bucket is empty
+ objs, _, err := s.MetaStorage.Db.ListObjects(ctx, bucketName, false, 1, nil)
+ if err != nil {
+ log.Errorf("list objects failed, err:%v\n", err)
+ return nil
+ }
+ if len(objs) != 0 {
+ log.Errorf("bucket[%s] is not empty.\n", bucketName)
+ err = ErrBucketNotEmpty
+ return nil
+ }
+ err = s.MetaStorage.Db.DeleteBucket(ctx, bucket)
+ if err != nil {
+ log.Errorf("delete bucket[%s] failed, err:%v\n", bucketName, err)
+ return nil
+ }
+
+ log.Infof("delete bucket[%s] successfully\n", bucketName)
+ return nil
+}
+
+func (s *s3Service) PutBucketLifecycle(ctx context.Context, in *pb.PutBucketLifecycleRequest, out *pb.BaseResponse) error {
+ log.Infof("set lifecycle for bucket[%s]\n", in.BucketName)
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, in.BucketName, true)
+ if err != nil {
+ log.Errorf("get bucket failed, err:%v\n", err)
+ return nil
+ }
+
+ _, _, _, err = CheckRights(ctx, bucket.TenantId)
+ if err != nil {
+ if err == ErrAccessDenied {
+ err = ErrBucketAccessForbidden
+ }
+ return nil
+ }
+
+ bucket.LifecycleConfiguration = in.Lc
+ err = s.MetaStorage.Db.PutBucket(ctx, bucket)
+ /* TODO: enable cache, see https://github.com/opensds/multi-cloud/issues/698
+ if err == nil {
+ s.MetaStorage.Cache.Remove(redis.BucketTable, meta.BUCKET_CACHE_PREFIX, bucketName)
+ }*/
+
+ return nil
+}
+
+func (s *s3Service) GetBucketLifecycle(ctx context.Context, in *pb.BaseRequest, out *pb.GetBucketLifecycleResponse) error {
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, in.Id, true)
+ if err != nil {
+ log.Errorf("get bucket failed, err:%v\n", err)
+ return nil
+ }
+
+ _, _, _, err = CheckRights(ctx, bucket.TenantId)
+ if err != nil {
+ if err == ErrAccessDenied {
+ err = ErrBucketAccessForbidden
+ }
+ return nil
+ }
+
+ if len(bucket.LifecycleConfiguration) == 0 {
+ log.Errorf("bucket[%s] has no lifecycle configuration\n", in.Id)
+ err = ErrNoSuchBucketLc
+ } else {
+ out.Lc = bucket.LifecycleConfiguration
+ }
+
+ return nil
+}
+
+func (s *s3Service) DeleteBucketLifecycle(ctx context.Context, in *pb.BaseRequest, out *pb.BaseResponse) error {
+ log.Infof("delete lifecycle for bucket:%s\n", in.Id)
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, in.Id, true)
+ if err != nil {
+ log.Errorf("get bucket err: %v\n", err)
+ return nil
+ }
+ _, _, _, err = CheckRights(ctx, bucket.TenantId)
+ if err != nil {
+ if err == ErrAccessDenied {
+ err = ErrBucketAccessForbidden
+ }
+ return nil
+ }
+
+ bucket.LifecycleConfiguration = nil
+ err = s.MetaStorage.Db.PutBucket(ctx, bucket)
+ if err != nil {
+ log.Errorf("update bucket failed, err: %v\n", err)
+ return nil
+ } else {
+ log.Infof("delete lifecycle for bucket:%s successfully\n", in.Id)
+ }
+
+ /* TODO: enable cache
+ if err == nil {
+ yig.MetaStorage.Cache.Remove(redis.BucketTable, meta.BUCKET_CACHE_PREFIX, bucketName)
+ }*/
+
+ return nil
+}
+
+// ListBucketLifecycle is used by lifecycle management service, not need to return error code
+func (s *s3Service) ListBucketLifecycle(ctx context.Context, in *pb.BaseRequest, out *pb.ListBucketsResponse) error {
+ log.Info("ListBucketLifecycle is called in s3 service.")
+ //buckets := []pb.Bucket{}
+ buckets, err := s.MetaStorage.Db.ListBucketLifecycle(ctx)
+ if err != nil {
+ log.Errorf("list buckets with lifecycle failed, err:%v\n", err)
+ return err
+ }
+
+ // TODO: paging list
+ for _, v := range buckets {
+ if v.Deleted != true {
+ out.Buckets = append(out.Buckets, &pb.Bucket{
+ Name: v.Name,
+ DefaultLocation: v.DefaultLocation,
+ LifecycleConfiguration: v.LifecycleConfiguration,
+ })
+ }
+ }
+
+ log.Info("list lifecycle successfully")
+ return nil
+}
+
+func (s *s3Service) PutBucketACL(ctx context.Context, in *pb.PutBucketACLRequest, out *pb.BaseResponse) error {
+ log.Info("PutBucketACL is called in s3 service.")
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, in.ACLConfig.BucketName, true)
+ if err != nil {
+ log.Errorf("failed to get bucket meta. err: %v\n", err)
+ return err
+ }
+
+ _, _, _, err = CheckRights(ctx, bucket.TenantId)
+ if err != nil {
+ log.Errorln("failed to check rights, err:", err)
+ return nil
+ }
+
+ bucket.Acl = &pb.Acl{CannedAcl: in.ACLConfig.CannedAcl}
+ err = s.MetaStorage.Db.PutBucket(ctx, bucket)
+ if err != nil {
+ log.Error("failed to put bucket, err:", err)
+ return err
+ }
+ log.Infoln("Put bucket acl successfully.")
+ return nil
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+ "context"
+ "crypto/md5"
+ "encoding/hex"
+ "io"
+ "strconv"
+ "time"
+
+ . "github.com/opensds/multi-cloud/s3/error"
+ dscommon "github.com/opensds/multi-cloud/s3/pkg/datastore/common"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/driver"
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/util"
+ "github.com/opensds/multi-cloud/s3/pkg/model"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ MAX_PART_SIZE = 5 << 30 // 5GB, max object size in single upload
+ MAX_PART_NUMBER = 10000 // max upload part number in one multipart upload
+)
+
+func (s *s3Service) ListBucketUploadRecords(ctx context.Context, in *pb.ListBucketUploadRequest, out *pb.ListBucketUploadResponse) error {
+ log.Info("ListBucketUploadRecords is called in s3 service.")
+ bucketName := in.BucketName
+
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, bucketName, true)
+ if err != nil {
+ log.Errorln("failed to get bucket meta. err:", err)
+ return err
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil && isAdmin == false {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return err
+ }
+
+ if !isAdmin {
+ switch bucket.Acl.CannedAcl {
+ case "public-read", "public-read-write":
+ break
+ default:
+ if bucket.TenantId != tenantId {
+ log.Errorln("bucket owner is not equal to request owner.")
+ err = ErrBucketAccessForbidden
+ return err
+ }
+ }
+ }
+
+ result, err := s.MetaStorage.Db.ListMultipartUploads(in)
+ if err != nil {
+ log.Errorln("failed to list multipart uploads in meta storage. err:", err)
+ return err
+ }
+ out.Result = result
+
+ log.Infoln("List bucket multipart uploads successfully.")
+ return nil
+}
+
+func (s *s3Service) InitMultipartUpload(ctx context.Context, in *pb.InitMultiPartRequest, out *pb.InitMultiPartResponse) error {
+ bucketName := in.BucketName
+ objectKey := in.ObjectKey
+ log.Infof("InitMultipartUpload is called in s3 service, bucketName=%s, objectKey=%s\n", bucketName, objectKey)
+
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil && isAdmin == false {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return err
+ }
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, bucketName, true)
+ if err != nil {
+ log.Errorln("failed to get bucket from meta storage. err:", err)
+ return err
+ }
+ if !isAdmin {
+ switch bucket.Acl.CannedAcl {
+ case "public-read-write":
+ break
+ default:
+ if bucket.TenantId != tenantId {
+ log.Errorln("bucket owner is not equal to request owner.")
+ err = ErrBucketAccessForbidden
+ return err
+ }
+ }
+ }
+
+ attrs := in.Attrs
+ contentType, ok := attrs["Content-Type"]
+ if !ok {
+ contentType = "application/octet-stream"
+ }
+
+ backendName := in.Location
+ if backendName == "" {
+ backendName = bucket.DefaultLocation
+ }
+ backend, err := utils.GetBackend(ctx, s.backendClient, backendName)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+
+ sd, err := driver.CreateStorageDriver(backend.Type, backend)
+ if err != nil {
+ log.Errorln("failed to create storage. err:", err)
+ return err
+ }
+ tier := in.Tier
+ if tier == 0 {
+ // if not set, use the default tier
+ tier = utils.Tier1
+ }
+ res, err := sd.InitMultipartUpload(ctx, &pb.Object{BucketName: bucketName, ObjectKey: objectKey, Tier: tier})
+ if err != nil {
+ log.Errorln("failed to init multipart upload. err:", err)
+ return err
+ }
+
+ multipartMetadata := MultipartMetadata{
+ InitiatorId: tenantId,
+ TenantId: bucket.TenantId,
+ UserId: bucket.UserId,
+ ContentType: contentType,
+ Attrs: attrs,
+ Tier: tier,
+ Location: backendName,
+ // TODO: add sse information
+ }
+ if in.Acl != nil {
+ multipartMetadata.Acl = *in.Acl
+ } else {
+ multipartMetadata.Acl.CannedAcl = "private"
+ }
+
+ multipart := Multipart{
+ BucketName: bucketName,
+ ObjectKey: objectKey,
+ UploadId: res.UploadId,
+ ObjectId: res.ObjectId,
+ InitialTime: time.Now().UTC(),
+ Metadata: multipartMetadata,
+ }
+
+ err = s.MetaStorage.Db.CreateMultipart(multipart)
+ if err != nil {
+ log.Errorln("failed to create multipart in meta. err:", err)
+ return err
+ }
+ out.UploadID = res.UploadId
+
+ return nil
+}
+
+func (s *s3Service) UploadPart(ctx context.Context, stream pb.S3_UploadPartStream) error {
+ log.Info("UploadPart is called in s3 service.")
+ var err error
+ uploadResponse := pb.UploadPartResponse{}
+ defer func() {
+ uploadResponse.ErrorCode = GetErrCode(err)
+ stream.SendMsg(&uploadResponse)
+ }()
+
+ uploadRequest := pb.UploadPartRequest{}
+ err = stream.RecvMsg(&uploadRequest)
+ if err != nil {
+ log.Errorln("failed to receive msg. err:", err)
+ return err
+ }
+ bucketName := uploadRequest.BucketName
+ objectKey := uploadRequest.ObjectKey
+ partId := uploadRequest.PartId
+ uploadId := uploadRequest.UploadId
+ size := uploadRequest.Size
+
+ log.Infof("uploadpart, bucketname:%v, objectkey:%v, partId:%v, uploadId:%v,size:%v", bucketName, objectKey, partId, uploadId, size)
+ multipart, err := s.MetaStorage.GetMultipart(bucketName, objectKey, uploadId)
+ if err != nil {
+ log.Infoln("failed to get multipart. err:", err)
+ return err
+ }
+
+ if size > MAX_PART_SIZE {
+ log.Errorf("object part size is too large. size:", size)
+ err = ErrEntityTooLarge
+ return err
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil && isAdmin == false {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return err
+ }
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, bucketName, true)
+ if err != nil {
+ log.Errorln("get bucket failed with err:", err)
+ return err
+ }
+
+ if !isAdmin {
+ switch bucket.Acl.CannedAcl {
+ case "public-read-write":
+ break
+ default:
+ if bucket.TenantId != tenantId {
+ err = ErrBucketAccessForbidden
+ log.Errorln("bucket owner is not equal to request owner.")
+ return err
+ }
+ }
+ }
+
+ backendName := bucket.DefaultLocation
+ backend, err := utils.GetBackend(ctx, s.backendClient, backendName)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+
+ md5Writer := md5.New()
+ data := &StreamReader{in: stream}
+ limitedDataReader := io.LimitReader(data, size)
+ dataReader := io.TeeReader(limitedDataReader, md5Writer)
+ sd, err := driver.CreateStorageDriver(backend.Type, backend)
+ if err != nil {
+ log.Errorln("failed to create storage. err:", err)
+ return nil
+ }
+ ctx = context.Background()
+ ctx = context.WithValue(ctx, dscommon.CONTEXT_KEY_MD5, uploadRequest.Md5Hex)
+ log.Infoln("bucketname:", bucketName, " objectKey:", objectKey, " uploadid:", uploadId, " objectId:", multipart.ObjectId, " partid:", partId)
+ _, err = sd.UploadPart(ctx, dataReader, &pb.MultipartUpload{
+ Bucket: bucketName,
+ Key: objectKey,
+ UploadId: uploadId,
+ ObjectId: multipart.ObjectId},
+ int64(partId), size)
+ if err != nil {
+ log.Errorln("failed to upload part to backend. err:", err)
+ return err
+ }
+
+ calculatedMd5 := hex.EncodeToString(md5Writer.Sum(nil))
+ part := Part{
+ PartNumber: int(partId),
+ Size: size,
+ ObjectId: multipart.ObjectId,
+ Etag: calculatedMd5,
+ LastModified: time.Now().UTC().Format(CREATE_TIME_LAYOUT),
+ }
+
+ err = s.MetaStorage.PutObjectPart(ctx, multipart, part)
+ if err != nil {
+ log.Errorln("failed to put object part. err:", err)
+ // because the backend will delete object part that has the same part id in next upload, we return error directly here
+ return err
+ }
+
+ uploadResponse.ETag = calculatedMd5
+
+ log.Infoln("UploadPart upload part successfully.")
+ return nil
+}
+
+func (s *s3Service) CompleteMultipartUpload(ctx context.Context, in *pb.CompleteMultipartRequest, out *pb.CompleteMultipartResponse) error {
+ log.Info("CompleteMultipartUpload is called in s3 service.")
+ bucketName := in.BucketName
+ objectKey := in.ObjectKey
+ uploadId := in.UploadId
+
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return err
+ }
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, bucketName, true)
+ if err != nil {
+ log.Errorf("failed to get bucket from meta stoarge. err:", err)
+ return err
+ }
+
+ if !isAdmin {
+ switch bucket.Acl.CannedAcl {
+ case "public-read-write":
+ break
+ default:
+ if bucket.TenantId != tenantId {
+ log.Errorln("bucket owner is not equal to request owner.")
+ err = ErrBucketAccessForbidden
+ return err
+ }
+ }
+ }
+
+ multipart, err := s.MetaStorage.GetMultipart(bucketName, objectKey, uploadId)
+ if err != nil {
+ log.Errorln("failed to get multipart info. err:", err)
+ return err
+ }
+
+ md5Writer := md5.New()
+ var totalSize int64 = 0
+ for i := 0; i < len(in.CompleteParts); i++ {
+ if in.CompleteParts[i].PartNumber != int64(i+1) {
+ log.Errorln("wrong order for part number. ")
+ err = ErrInvalidPart
+ return err
+ }
+ part, ok := multipart.Parts[i+1]
+ if !ok {
+ log.Errorln("missed object part. partno:", i)
+ err = ErrInvalidPart
+ return err
+ }
+
+ if part.Etag != in.CompleteParts[i].ETag {
+ log.Errorln("part etag in meta store is not the same with client's part, partno:", i)
+ err = ErrInvalidPart
+ return err
+ }
+ var etagBytes []byte
+ etagBytes, err = hex.DecodeString(part.Etag)
+ if err != nil {
+ log.Errorln("failed to decode etag string. err:", err)
+ err = ErrInvalidPart
+ return err
+ }
+ part.Offset = totalSize
+ totalSize += part.Size
+ md5Writer.Write(etagBytes)
+ }
+ eTag := hex.EncodeToString(md5Writer.Sum(nil))
+ eTag += "-" + strconv.Itoa(len(in.CompleteParts))
+
+ backendName := bucket.DefaultLocation
+ backend, err := utils.GetBackend(ctx, s.backendClient, backendName)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+ sd, err := driver.CreateStorageDriver(backend.Type, backend)
+ if err != nil {
+ log.Errorln("failed to create storage. err:", err)
+ return nil
+ }
+
+ parts := make([]model.Part, 0)
+ completeUpload := &model.CompleteMultipartUpload{}
+ for _, part := range in.CompleteParts {
+ parts = append(parts, model.Part{
+ PartNumber: part.PartNumber,
+ ETag: part.ETag,
+ })
+ }
+ completeUpload.Parts = parts
+ _, err = sd.CompleteMultipartUpload(ctx, &pb.MultipartUpload{
+ Bucket: bucketName,
+ Key: objectKey,
+ UploadId: uploadId,
+ ObjectId: multipart.ObjectId,
+ Location: multipart.Metadata.Location,
+ Tier: multipart.Metadata.Tier,
+ }, completeUpload)
+ if err != nil {
+ log.Errorln("failed to complete multipart. err:", err)
+ return err
+ }
+
+ // TODO: if versioning is enabled, not need to delete oldObj
+ oldObj, err := s.MetaStorage.GetObject(ctx, bucketName, objectKey, "", false)
+ if err != nil && err != ErrNoSuchKey {
+ log.Errorf("get object[%s] failed, err:%v\n", objectKey, err)
+ return ErrInternalError
+ }
+ log.Debugf("bucketName=%s,objectKey=%s,version=%s,existObj=%v, err=%v\n", bucketName, objectKey, in.SourceVersionID, oldObj, err)
+
+ // Add to objects table
+ contentType := multipart.Metadata.ContentType
+ object := &pb.Object{
+ BucketName: bucketName,
+ ObjectKey: objectKey,
+ TenantId: multipart.Metadata.TenantId,
+ UserId: multipart.Metadata.UserId,
+ ContentType: contentType,
+ ObjectId: multipart.ObjectId,
+ LastModified: time.Now().UTC().Unix(),
+ Etag: eTag,
+ DeleteMarker: false,
+ CustomAttributes: multipart.Metadata.Attrs,
+ Type: ObjectTypeNormal,
+ Tier: multipart.Metadata.Tier,
+ Size: totalSize,
+ Location: multipart.Metadata.Location,
+ Acl: &multipart.Metadata.Acl,
+ }
+
+ if in.RequestType != utils.RequestType_Lifecycle {
+ err = s.MetaStorage.PutObject(ctx, &Object{Object: object}, oldObj, &multipart, nil, true)
+ if err != nil {
+ log.Errorf("failed to put object meta[object:%+v, oldObj:%+v]. err:%v\n", object, oldObj, err)
+ // TODO: consistent check & clean
+ return ErrDBError
+ }
+ } else {
+ err = s.MetaStorage.UpdateObject4Lifecycle(ctx, oldObj, &Object{Object: object}, &multipart)
+ if err != nil {
+ log.Errorln("failed to put object meta. err:", err)
+ // delete new object, lifecycle will try again in the next schedule round
+ s.cleanObject(ctx, &Object{Object: object}, sd)
+ return err
+ }
+ }
+
+ log.Infoln("CompleteMultipartUpload upload part successfully.")
+ return nil
+}
+
+func (s *s3Service) AbortMultipartUpload(ctx context.Context, in *pb.AbortMultipartRequest, out *pb.BaseResponse) error {
+ log.Info("AbortMultipartUpload is called in s3 service.")
+ bucketName := in.BucketName
+ objectKey := in.ObjectKey
+ uploadId := in.UploadId
+
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil && isAdmin == false {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return err
+ }
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, bucketName, true)
+ if err != nil {
+ log.Errorln("failed to get bucket from meta storage. err:", err)
+ return err
+ }
+
+ if !isAdmin {
+ switch bucket.Acl.CannedAcl {
+ case "public-read-write":
+ break
+ default:
+ if bucket.TenantId != tenantId {
+ log.Errorln("bucket owner is not equal to request owner.")
+ return ErrBucketAccessForbidden
+ }
+ }
+ }
+
+ multipart, err := s.MetaStorage.GetMultipart(bucketName, objectKey, uploadId)
+ if err != nil {
+ log.Errorln("failed to get multipart info. err:", err)
+ return err
+ }
+
+ backendName := bucket.DefaultLocation
+ backend, err := utils.GetBackend(ctx, s.backendClient, backendName)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+ sd, err := driver.CreateStorageDriver(backend.Type, backend)
+ if err != nil {
+ log.Errorln("failed to create storage. err:", err)
+ return err
+ }
+
+ err = sd.AbortMultipartUpload(ctx, &pb.MultipartUpload{
+ Bucket: bucketName,
+ Key: objectKey,
+ UploadId: uploadId,
+ ObjectId: multipart.ObjectId})
+ if err != nil {
+ log.Errorln("failed to abort multipart. err:", err)
+ return err
+ }
+
+ err = s.MetaStorage.DeleteMultipart(ctx, multipart)
+ if err != nil {
+ log.Errorln("failed to delete multipart. err:", err)
+ return err
+ }
+
+ log.Infoln("Abort multipart successfully.")
+ return nil
+}
+
+func (s *s3Service) ListObjectParts(ctx context.Context, in *pb.ListObjectPartsRequest, out *pb.ListObjectPartsResponse) error {
+ log.Info("ListObjectParts is called in s3 service.")
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucketName := in.BucketName
+ objectKey := in.ObjectKey
+ uploadId := in.UploadId
+
+ multipart, err := s.MetaStorage.GetMultipart(bucketName, objectKey, uploadId)
+ if err != nil {
+ log.Errorln("failed to get multipart info. err:", err)
+ return err
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return err
+ }
+
+ if !isAdmin {
+ switch multipart.Metadata.Acl.CannedAcl {
+ case "public-read", "public-read-write":
+ break
+ default:
+ if multipart.Metadata.TenantId != tenantId {
+ err = ErrAccessDenied
+ return err
+ }
+ }
+ }
+
+ out.Initiator = &pb.Owner{Id: multipart.Metadata.InitiatorId, DisplayName: multipart.Metadata.InitiatorId}
+ out.Owner = &pb.Owner{Id: multipart.Metadata.TenantId, DisplayName: multipart.Metadata.TenantId}
+ out.MaxParts = int64(in.MaxParts)
+ out.Parts = make([]*pb.Part, 0)
+ for i := in.PartNumberMarker + 1; i <= MAX_PART_NUMBER; i++ {
+ if p, ok := multipart.Parts[int(i)]; ok {
+ out.Parts = append(out.Parts, &pb.Part{
+ PartNumber: i,
+ ETag: "\"" + p.Etag + "\"",
+ Size: p.Size,
+ LastModified: p.LastModified,
+ })
+
+ if int64(len(out.Parts)) > in.MaxParts {
+ break
+ }
+ }
+ }
+ if int64(len(out.Parts)) == in.MaxParts+1 {
+ out.IsTruncated = true
+ out.NextPartNumberMarker = out.Parts[out.MaxParts].PartNumber
+ out.Parts = out.Parts[:in.MaxParts]
+ }
+ out.PartNumberMarker = in.PartNumberMarker
+
+ log.Infof("list object part successfully. ")
+
+ return nil
+}
+
+func (s *s3Service) CopyObjPart(ctx context.Context, in *pb.CopyObjPartRequest, out *pb.CopyObjPartResponse) error {
+ log.Info("CopyObjPart is called in s3 service.")
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ srcBucketName := in.SourceBucket
+ srcObjectName := in.SourceObject
+ targetBucketName := in.TargetBucket
+ targetObjectName := in.TargetObject
+ uploadId := in.UploadID
+ partId := in.PartID
+ size := in.ReadLength
+ offset := in.ReadOffset
+
+ srcBucket, err := s.MetaStorage.GetBucket(ctx, srcBucketName, true)
+ if err != nil {
+ log.Errorln("get bucket failed with err:", err)
+ return err
+ }
+ srcObject, err := s.MetaStorage.GetObject(ctx, srcBucketName, srcObjectName, "", true)
+ if err != nil {
+ log.Errorln("failed to get object info from meta storage. err:", err)
+ return err
+ }
+ targetBucket, err := s.MetaStorage.GetBucket(ctx, targetBucketName, true)
+ if err != nil {
+ log.Errorln("get bucket failed with err:", err)
+ return err
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil {
+ log.Error("get tenant id failed, err:", err)
+ err = ErrInternalError
+ return err
+ }
+
+ if !isAdmin {
+ //check source object acl
+ switch srcObject.Acl.CannedAcl {
+ case "public-read", "public-read-write":
+ break
+ default:
+ if srcObject.TenantId != tenantId {
+ err = ErrAccessDenied
+ return err
+ }
+ }
+ //check target acl
+ switch targetBucket.Acl.CannedAcl {
+ case "public-read-write":
+ break
+ default:
+ if targetBucket.TenantId != tenantId {
+ err = ErrBucketAccessForbidden
+ return err
+ }
+ } // TODO policy and fancy ACL
+ }
+
+ backendName := srcBucket.DefaultLocation
+ if srcObject.Location != "" {
+ backendName = srcObject.Location
+ }
+ srcBackend, err := utils.GetBackend(ctx, s.backendClient, backendName)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+ srcSd, err := driver.CreateStorageDriver(srcBackend.Type, srcBackend)
+ if err != nil {
+ log.Errorln("failed to create storage driver. err:", err)
+ return err
+ }
+ targetBackendName := targetBucket.DefaultLocation
+ if in.TargetLocation != "" {
+ targetBackendName = in.TargetLocation
+ }
+ targetBackend, err := utils.GetBackend(ctx, s.backendClient, targetBackendName)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+ targetSd, err := driver.CreateStorageDriver(targetBackend.Type, targetBackend)
+ if err != nil {
+ log.Errorln("failed to create storage driver. err:", err)
+ return err
+ }
+ reader, err := srcSd.Get(ctx, srcObject.Object, offset, offset+size-1)
+ if err != nil {
+ log.Errorln("failed to get data reader. err:", err)
+ return err
+ }
+ limitedDataReader := io.LimitReader(reader, size)
+ md5Writer := md5.New()
+ dataReader := io.TeeReader(limitedDataReader, md5Writer)
+
+ multipart, err := s.MetaStorage.GetMultipart(targetBucketName, targetObjectName, uploadId)
+ if err != nil {
+ log.Errorln("failed to get multipart. err:", err)
+ return err
+ }
+ _, err = targetSd.UploadPart(ctx, dataReader, &pb.MultipartUpload{
+ Bucket: targetBucketName,
+ Key: targetObjectName,
+ UploadId: uploadId,
+ ObjectId: multipart.ObjectId},
+ partId, size)
+ if err != nil {
+ log.Errorln("failed to upload part to backend. err:", err)
+ return err
+ }
+ calculatedMd5 := hex.EncodeToString(md5Writer.Sum(nil))
+ part := Part{
+ PartNumber: int(partId),
+ Size: size,
+ ObjectId: multipart.ObjectId,
+ Etag: calculatedMd5,
+ LastModified: time.Now().UTC().Format(CREATE_TIME_LAYOUT),
+ }
+
+ err = s.MetaStorage.PutObjectPart(ctx, multipart, part)
+ if err != nil {
+ log.Errorln("failed to put object part. err:", err)
+ // because the backend will delete object part that has the same part id in next upload, we return error directly here
+ return err
+ }
+
+ out.Etag = calculatedMd5
+ out.LastModified = time.Now().UTC().Unix()
+
+ log.Infoln("copy object part successfully.")
+ return nil
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "io"
+ "io/ioutil"
+ "net/url"
+ "time"
+
+ "github.com/journeymidnight/yig/helper"
+ "github.com/micro/go-micro/metadata"
+ "github.com/opensds/multi-cloud/api/pkg/common"
+ "github.com/opensds/multi-cloud/api/pkg/utils/constants"
+ . "github.com/opensds/multi-cloud/s3/error"
+ dscommon "github.com/opensds/multi-cloud/s3/pkg/datastore/common"
+ "github.com/opensds/multi-cloud/s3/pkg/datastore/driver"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ . "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ meta "github.com/opensds/multi-cloud/s3/pkg/meta/types"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/util"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+var ChunkSize int = 2048
+
+func (s *s3Service) CreateObject(ctx context.Context, in *pb.Object, out *pb.BaseResponse) error {
+ log.Infoln("CreateObject is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) UpdateObject(ctx context.Context, in *pb.Object, out *pb.BaseResponse) error {
+ log.Infoln("PutObject is called in s3 service.")
+
+ return nil
+}
+
+type DataStreamRecv interface {
+ Recv() (*pb.PutDataStream, error)
+}
+
+type StreamReader struct {
+ in DataStreamRecv
+ req *pb.PutDataStream
+ curr int
+}
+
+func (dr *StreamReader) Read(p []byte) (n int, err error) {
+ left := len(p)
+ for left > 0 {
+ if dr.curr == 0 || (dr.req != nil && dr.curr == len(dr.req.Data)) {
+ dr.req, err = dr.in.Recv()
+ if err != nil && err != io.EOF {
+ log.Errorln("failed to recv data with err:", err)
+ return
+ }
+ if dr.req == nil || len(dr.req.Data) == 0 {
+ log.Errorln("no data left to read.")
+ err = io.EOF
+ return
+ }
+ dr.curr = 0
+ }
+
+ copyLen := 0
+ if len(dr.req.Data)-dr.curr > left {
+ copyLen = left
+ } else {
+ copyLen = len(dr.req.Data) - dr.curr
+ }
+ log.Traceln("copy len:", copyLen)
+ copy(p[n:], dr.req.Data[dr.curr:(dr.curr+copyLen)])
+ dr.curr += copyLen
+ left -= copyLen
+ n += copyLen
+ }
+ return
+}
+
+func (s *s3Service) removeObjectFromBackend(ctx context.Context, sd driver.StorageDriver, obj *pb.DeleteObjectInput) error {
+ if obj != nil {
+ err := sd.Delete(ctx, obj)
+ if err != nil {
+ log.Errorln("failed to delete written object. err:", err)
+ return err
+ }
+ }
+ return nil
+}
+
+func (s *s3Service) PutObject(ctx context.Context, in pb.S3_PutObjectStream) error {
+ log.Infoln("PutObject is called in s3 service.")
+
+ var err error
+ result := &pb.PutObjectResponse{}
+ defer func() {
+ result.ErrorCode = GetErrCode(err)
+ in.SendMsg(result)
+ }()
+
+ isAdmin, tenantId, userId, err := util.GetCredentialFromCtx(ctx)
+ if err != nil {
+ log.Errorln("failed to get credential info. err:", err)
+ return nil
+ }
+
+ req := &pb.PutObjectRequest{}
+ err = in.RecvMsg(req)
+ if err != nil {
+ log.Errorln("failed to get msg with err:", err)
+ return ErrInternalError
+ }
+
+ log.Infof("*********bucket:%s,key:%s,size:%d\n", req.BucketName, req.ObjectKey, req.Size)
+ bucket, err := s.MetaStorage.GetBucket(ctx, req.BucketName, true)
+ if err != nil {
+ log.Errorln("get bucket failed with err:", err)
+ return err
+ }
+
+ if !isAdmin {
+ switch bucket.Acl.CannedAcl {
+ case "public-read-write":
+ break
+ default:
+ if bucket.TenantId != tenantId {
+ err = ErrBucketAccessForbidden
+ return err
+ }
+ }
+ }
+
+ // get old object meta if it exist, this is not needed if versioning is enabled
+ oldObj, err := s.MetaStorage.GetObject(ctx, bucket.Name, req.ObjectKey, "", false)
+ if err != nil && err != ErrNoSuchKey {
+ log.Errorf("get object[%s] failed, err:%v\n", req.ObjectKey, err)
+ return ErrInternalError
+ }
+ log.Debugf("existObj=%v, err=%v\n", oldObj, err)
+
+ data := &StreamReader{in: in}
+ var limitedDataReader io.Reader
+ if req.Size > 0 { // request.ContentLength is -1 if length is unknown
+ limitedDataReader = io.LimitReader(data, req.Size)
+ } else {
+ limitedDataReader = data
+ }
+
+ actualSize := req.Size
+ // encrypt if needed
+ if bucket.ServerSideEncryption.SseType == "SSE" {
+ byteArr, _ := ioutil.ReadAll(limitedDataReader)
+ _, encBuf := utils.EncryptWithAES256RandomKey(byteArr, bucket.ServerSideEncryption.EncryptionKey)
+ reader := bytes.NewReader(encBuf)
+ limitedDataReader = io.LimitReader(reader, int64(binary.Size(encBuf)))
+ req.Size = int64(binary.Size(encBuf))
+ }
+
+ backendName := bucket.DefaultLocation
+ if req.Location != "" {
+ backendName = req.Location
+ }
+ backend, err := utils.GetBackend(ctx, s.backendClient, backendName)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+
+ log.Infoln("bucket location:", req.Location, " backendtype:", backend.Type, " endpoint:", backend.Endpoint)
+ bodyMd5 := req.Attrs["md5Sum"]
+ ctx = context.Background()
+ ctx = context.WithValue(ctx, dscommon.CONTEXT_KEY_SIZE, req.Size)
+ ctx = context.WithValue(ctx, dscommon.CONTEXT_KEY_MD5, bodyMd5)
+ sd, err := driver.CreateStorageDriver(backend.Type, backend)
+ if err != nil {
+ log.Errorln("failed to create storage. err:", err)
+ return err
+ }
+ obj := &pb.Object{BucketName: req.BucketName, ObjectKey: req.ObjectKey}
+ if oldObj != nil && oldObj.Location == backendName {
+ obj.StorageMeta = oldObj.StorageMeta
+ obj.ObjectId = oldObj.ObjectId
+ }
+ res, err := sd.Put(ctx, limitedDataReader, obj)
+ if err != nil {
+ log.Errorln("failed to put data. err:", err)
+ return err
+ }
+
+ obj.BucketName = req.BucketName
+ obj.ObjectKey = req.ObjectKey
+ obj.Acl = req.Acl
+ obj.TenantId = tenantId
+ obj.UserId = userId
+ obj.ObjectId = res.ObjectId
+ obj.LastModified = time.Now().UTC().Unix()
+ obj.Etag = res.Etag
+ obj.ContentType = req.ContentType
+ obj.DeleteMarker = false
+ obj.CustomAttributes = req.Attrs
+ obj.Type = meta.ObjectTypeNormal
+ obj.Tier = utils.Tier1 // Currently only support tier1
+ obj.StorageMeta = res.Meta
+ obj.Size = actualSize
+ obj.EncSize = req.Size
+ obj.Location = backendName
+
+ object := &meta.Object{Object: obj}
+
+ result.Md5 = res.Etag
+ result.LastModified = object.LastModified
+
+ err = s.MetaStorage.PutObject(ctx, object, oldObj, nil, nil, true)
+ if err != nil {
+ log.Errorf("failed to put object meta[object:%+v, oldObj:%+v]. err:%v\n", object, oldObj, err)
+ // TODO: consistent check & clean
+ return ErrDBError
+ }
+
+ return nil
+}
+
+func (s *s3Service) checkGetObjectRights(ctx context.Context, isAdmin bool, tenantId string, bucket *pb.Bucket, object *pb.Object) (err error) {
+ if !isAdmin {
+ switch object.Acl.CannedAcl {
+ case "public-read", "public-read-write":
+ break
+ default:
+ if object.TenantId != tenantId {
+ err = ErrAccessDenied
+ return
+ }
+ }
+ }
+ return
+}
+
+func (s *s3Service) GetObjectMeta(ctx context.Context, in *pb.Object, out *pb.GetObjectMetaResult) error {
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, in.BucketName, true)
+ if err != nil {
+ log.Errorln("failed to get bucket from meta storage. err:", err)
+ return err
+ }
+
+ object, err := s.MetaStorage.GetObject(ctx, in.BucketName, in.ObjectKey, "", true)
+ if err != nil {
+ log.Errorln("failed to get object info from meta storage. err:", err)
+ return nil
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return nil
+ }
+
+ err = s.checkGetObjectRights(ctx, isAdmin, tenantId, bucket.Bucket, object.Object)
+ if err != nil {
+ log.Errorln("failed to check source object rights. err:", err)
+ return err
+ }
+
+ out.Object = object.Object
+ object.StorageClass, _ = GetNameFromTier(object.Tier, utils.OSTYPE_OPENSDS)
+ return nil
+}
+
+func (s *s3Service) GetObject(ctx context.Context, req *pb.GetObjectInput, stream pb.S3_GetObjectStream) error {
+ log.Infoln("GetObject is called in s3 service.")
+ bucketName := req.Bucket
+ bucket, err := s.MetaStorage.GetBucket(ctx, bucketName, true)
+ if err != nil {
+ log.Errorln("failed to get bucket from meta storage. err:", err)
+ return err
+ }
+ if bucket.ServerSideEncryption != nil && bucket.ServerSideEncryption.SseType == "SSE" {
+ return GetEncObject(ctx, req, stream, s)
+ } else {
+ return GetObject(ctx, req, stream, s)
+ }
+}
+
+func GetObject(ctx context.Context, req *pb.GetObjectInput, stream pb.S3_GetObjectStream, s *s3Service) error {
+ log.Infoln("GetObject is called in s3 service.")
+ bucketName := req.Bucket
+ objectName := req.Key
+ offset := req.Offset
+ length := req.Length
+
+ var err error
+ getObjRes := &pb.GetObjectResponse{}
+ defer func() {
+ getObjRes.ErrorCode = GetErrCode(err)
+ stream.SendMsg(getObjRes)
+ }()
+
+ object, err := s.MetaStorage.GetObject(ctx, bucketName, objectName, "", true)
+ if err != nil {
+ log.Errorln("failed to get object info from meta storage. err:", err)
+ return err
+ }
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, bucketName, true)
+ if err != nil {
+ log.Errorln("failed to get bucket from meta storage. err:", err)
+ return err
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil && isAdmin == false {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return nil
+ }
+
+ err = s.checkGetObjectRights(ctx, isAdmin, tenantId, bucket.Bucket, object.Object)
+ if err != nil {
+ log.Errorln("failed to check source object rights. err:", err)
+ return err
+ }
+
+ backendName := bucket.DefaultLocation
+ if object.Location != "" {
+ backendName = object.Location
+ }
+ // if this object has only one part
+ backend, err := utils.GetBackend(ctx, s.backendClient, backendName)
+ if err != nil {
+ log.Errorln("unable to get backend. err:", err)
+ return err
+ }
+ sd, err := driver.CreateStorageDriver(backend.Type, backend)
+ if err != nil {
+ log.Errorln("failed to create storage driver. err:", err)
+ return err
+ }
+ log.Infof("get object offset %v, length %v", offset, length)
+ reader, err := sd.Get(ctx, object.Object, offset, offset+length-1)
+ if err != nil {
+ log.Errorln("failed to get data. err:", err)
+ return err
+ }
+
+ eof := false
+ left := object.Size
+ buf := make([]byte, ChunkSize)
+ for !eof && left > 0 {
+ n, err := reader.Read(buf)
+ if err != nil && err != io.EOF {
+ log.Errorln("failed to read, err:", err)
+ break
+ }
+ // From https://golang.org/pkg/io/, a Reader returning a non-zero number of bytes at the end of the input stream
+ // may return either err == EOF or err == nil. The next Read should return 0, EOF.
+ // If err is equal to io.EOF, a non-zero number of bytes may be returned.
+ if err == io.EOF {
+ log.Debugln("finished read")
+ eof = true
+ }
+ // From https://golang.org/pkg/io/, there is the following statement.
+ // Implementations of Read are discouraged from returning a zero byte count with a nil error, except when len(p) ==
+ // 0. Callers should treat a return of 0 and nil as indicating that nothing happened; in particular it does not indicate EOF.
+ // If n is equal 0, it indicate that there is no more data to read
+ if n == 0 {
+ log.Infoln("reader return zero bytes.")
+ break
+ }
+
+ err = stream.Send(&pb.GetObjectResponse{ErrorCode: int32(ErrNoErr), Data: buf[0:n]})
+ if err != nil {
+ log.Infof("stream send error: %v\n", err)
+ break
+ }
+ left -= int64(n)
+ }
+
+ log.Infof("get object end, object:%s, left bytes:%d, err:%v\n", objectName, left, err)
+ return err
+}
+
+func GetEncObject(ctx context.Context, req *pb.GetObjectInput, stream pb.S3_GetObjectStream, s *s3Service) error {
+ log.Infoln("GetObject is called in s3 service.")
+ bucketName := req.Bucket
+ objectName := req.Key
+ offset := req.Offset
+ length := req.Length
+
+ var err error
+ getObjRes := &pb.GetObjectResponse{}
+ defer func() {
+ getObjRes.ErrorCode = GetErrCode(err)
+ stream.SendMsg(getObjRes)
+ }()
+
+ object, err := s.MetaStorage.GetObject(ctx, bucketName, objectName, "", true)
+ if err != nil {
+ log.Errorln("failed to get object info from meta storage. err:", err)
+ return err
+ }
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, bucketName, true)
+ if err != nil {
+ log.Errorln("failed to get bucket from meta storage. err:", err)
+ return err
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil && isAdmin == false {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return nil
+ }
+
+ err = s.checkGetObjectRights(ctx, isAdmin, tenantId, bucket.Bucket, object.Object)
+ if err != nil {
+ log.Errorln("failed to check source object rights. err:", err)
+ return err
+ }
+
+ backendName := bucket.DefaultLocation
+ if object.Location != "" {
+ backendName = object.Location
+ }
+ // if this object has only one part
+ backend, err := utils.GetBackend(ctx, s.backendClient, backendName)
+ if err != nil {
+ log.Errorln("unable to get backend. err:", err)
+ return err
+ }
+ sd, err := driver.CreateStorageDriver(backend.Type, backend)
+ if err != nil {
+ log.Errorln("failed to create storage driver. err:", err)
+ return err
+ }
+ if bucket.ServerSideEncryption.SseType == "SSE" {
+ length = object.EncSize
+ }
+ log.Infof("get object offset %v, length %v", offset, length)
+ reader, err := sd.Get(ctx, object.Object, offset, offset+length-1)
+ if err != nil {
+ log.Errorln("failed to get data. err:", err)
+ return err
+ }
+
+ buf := make([]byte, object.EncSize)
+
+ n, err := reader.Read(buf)
+ if err != nil && err != io.EOF {
+ log.Errorln("failed to read, err:", err)
+ return err
+ }
+ // From https://golang.org/pkg/io/, a Reader returning a non-zero number of bytes at the end of the input stream
+ // may return either err == EOF or err == nil. The next Read should return 0, EOF.
+ // If err is equal to io.EOF, a non-zero number of bytes may be returned.
+ if err == io.EOF {
+ log.Debugln("finished read")
+ }
+ // From https://golang.org/pkg/io/, there is the following statement.
+ // Implementations of Read are discouraged from returning a zero byte count with a nil error, except when len(p) ==
+ // 0. Callers should treat a return of 0 and nil as indicating that nothing happened; in particular it does not indicate EOF.
+ // If n is equal 0, it indicate that there is no more data to read
+ if n == 0 {
+ log.Infoln("reader return zero bytes.")
+ return err
+ }
+
+ // decrypt and write
+ decErr, decBytes := utils.DecryptWithAES256(buf[0:object.EncSize], bucket.ServerSideEncryption.EncryptionKey)
+ if decErr != nil {
+ log.Errorln("failed to decrypt data. err:", decErr)
+ return decErr
+ }
+ log.Infoln("successfully decrypted")
+ buf = decBytes
+ n = binary.Size(decBytes)
+
+ err = stream.Send(&pb.GetObjectResponse{ErrorCode: int32(ErrNoErr), Data: buf[0:n]})
+ if err != nil {
+ log.Infof("stream send error: %v\n", err)
+ return err
+ }
+
+ log.Infoln("get object successfully")
+ return err
+}
+
+func (s *s3Service) UpdateObjectMeta(ctx context.Context, in *pb.Object, out *pb.PutObjectResponse) error {
+ log.Infoln("UpdateObjectMeta is called in s3 service.")
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ object, err := s.MetaStorage.GetObject(ctx, in.BucketName, in.ObjectKey, "", true)
+ if err != nil {
+ log.Errorln("failed to get object info from meta storage. err:", err)
+ return err
+ }
+ _, _, _, err = CheckRights(ctx, object.TenantId)
+ if err != nil {
+ log.Errorln("failed to check rights, err:", err)
+ return err
+ }
+
+ err = s.MetaStorage.UpdateObjectMeta(&meta.Object{Object: in})
+ if err != nil {
+ log.Errorf("failed to update object meta storage, err:", err)
+ err = ErrInternalError
+ return err
+ }
+ out.LastModified = in.LastModified
+ out.Md5 = in.Etag
+ out.VersionId = in.GetVersionId()
+
+ return nil
+}
+
+func (s *s3Service) CopyObject(ctx context.Context, in *pb.CopyObjectRequest, out *pb.CopyObjectResponse) error {
+ log.Infoln("CopyObject is called in s3 service.")
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ srcBucketName := in.SrcBucketName
+ srcObjectName := in.SrcObjectName
+ targetBucketName := in.TargetBucketName
+ targetObjectName := in.TargetObjectName
+ targetBucket, err := s.MetaStorage.GetBucket(ctx, targetBucketName, true)
+ if err != nil {
+ log.Errorln("get bucket failed with err:", err)
+ return err
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil {
+ log.Errorf("get credential faied, err:%v\n", err)
+ return err
+ }
+
+ if !isAdmin {
+ switch targetBucket.Acl.CannedAcl {
+ case "public-read-write":
+ break
+ default:
+ if targetBucket.TenantId != tenantId {
+ err = ErrBucketAccessForbidden
+ return err
+ }
+ }
+ }
+
+ srcBucket, err := s.MetaStorage.GetBucket(ctx, srcBucketName, true)
+ if err != nil {
+ log.Errorln("get bucket failed with err:", err)
+ return err
+ }
+ srcObject, err := s.MetaStorage.GetObject(ctx, srcBucketName, srcObjectName, "", true)
+ if err != nil {
+ log.Errorln("failed to get object info from meta storage. err:", err)
+ return err
+ }
+
+ err = s.checkGetObjectRights(ctx, isAdmin, tenantId, srcBucket.Bucket, srcObject.Object)
+ if err != nil {
+ log.Errorln("failed to check source object rights. err:", err)
+ return err
+ }
+
+ backendName := srcBucket.DefaultLocation
+ if srcObject.Location != "" {
+ backendName = srcObject.Location
+ }
+ srcBackend, err := utils.GetBackend(ctx, s.backendClient, backendName)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+ srcSd, err := driver.CreateStorageDriver(srcBackend.Type, srcBackend)
+ if err != nil {
+ log.Errorln("failed to create storage. err:", err)
+ return err
+ }
+
+ targetBackendName := targetBucket.DefaultLocation
+ targetBackend, err := utils.GetBackend(ctx, s.backendClient, targetBackendName)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+ // get old object meta if it exist
+ oldObj, err := s.MetaStorage.GetObject(ctx, targetBucketName, targetObjectName, "", false)
+ if err != nil && err != ErrNoSuchKey {
+ log.Errorf("get object[%s] failed, err:%v\n", targetObjectName, err)
+ return ErrInternalError
+ }
+ log.Debugf("existObj=%v, err=%v\n", oldObj, err)
+ targetSd, err := driver.CreateStorageDriver(targetBackend.Type, targetBackend)
+ if err != nil {
+ log.Errorln("failed to create storage. err:", err)
+ return err
+ }
+ log.Debugf("***ctx:%+v\n", ctx)
+ reader, err := srcSd.Get(ctx, srcObject.Object, 0, srcObject.Size-1)
+ if err != nil {
+ log.Errorln("failed to put data. err:", err)
+ return err
+ }
+ limitedDataReader := io.LimitReader(reader, srcObject.Size)
+
+ targetObject := &pb.Object{
+ ObjectKey: targetObjectName,
+ BucketName: targetBucketName,
+ Size: srcObject.Size,
+ }
+ if oldObj != nil && oldObj.Location == targetBackendName {
+ targetObject.StorageMeta = oldObj.StorageMeta
+ targetObject.ObjectId = oldObj.ObjectId
+ }
+ ctx = context.WithValue(ctx, dscommon.CONTEXT_KEY_SIZE, srcObject.Size)
+ res, err := targetSd.Put(ctx, limitedDataReader, targetObject)
+ if err != nil {
+ log.Errorln("failed to put data. err:", err)
+ return err
+ }
+ if res.Written < srcObject.Size {
+ // TODO: delete incomplete object at backend
+ log.Warnf("write objects, already written(%d), total size(%d)\n", res.Written, srcObject.Size)
+ err = ErrIncompleteBody
+ return err
+ }
+
+ targetObject.Etag = res.Etag
+ targetObject.ObjectId = res.ObjectId
+ targetObject.LastModified = time.Now().UTC().Unix()
+ targetObject.ContentType = srcObject.ContentType
+ targetObject.DeleteMarker = false
+ targetObject.CustomAttributes = srcObject.CustomAttributes
+ targetObject.Type = meta.ObjectTypeNormal
+ targetObject.StorageMeta = res.Meta
+ targetObject.Location = targetBackendName
+ targetObject.TenantId = tenantId
+ // this is the default acl setting
+ targetObject.Acl = &pb.Acl{CannedAcl: "private"}
+ // we only support copy data with sse but not support copy data without sse right now
+ targetObject.ServerSideEncryption = srcObject.ServerSideEncryption
+
+ err = s.MetaStorage.PutObject(ctx, &meta.Object{Object: targetObject}, oldObj, nil, nil, true)
+ if err != nil {
+ log.Errorf("failed to put object meta[object:%+v, oldObj:%+v]. err:%v\n", targetObject, oldObj, err)
+ // TODO: consistent check & clean
+ err = ErrDBError
+ return err
+ }
+
+ out.Md5 = res.Etag
+ out.LastModified = targetObject.LastModified
+
+ log.Infoln("Successfully copy object ", res.Written, " bytes.")
+ return nil
+}
+
+func initTargeObject(ctx context.Context, in *pb.MoveObjectRequest, srcObject *pb.Object) (*pb.Object, error) {
+ md, ok := metadata.FromContext(ctx)
+ if !ok {
+ log.Error("get metadata from ctx failed.")
+ return nil, ErrInternalError
+ }
+
+ targetObject := &pb.Object{
+ ObjectKey: srcObject.ObjectKey,
+ BucketName: srcObject.BucketName,
+ ObjectId: srcObject.ObjectId,
+ Size: srcObject.Size,
+ Etag: srcObject.Etag,
+ Location: srcObject.Location,
+ Tier: srcObject.Tier,
+ TenantId: srcObject.TenantId,
+ UserId: srcObject.UserId,
+ StorageMeta: srcObject.StorageMeta,
+ LastModified: srcObject.LastModified,
+ ContentType: srcObject.ContentType,
+ ServerSideEncryption: srcObject.ServerSideEncryption,
+ Acl: srcObject.Acl,
+ Type: srcObject.Type,
+ DeleteMarker: false,
+ CustomAttributes: md, /* TODO: only reserve http header attr*/
+ }
+
+ if in.TargetTier > 0 {
+ targetObject.Tier = in.TargetTier
+ }
+
+ return targetObject, nil
+}
+
+// This is for lifecycle management.
+func (s *s3Service) MoveObject(ctx context.Context, in *pb.MoveObjectRequest, out *pb.MoveObjectResponse) error {
+ log.Infoln("MoveObject is called in s3 service.")
+
+ err := s.checkMoveRequest(ctx, in)
+ if err != nil {
+ return err
+ }
+
+ srcObject, err := s.MetaStorage.GetObject(ctx, in.SrcBucket, in.SrcObject, in.SrcObjectVersion, true)
+ if err != nil {
+ log.Errorf("failed to get object[%s] of bucket[%s]. err:%v\n", in.SrcObject, in.SrcBucket, err)
+ return err
+ }
+
+ targetObject, err := initTargeObject(ctx, in, srcObject.Object)
+ if err != nil {
+ log.Errorf("failed to get init target obejct. err:%v\n", err)
+ return err
+ }
+
+ var srcSd, targetSd driver.StorageDriver
+ var srcBucket, targetBucket *types.Bucket
+ srcBucket, err = s.MetaStorage.GetBucket(ctx, in.SrcBucket, true)
+ if err != nil {
+ log.Errorf("get source bucket[%s] failed with err:%v", in.SrcBucket, err)
+ return err
+ }
+
+ srcBackend, err := utils.GetBackend(ctx, s.backendClient, srcObject.Location)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+ srcSd, err = driver.CreateStorageDriver(srcBackend.Type, srcBackend)
+ if err != nil {
+ log.Errorln("failed to create storage. err:", err)
+ return err
+ }
+
+ if in.MoveType == utils.MoveType_ChangeStorageTier {
+ log.Infof("chagne storage class of %s\n", targetObject.ObjectKey)
+ // just change storage tier
+ targetBucket = srcBucket
+ className, err := GetNameFromTier(in.TargetTier, srcBackend.Type)
+ if err != nil {
+ return ErrInternalError
+ }
+ err = srcSd.ChangeStorageClass(ctx, targetObject, &className)
+ if err != nil {
+ log.Errorf("change storage class of object[%s] failed, err:%v\n", targetObject.ObjectKey, err)
+ return err
+ }
+ newObj := &meta.Object{Object: targetObject}
+ err = s.MetaStorage.UpdateObject4Lifecycle(ctx, srcObject, newObj, nil)
+ } else {
+ // need move data, get target location first
+ if in.MoveType == utils.MoveType_ChangeLocation {
+ targetBucket = srcBucket
+ targetObject.Location = in.TargetLocation
+ log.Infof("move %s cross backends, srcBackend=%s, targetBackend=%s, targetTier=%d\n",
+ srcObject.ObjectKey, srcObject.Location, targetObject.Location, targetObject.Tier)
+ } else { // MoveType_MoveCrossBuckets
+ log.Infof("move %s from bucket[%s] to bucket[%s]\n", targetObject.ObjectKey, srcObject.BucketName,
+ targetObject.BucketName)
+ targetBucket, err = s.MetaStorage.GetBucket(ctx, in.TargetBucket, true)
+ if err != nil {
+ log.Errorf("get bucket[%s] failed with err:%v\n", in.TargetBucket, err)
+ return err
+ }
+ targetObject.ObjectKey = in.TargetObject
+ targetObject.Location = targetBucket.DefaultLocation
+ targetObject.BucketName = targetBucket.Name
+ log.Infof("move %s cross buckets, targetBucket=%s, targetBackend=%s, targetTier=%d\n",
+ srcObject.ObjectKey, targetObject.BucketName, targetObject.Location, targetObject.Tier)
+ }
+
+ // get storage driver
+ targetBackend, err := utils.GetBackend(ctx, s.backendClient, targetObject.Location)
+ if err != nil {
+ log.Errorln("failed to get backend client with err:", err)
+ return err
+ }
+ targetSd, err = driver.CreateStorageDriver(targetBackend.Type, targetBackend)
+ if err != nil {
+ log.Errorln("failed to create storage. err:", err)
+ return err
+ }
+
+ // copy data from one backend to another
+ err = s.copyData(ctx, srcSd, targetSd, srcObject.Object, targetObject)
+ if err != nil {
+ log.Errorf("failed to copy object[%s], err:%v", srcObject.ObjectKey, err)
+ return err
+ }
+ newObj := &meta.Object{Object: targetObject}
+ if srcObject.Etag != targetObject.Etag {
+ log.Errorf("data integrity check failed, etag of source object is %s, etag of target object is:%s\n",
+ srcObject.Etag, targetObject.Etag)
+ // if failed, delete target object
+ s.cleanObject(ctx, newObj, targetSd)
+ return err
+ }
+ out.Md5 = targetObject.Etag
+ out.LastModified = targetObject.LastModified
+ // update object meta data
+ err = s.MetaStorage.UpdateObject4Lifecycle(ctx, srcObject, newObj, nil)
+ if err != nil {
+ log.Errorln("failed to update meta data after copy, err:", err)
+ // if failed, delete target object
+ s.cleanObject(ctx, newObj, targetSd)
+ return err
+ }
+ // delete source object
+ s.cleanObject(ctx, srcObject, srcSd)
+ log.Infof("delete source object[key=%s]\n", srcObject.ObjectKey)
+ }
+
+ log.Infoln("MoveObject is finished.")
+
+ return nil
+}
+
+func (s *s3Service) copyData(ctx context.Context, srcSd, targetSd driver.StorageDriver, srcObj, targetObj *pb.Object) error {
+ log.Infof("copy object data")
+ reader, err := srcSd.Get(ctx, srcObj, 0, srcObj.Size-1)
+ if err != nil {
+ log.Errorln("failed to get data. err:", err)
+ return err
+ }
+ limitedDataReader := io.LimitReader(reader, srcObj.Size)
+ res, err := targetSd.Put(ctx, limitedDataReader, targetObj)
+ if err != nil {
+ log.Errorln("failed to put data. err:", err)
+ return err
+ }
+ log.Infoln("Successfully copy ", res.Written, " bytes.")
+
+ targetObj.Etag = res.Etag
+ targetObj.ObjectId = res.ObjectId
+ targetObj.StorageMeta = res.Meta
+
+ return nil
+}
+
+func (s *s3Service) checkMoveRequest(ctx context.Context, in *pb.MoveObjectRequest) (err error) {
+ log.Infoln("check copy request")
+
+ if in.SrcBucket == "" || in.SrcObject == "" {
+ log.Errorf("invalid copy source.")
+ err = ErrInvalidCopySource
+ return
+ }
+
+ switch in.MoveType {
+ case utils.MoveType_ChangeStorageTier:
+ if !validTier(in.TargetTier) {
+ log.Error("cannot copy object to it's self.")
+ err = ErrInvalidCopyDest
+ }
+ case utils.MoveType_ChangeLocation:
+ if in.TargetLocation == "" {
+ log.Errorf("no target lcoation provided for change location copy")
+ err = ErrInvalidCopyDest
+ }
+ // in.TargetTier > 0 means need to change storage class
+ if in.TargetTier > 0 {
+ if !validTier(in.TargetTier) {
+ log.Error("cannot copy object to it's self.")
+ err = ErrInvalidCopyDest
+ }
+ }
+ case utils.MoveType_MoveCrossBuckets:
+ default:
+ // copy cross buckets as default
+ if in.TargetObject == "" || in.TargetBucket == "" {
+ log.Errorf("invalid copy target")
+ err = ErrInvalidCopyDest
+ return
+ }
+ // in.TargetTier > 0 means need to change storage class
+ if in.TargetTier > 0 {
+ if !validTier(in.TargetTier) {
+ log.Error("cannot copy object to it's self.")
+ err = ErrInvalidCopyDest
+ }
+ }
+ }
+
+ log.Infof("MoveType:%d, srcObject:%v\n", in.MoveType, in.SrcObject)
+ return
+}
+
+// When bucket versioning is Disabled/Enabled/Suspended, and request versionId is set/unset:
+//
+// | | with versionId | without versionId |
+// |-----------|------------------------------|--------------------------------------------------------|
+// | Disabled | error | remove object |
+// | Enabled | remove corresponding version | add a delete marker |
+// | Suspended | remove corresponding version | remove null version object(if exists) and add a |
+// | | | null version delete marker |
+//
+// See http://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html
+func (s *s3Service) DeleteObject(ctx context.Context, in *pb.DeleteObjectInput, out *pb.DeleteObjectOutput) error {
+ log.Infoln("DeleteObject is called in s3 service, bucket:%s, key:%s, version:%s", in.Bucket, in.Key, in.VersioId)
+
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, in.Bucket, true)
+ if err != nil {
+ log.Errorln("get bucket failed with err:", err)
+ return nil
+ }
+
+ object, err := s.MetaStorage.GetObject(ctx, in.Bucket, in.Key, in.VersioId, true)
+ if err != nil {
+ log.Errorln("failed to get object info from meta storage. err:", err)
+ return err
+ }
+ isAdmin, tenantId, _, err := CheckRights(ctx, object.TenantId)
+ if err != nil {
+ log.Errorf("no rights to access the object[%s]\n", object.ObjectKey)
+ return nil
+ }
+
+ // administrator can delete any resource
+ if isAdmin == false {
+ switch bucket.Acl.CannedAcl {
+ case "public-read-write":
+ break
+ default:
+ if bucket.TenantId != tenantId && tenantId != "" {
+ log.Errorf("delete object failed: tenant[id=%s] has no access right.", tenantId)
+ err = ErrBucketAccessForbidden
+ return nil
+ }
+ } // TODO policy and fancy ACL
+ }
+
+ switch bucket.Versioning.Status {
+ case utils.VersioningDisabled:
+ err = s.removeObject(ctx, bucket, object)
+ case utils.VersioningEnabled:
+ // TODO: versioning
+ err = ErrInternalError
+ case utils.VersioningSuspended:
+ // TODO: versioning
+ err = ErrInternalError
+ default:
+ log.Errorf("versioing of bucket[%s] is invalid:%s\n", bucket.Name, bucket.Versioning)
+ err = ErrInternalError
+ }
+
+ // TODO: need to refresh cache if it is enabled
+
+ return nil
+}
+
+func (s *s3Service) removeObject(ctx context.Context, bucket *meta.Bucket, obj *Object) error {
+ if obj == nil {
+ log.Infof("no need remove")
+ return nil
+ }
+ log.Infof("remove object[%s] from bucket[%s]\n", obj.ObjectKey, bucket.Name)
+ backendName := bucket.DefaultLocation
+ if obj.Location != "" {
+ backendName = obj.Location
+ }
+ backend, err := utils.GetBackend(ctx, s.backendClient, backendName)
+ if err != nil {
+ log.Errorln("failed to get backend with err:", err)
+ return err
+ }
+ sd, err := driver.CreateStorageDriver(backend.Type, backend)
+ if err != nil {
+ log.Errorln("failed to create storage, err:", err)
+ return err
+ }
+
+ // mark object as deleted
+ err = s.MetaStorage.MarkObjectAsDeleted(ctx, obj)
+ if err != nil {
+ log.Errorln("failed to mark object as deleted, err:", err)
+ return err
+ }
+
+ // delete object data in backend
+ err = sd.Delete(ctx, &pb.DeleteObjectInput{Bucket: bucket.Name, Key: obj.ObjectKey, VersioId: obj.VersionId,
+ ETag: obj.Etag, StorageMeta: obj.StorageMeta, ObjectId: obj.ObjectId})
+ if err != nil {
+ log.Errorf("failed to delete obejct[%s,versionid=%s] from backend storage, err:%v\n", obj.ObjectKey, obj.VersionId, err)
+ return err
+ } else {
+ log.Infof("delete obejct[%s,versionid=%s] from backend storage successfully.\n", obj.ObjectKey, obj.VersionId)
+ }
+
+ // delete object meta data from database
+ err = s.MetaStorage.DeleteObject(ctx, obj)
+ if err != nil {
+ log.Errorf("failed to delete obejct[key=%s,versionid=%s] metadata, err:%v", obj.ObjectKey, obj.VersionId, err)
+ } else {
+ log.Infof("delete obejct[key=%s,versionid=%s] metadata successfully.", obj.ObjectKey, obj.VersionId)
+ }
+
+ return err
+}
+
+func (s *s3Service) ListObjects(ctx context.Context, in *pb.ListObjectsRequest, out *pb.ListObjectsResponse) error {
+ log.Infof("ListObject is called in s3 service, bucket is %s.\n", in.Bucket)
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+ // Check ACL
+ bucket, err := s.MetaStorage.GetBucket(ctx, in.Bucket, true)
+ if err != nil {
+ log.Errorf("err:%v\n", err)
+ return nil
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return nil
+ }
+
+ // administrator can get any resource
+ if isAdmin == false {
+ switch bucket.Acl.CannedAcl {
+ case "public-read", "public-read-write":
+ break
+ default:
+ if bucket.TenantId != tenantId {
+ log.Errorf("tenantId(%s) does not much bucket.TenantId(%s)", tenantId, bucket.TenantId)
+ err = ErrBucketAccessForbidden
+ return nil
+ }
+ }
+ // TODO validate user policy and ACL
+ }
+
+ retObjects, appendInfo, err := s.ListObjectsInternal(ctx, in)
+ if appendInfo.Truncated && len(appendInfo.NextMarker) != 0 {
+ out.NextMarker = appendInfo.NextMarker
+ }
+ if in.Version == constants.ListObjectsType2Int {
+ out.NextMarker = util.Encrypt(out.NextMarker)
+ }
+
+ objects := make([]*pb.Object, 0, len(retObjects))
+ for _, obj := range retObjects {
+ object := pb.Object{
+ LastModified: obj.LastModified,
+ Etag: obj.Etag,
+ Size: obj.Size,
+ Tier: obj.Tier,
+ Location: obj.Location,
+ TenantId: obj.TenantId,
+ BucketName: obj.BucketName,
+ VersionId: obj.VersionId,
+ CustomAttributes: obj.CustomAttributes,
+ ContentType: obj.ContentType,
+ StorageMeta: obj.StorageMeta,
+ }
+ if in.EncodingType != "" { // only support "url" encoding for now
+ object.ObjectKey = url.QueryEscape(obj.ObjectKey)
+ } else {
+ object.ObjectKey = obj.ObjectKey
+ }
+ object.StorageClass, _ = GetNameFromTier(obj.Tier, utils.OSTYPE_OPENSDS)
+ objects = append(objects, &object)
+ log.Debugf("object:%+v\n", object)
+ }
+ out.Objects = objects
+ out.Prefixes = appendInfo.Prefixes
+ out.IsTruncated = appendInfo.Truncated
+
+ if in.EncodingType != "" { // only support "url" encoding for now
+ out.Prefixes = helper.Map(out.Prefixes, func(s string) string {
+ return url.QueryEscape(s)
+ })
+ out.NextMarker = url.QueryEscape(out.NextMarker)
+ }
+
+ err = ErrNoErr
+ return nil
+}
+
+func (s *s3Service) ListObjectsInternal(ctx context.Context, request *pb.ListObjectsRequest) (retObjects []*meta.Object,
+ appendInfo utils.ListObjsAppendInfo, err error) {
+ log.Infoln("Prefix:", request.Prefix, "Marker:", request.Marker, "MaxKeys:",
+ request.MaxKeys, "Delimiter:", request.Delimiter, "Version:", request.Version,
+ "keyMarker:", request.KeyMarker, "versionIdMarker:", request.VersionIdMarker)
+
+ filt := make(map[string]string)
+ if request.Versioned {
+ filt[common.KMarker] = request.KeyMarker
+ filt[common.KVerMarker] = request.VersionIdMarker
+ } else if request.Version == constants.ListObjectsType2Int {
+ if request.ContinuationToken != "" {
+ var marker string
+ marker, err = util.Decrypt(request.ContinuationToken)
+ if err != nil {
+ err = ErrInvalidContinuationToken
+ return
+ }
+ filt[common.KMarker] = marker
+ } else {
+ filt[common.KMarker] = request.StartAfter
+ }
+ } else { // version 1
+ filt[common.KMarker] = request.Marker
+ }
+
+ filt[common.KPrefix] = request.Prefix
+ filt[common.KDelimiter] = request.Delimiter
+ // currentlly, request.Filter only support filter by 'lastmodified' and 'tier'
+ for k, v := range request.Filter {
+ filt[k] = v
+ }
+
+ return s.MetaStorage.Db.ListObjects(ctx, request.Bucket, request.Versioned, int(request.MaxKeys), filt)
+}
+
+func (s *s3Service) cleanObject(ctx context.Context, object *Object, sd driver.StorageDriver) error {
+ delInput := &pb.DeleteObjectInput{
+ Bucket: object.BucketName, Key: object.ObjectKey, ObjectId: object.ObjectId,
+ VersioId: object.VersionId, StorageMeta: object.StorageMeta,
+ }
+
+ err := sd.Delete(ctx, delInput)
+ if err != nil {
+ log.Warnf("clean object[%v] from backend failed, err:%v\n", err)
+ ierr := s.MetaStorage.AddGcobjRecord(ctx, object)
+ if ierr != nil {
+ log.Warnf("add gc record failed, object:%v, err:%v\n", object, ierr)
+ }
+ }
+
+ return err
+}
+
+func (s *s3Service) PutObjACL(ctx context.Context, in *pb.PutObjACLRequest, out *pb.BaseResponse) error {
+ log.Info("PutObjACL is called in s3 service.")
+ var err error
+ defer func() {
+ out.ErrorCode = GetErrCode(err)
+ }()
+
+ bucket, err := s.MetaStorage.GetBucket(ctx, in.ACLConfig.BucketName, true)
+ if err != nil {
+ log.Errorf("failed to get bucket meta. err: %v\n", err)
+ return err
+ }
+
+ isAdmin, tenantId, _, err := util.GetCredentialFromCtx(ctx)
+ if err != nil && isAdmin == false {
+ log.Error("get tenant id failed")
+ err = ErrInternalError
+ return nil
+ }
+
+ // administrator can get any resource
+ if isAdmin == false {
+ switch bucket.Acl.CannedAcl {
+ case "bucket-owner-full-control":
+ if bucket.TenantId != tenantId {
+ err = ErrAccessDenied
+ return err
+ }
+ default:
+ if bucket.TenantId != tenantId {
+ err = ErrAccessDenied
+ return err
+ }
+ }
+ // TODO validate user policy and ACL
+ }
+
+ object, err := s.MetaStorage.GetObject(ctx, in.ACLConfig.BucketName, in.ACLConfig.ObjectKey, "", true)
+ if err != nil {
+ log.Errorln("failed to get object info from meta storage. err:", err)
+ return err
+ }
+ object.Acl = &pb.Acl{CannedAcl: in.ACLConfig.CannedAcl}
+ err = s.MetaStorage.UpdateObjectMeta(object)
+ if err != nil {
+ log.Infoln("failed to update object meta. err:", err)
+ return err
+ }
+
+ log.Infoln("Put object acl successfully.")
+ return nil
+}
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+ "context"
+ "fmt"
+ "github.com/opensds/multi-cloud/s3/pkg/utils"
+ "os"
+ "strconv"
+
+ "github.com/Azure/azure-storage-blob-go/azblob"
+ "github.com/micro/go-micro/client"
+ "github.com/opensds/multi-cloud/api/pkg/utils/obs"
+ backend "github.com/opensds/multi-cloud/backend/proto"
+ . "github.com/opensds/multi-cloud/s3/error"
+ "github.com/opensds/multi-cloud/s3/pkg/db"
+ "github.com/opensds/multi-cloud/s3/pkg/gc"
+ "github.com/opensds/multi-cloud/s3/pkg/helper"
+ "github.com/opensds/multi-cloud/s3/pkg/meta"
+ "github.com/opensds/multi-cloud/s3/pkg/meta/util"
+ . "github.com/opensds/multi-cloud/s3/pkg/utils"
+ pb "github.com/opensds/multi-cloud/s3/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+type Int2String map[int32]string
+type String2Int map[string]int32
+
+// map from cloud vendor name to a map, which is used to map from internal tier to it's storage class name.
+var Int2ExtTierMap map[string]*Int2String
+
+// map from cloud vendor name to a map, which is used to map from storage class name to internal tier.
+var Ext2IntTierMap map[string]*String2Int
+
+// map from a specific tier to an array of tiers, that means transition can happens from the specific tier to those tiers in the array.
+var TransitionMap map[int32][]int32
+var SupportedClasses []pb.StorageClass
+
+type s3Service struct {
+ MetaStorage *meta.Meta
+ backendClient backend.BackendService
+}
+
+func NewS3Service() pb.S3Handler {
+ host := os.Getenv("DB_HOST")
+ dbstor := Database{Credential: "unkonwn", Driver: "tidb", Endpoint: host}
+ db.Init(&dbstor)
+
+ initStorageClass()
+ cfg := meta.MetaConfig{
+ CacheType: meta.CacheType(helper.CONFIG.MetaCacheType),
+ TidbInfo: helper.CONFIG.TidbInfo,
+ }
+
+ ctx, cancelFunc := context.WithCancel(context.Background())
+ metaStor := meta.New(cfg)
+ gc.Init(ctx, cancelFunc, metaStor)
+ return &s3Service{
+ MetaStorage: metaStor,
+ backendClient: backend.NewBackendService("backend", client.DefaultClient),
+ }
+}
+
+func GetNameFromTier(tier int32, backendType string) (string, error) {
+ v, ok := Int2ExtTierMap[backendType]
+ if !ok {
+ log.Errorf("get storage class of failed, no such backend type:%s.\n", backendType)
+ return "", ErrInternalError
+ }
+
+ v2, ok := (*v)[tier]
+ if !ok {
+ log.Errorf("get storage class of tier[%d] failed, backendType=%s.\n", tier, backendType)
+ return "", ErrInternalError
+ }
+
+ log.Infof("storage class of tier[%d] for backend type[%s] is %s.\n", tier, backendType, v2)
+ return v2, nil
+}
+
+func loadAWSDefault(i2e *map[string]*Int2String, e2i *map[string]*String2Int) {
+ t2n := make(Int2String)
+ t2n[Tier1] = AWS_STANDARD
+ t2n[Tier99] = AWS_STANDARD_IA
+ t2n[Tier999] = AWS_GLACIER
+ (*i2e)[OSTYPE_AWS] = &t2n
+
+ n2t := make(String2Int)
+ n2t[AWS_STANDARD] = Tier1
+ n2t[AWS_STANDARD_IA] = Tier99
+ n2t[AWS_GLACIER] = Tier999
+ (*e2i)[OSTYPE_AWS] = &n2t
+}
+
+func loadOpenSDSDefault(i2e *map[string]*Int2String, e2i *map[string]*String2Int) {
+ t2n := make(Int2String)
+ t2n[Tier1] = AWS_STANDARD
+ t2n[Tier99] = AWS_STANDARD_IA
+ t2n[Tier999] = AWS_GLACIER
+ (*i2e)[OSTYPE_OPENSDS] = &t2n
+
+ n2t := make(String2Int)
+ n2t[AWS_STANDARD] = Tier1
+ n2t[AWS_STANDARD_IA] = Tier99
+ n2t[AWS_GLACIER] = Tier999
+ (*e2i)[OSTYPE_OPENSDS] = &n2t
+}
+
+func loadAzureDefault(i2e *map[string]*Int2String, e2i *map[string]*String2Int) {
+ t2n := make(Int2String)
+ t2n[Tier1] = string(azblob.AccessTierHot)
+ t2n[Tier99] = string(azblob.AccessTierCool)
+ t2n[Tier999] = string(azblob.AccessTierArchive)
+ (*i2e)[OSTYPE_Azure] = &t2n
+
+ n2t := make(String2Int)
+ n2t[string(azblob.AccessTierHot)] = Tier1
+ n2t[string(azblob.AccessTierCool)] = Tier99
+ n2t[string(string(azblob.AccessTierArchive))] = Tier999
+ (*e2i)[OSTYPE_Azure] = &n2t
+}
+
+func loadHWDefault(i2e *map[string]*Int2String, e2i *map[string]*String2Int) {
+ t2n := make(Int2String)
+ t2n[Tier1] = string(obs.StorageClassStandard)
+ t2n[Tier99] = string(obs.StorageClassWarm)
+ t2n[Tier999] = string(obs.StorageClassCold)
+ (*i2e)[OSTYPE_OBS] = &t2n
+
+ n2t := make(String2Int)
+ n2t[string(obs.StorageClassStandard)] = Tier1
+ n2t[string(obs.StorageClassWarm)] = Tier99
+ n2t[string(obs.StorageClassCold)] = Tier999
+ (*e2i)[OSTYPE_OBS] = &n2t
+}
+
+func loadGCPDefault(i2e *map[string]*Int2String, e2i *map[string]*String2Int) {
+ t2n := make(Int2String)
+ t2n[Tier1] = GCS_MULTI_REGIONAL
+ //t2n[Tier99] = GCS_NEARLINE
+ //t2n[Tier999] = GCS_COLDLINE
+ (*i2e)[OSTYPE_GCS] = &t2n
+
+ n2t := make(String2Int)
+ n2t[GCS_MULTI_REGIONAL] = Tier1
+ //n2t[GCS_NEARLINE] = Tier99
+ //n2t[GCS_COLDLINE] = Tier999
+ (*e2i)[OSTYPE_GCS] = &n2t
+}
+
+func loadCephDefault(i2e *map[string]*Int2String, e2i *map[string]*String2Int) {
+ t2n := make(Int2String)
+ t2n[Tier1] = CEPH_STANDARD
+ (*i2e)[OSTYPE_CEPH] = &t2n
+
+ n2t := make(String2Int)
+ n2t[CEPH_STANDARD] = Tier1
+ (*e2i)[OSTYPE_CEPH] = &n2t
+}
+
+func loadFusionStroageDefault(i2e *map[string]*Int2String, e2i *map[string]*String2Int) {
+ t2n := make(Int2String)
+ t2n[Tier1] = string(obs.StorageClassStandard)
+ (*i2e)[OSTYPE_FUSIONSTORAGE] = &t2n
+
+ n2t := make(String2Int)
+ n2t[string(obs.StorageClassStandard)] = Tier1
+ (*e2i)[OSTYPE_FUSIONSTORAGE] = &n2t
+}
+
+func loadDefaultStorageClass() error {
+ /* Default storage class definition:
+ T1 T99 T999
+ AWS S3: STANDARD STANDARD_IA GLACIER
+ Azure Blob: HOT COOL ARCHIVE
+ HW OBS: STANDARD WARM COLD
+ GCP: Multi-Regional NearLine ColdLine
+ Ceph S3: STANDARD - -
+ FusinoStorage Object: STANDARD - -
+ */
+ /* Lifecycle transition:
+ T1 -> T99: allowed
+ T1 -> T999: allowed
+ T99 -> T999: allowed
+ T99 -> T1: not allowed
+ T999 -> T1: not allowed
+ T999 -> T99: not allowed
+ */
+
+ SupportedClasses = append(SupportedClasses, pb.StorageClass{Name: string(AWS_STANDARD), Tier: int32(Tier1)})
+ SupportedClasses = append(SupportedClasses, pb.StorageClass{Name: string(AWS_STANDARD_IA), Tier: int32(Tier99)})
+ SupportedClasses = append(SupportedClasses, pb.StorageClass{Name: string(AWS_GLACIER), Tier: int32(Tier999)})
+
+ log.Infof("Supported storage classes:%v\n", SupportedClasses)
+
+ Int2ExtTierMap = make(map[string]*Int2String)
+ Ext2IntTierMap = make(map[string]*String2Int)
+ loadAWSDefault(&Int2ExtTierMap, &Ext2IntTierMap)
+ loadOpenSDSDefault(&Int2ExtTierMap, &Ext2IntTierMap)
+ loadAzureDefault(&Int2ExtTierMap, &Ext2IntTierMap)
+ loadHWDefault(&Int2ExtTierMap, &Ext2IntTierMap)
+ loadGCPDefault(&Int2ExtTierMap, &Ext2IntTierMap)
+ loadCephDefault(&Int2ExtTierMap, &Ext2IntTierMap)
+ loadFusionStroageDefault(&Int2ExtTierMap, &Ext2IntTierMap)
+
+ log.Infof("Int2ExtTierMap:%v\n", Int2ExtTierMap)
+ log.Infof("Ext2IntTierMap:%v\n", Ext2IntTierMap)
+
+ return nil
+}
+
+// Currently user defined storage tiers and classes are not supported.
+func loadUserDefinedStorageClass() error {
+ log.Info("user defined storage class is not supported now")
+ return fmt.Errorf("user defined storage class is not supported now")
+}
+
+func loadDefaultTransition() error {
+ // transition from a tier to the same tier is valid in case cross-cloud transition
+ TransitionMap = make(map[int32][]int32)
+ TransitionMap[Tier1] = []int32{Tier1}
+ TransitionMap[Tier99] = []int32{Tier1, Tier99}
+ TransitionMap[Tier999] = []int32{Tier1, Tier99, Tier999}
+
+ log.Infof("loadDefaultTransition:%+v\n", TransitionMap)
+ return nil
+}
+
+func validTier(tier int32) bool {
+ for _, v := range SupportedClasses {
+ if v.Tier == tier {
+ return true
+ }
+ }
+
+ return false
+}
+
+func loadUserDefinedTransition() error {
+ log.Info("user defined storage class is not supported now")
+ return fmt.Errorf("user defined storage class is not supported now")
+}
+
+func initStorageClass() {
+ // Check if use the default storage class.
+ set := os.Getenv("USE_DEFAULT_STORAGE_CLASS")
+ val, err := strconv.ParseInt(set, 10, 64)
+ log.Infof("USE_DEFAULT_STORAGE_CLASS:set=%s, val=%d, err=%v.\n", set, val, err)
+ if err != nil {
+ log.Errorf("invalid USE_DEFAULT_STORAGE_CLASS:%s\n", set)
+ panic("init s3service failed")
+ }
+
+ // Load storage class definition and transition relationship.
+ var err1, err2 error
+ if val > 0 {
+ err1 = loadDefaultStorageClass()
+ err2 = loadDefaultTransition()
+ } else {
+ err1 = loadUserDefinedStorageClass()
+ err2 = loadUserDefinedTransition()
+ }
+ // Exit if init failed.
+ if err1 != nil || err2 != nil {
+ panic("init s3service failed")
+ }
+}
+
+func (s *s3Service) GetStorageClasses(ctx context.Context, in *pb.BaseRequest, out *pb.GetStorageClassesResponse) error {
+ classes := []*pb.StorageClass{}
+ for _, v := range SupportedClasses {
+ classes = append(classes, &pb.StorageClass{Name: v.Name, Tier: v.Tier})
+ }
+
+ out.Classes = classes
+
+ return nil
+}
+
+func (s *s3Service) GetTierMap(ctx context.Context, in *pb.BaseRequest, out *pb.GetTierMapResponse) error {
+ log.Info("GetTierMap ...")
+
+ // Get map from internal tier to external class name.
+ out.Tier2Name = make(map[string]*pb.Tier2ClassName)
+ for k, v := range Int2ExtTierMap {
+ var val pb.Tier2ClassName
+ val.Lst = make(map[int32]string)
+ for k1, v1 := range *v {
+ val.Lst[k1] = v1
+ }
+ out.Tier2Name[k] = &val
+ }
+
+ // Get transition map.
+ for k, v := range TransitionMap {
+ for _, t := range v {
+ trans := fmt.Sprintf("%d:%d", t, k)
+ out.Transition = append(out.Transition, trans)
+ }
+ }
+
+ log.Infof("out.Transition:%v\n", out.Transition)
+ return nil
+}
+
+func (s *s3Service) UpdateBucket(ctx context.Context, in *pb.Bucket, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ //update versioning if not nil
+ if in.Versioning != nil {
+ err := s.MetaStorage.Db.UpdateBucketVersioning(ctx, in.Name, in.Versioning.Status)
+ if err != nil {
+ log.Errorf("get bucket[%s] failed, err:%v\n", in.Name, err)
+ return err
+ }
+ }
+ if in.ServerSideEncryption != nil{
+ byteArr, keyErr := utils.GetRandom32BitKey()
+ if keyErr != nil {
+ log.Error("Error generating SSE key", keyErr)
+ return keyErr
+ }
+ sseErr := s.MetaStorage.Db.UpdateBucketSSE(ctx, in.Name, in.ServerSideEncryption.SseType, byteArr)
+ if sseErr != nil {
+ log.Error("Error creating SSE entry: ", sseErr)
+ return sseErr
+ }
+ }
+ return nil
+}
+
+func (s *s3Service) AppendObject(ctx context.Context, in *pb.BaseRequest, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) PostObject(ctx context.Context, in *pb.BaseRequest, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) HeadObject(ctx context.Context, in *pb.BaseObjRequest, out *pb.Object) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) GetObjACL(ctx context.Context, in *pb.BaseObjRequest, out *pb.ObjACL) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) GetBucketLocation(ctx context.Context, in *pb.BaseRequest, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) GetBucketVersioning(ctx context.Context, in *pb.BaseBucketRequest, out *pb.BucketVersioning) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+//TODO Will check whether we need another interface for put bucket version
+/*func (s *s3Service) PutBucketVersioning(ctx context.Context, in *pb.PutBucketVersioningRequest, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+*/
+
+func (s *s3Service) GetBucketACL(ctx context.Context, in *pb.BaseBucketRequest, out *pb.BucketACL) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) PutBucketCORS(ctx context.Context, in *pb.BaseRequest, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) GetBucketCORS(ctx context.Context, in *pb.BaseRequest, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) DeleteBucketCORS(ctx context.Context, in *pb.BaseRequest, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) PutBucketPolicy(ctx context.Context, in *pb.BaseRequest, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) GetBucketPolicy(ctx context.Context, in *pb.BaseRequest, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) DeleteBucketPolicy(ctx context.Context, in *pb.BaseRequest, out *pb.BaseResponse) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) HeadBucket(ctx context.Context, in *pb.BaseRequest, out *pb.Bucket) error {
+ log.Info("UpdateBucket is called in s3 service.")
+
+ return nil
+}
+
+func (s *s3Service) UpdateObjMeta(ctx context.Context, in *pb.UpdateObjMetaRequest, out *pb.BaseResponse) error {
+ log.Infof("Update meatadata, objkey:%s, lastmodified:%d, setting:%v\n", in.ObjKey, in.LastModified, in.Setting)
+ /*valid := make(map[string]struct{})
+ valid["tier"] = struct{}{}
+ valid["backend"] = struct{}{}
+ set, err := CheckReqObjMeta(in.Setting, valid)
+ if err.Code != ERR_OK {
+ out.ErrorCode = fmt.Sprintf("%s", err.Code)
+ out.Msg = err.Description
+ return err.Error()
+ }
+
+ err = db.DbAdapter.UpdateObjMeta(ctx, &in.ObjKey, &in.BucketName, in.LastModified, set)
+ if err.Code != ERR_OK {
+ out.ErrorCode = fmt.Sprintf("%s", err.Code)
+ out.Msg = err.Description
+ return err.Error()
+ }
+
+ out.Msg = "update object meta data successfully."
+ */
+ return nil
+}
+
+func (s *s3Service) GetBackendTypeByTier(ctx context.Context, in *pb.GetBackendTypeByTierRequest, out *pb.GetBackendTypeByTierResponse) error {
+ for k, v := range Int2ExtTierMap {
+ for k1, _ := range *v {
+ if k1 == in.Tier {
+ out.Types = append(out.Types, k)
+ }
+ }
+ }
+
+ log.Infof("GetBackendTypesByTier, types:%v\n", out.Types)
+
+ return nil
+}
+
+func (s *s3Service) AddUploadRecord(ctx context.Context, record *pb.MultipartUploadRecord, out *pb.BaseResponse) error {
+ log.Infof("add multipart upload record")
+
+ return nil
+}
+
+func (s *s3Service) DeleteUploadRecord(ctx context.Context, record *pb.MultipartUploadRecord, out *pb.BaseResponse) error {
+ log.Infof("delete multipart upload record")
+
+ return nil
+}
+
+func (s *s3Service) CountObjects(ctx context.Context, in *pb.ListObjectsRequest, out *pb.CountObjectsResponse) error {
+ log.Info("Count objects is called in s3 service.")
+
+ rsp, err := s.MetaStorage.Db.CountObjects(ctx, in.Bucket, in.Prefix)
+ if err != nil {
+ return err
+ }
+ out.Count = rsp.Count
+ out.Size = rsp.Size
+
+ return nil
+}
+
+func GetErrCode(err error) (errCode int32) {
+ if err == nil {
+ errCode = int32(ErrNoErr)
+ return
+ }
+
+ errCode = int32(ErrInternalError)
+ s3err, ok := err.(S3ErrorCode)
+ if ok {
+ errCode = int32(s3err)
+ }
+
+ return errCode
+}
+
+func CheckRights(ctx context.Context, tenantId4Source string) (bool, string, string, error) {
+ isAdmin, tenantId, userId, err := util.GetCredentialFromCtx(ctx)
+ if err != nil {
+ log.Errorf("get credential faied, err:%v\n", err)
+ return isAdmin, tenantId, userId, ErrInternalError
+ }
+ if !isAdmin && tenantId != tenantId4Source {
+ log.Errorf("access forbidden, tenantId=%s, tenantId4Source=%s\n", tenantId, tenantId4Source)
+ return isAdmin, tenantId, userId, ErrAccessDenied
+ }
+
+ return isAdmin, tenantId, userId, nil
+}
+
+
+
package utils
+
+import (
+ "crypto/aes"
+ "crypto/cipher"
+ "crypto/rand"
+ "io"
+)
+import log "github.com/sirupsen/logrus"
+
+func EncryptWithAES256RandomKey(data []byte, key []byte) (error, []byte) {
+ // use the key, get the cipher
+ cipherBlock, cipherErr := aes.NewCipher(key)
+ if cipherErr != nil {
+ log.Errorf("Encryption error, cipher not generated")
+ return cipherErr, nil
+ }
+ // use the cipher block to encrypt
+ // gcm or Galois/Counter Mode, is a mode of operation
+ // for symmetric key cryptographic block ciphers
+ // https://en.wikipedia.org/wiki/Galois/Counter_Mode
+ aesgcm, gcmErr := cipher.NewGCM(cipherBlock)
+ if gcmErr != nil {
+ log.Errorf("Encryption error, GCM not generated")
+ return gcmErr, nil
+ }
+ // NonceSize is default 12 bytes
+ nonce := make([]byte, aesgcm.NonceSize())
+ _, nonceErr := io.ReadFull(rand.Reader, nonce)
+ if nonceErr != nil {
+ log.Errorf("Encryption error, GCM nonce not created")
+ return nonceErr, nil
+ }
+
+ // use the aes gcm to seal the data
+ encBytes := aesgcm.Seal(nonce, nonce, data, nil)
+ return nil, encBytes
+}
+
+func DecryptWithAES256(data []byte, key []byte) (error, []byte) {
+ // use the key, get the cipher
+ cipherBlock, cipherErr := aes.NewCipher(key)
+ if cipherErr != nil {
+ log.Errorf("Decryption error, cipher not generated")
+ return cipherErr, nil
+ }
+ // use the cipher block to decrypt
+ // gcm or Galois/Counter Mode, is a mode of operation
+ // for symmetric key cryptographic block ciphers
+ // https://en.wikipedia.org/wiki/Galois/Counter_Mode
+ aesgcm, gcmErr := cipher.NewGCM(cipherBlock)
+ if gcmErr != nil {
+ log.Errorf("Decryption error, GCM not generated")
+ return gcmErr, nil
+ }
+ nonceSize := aesgcm.NonceSize()
+ nonce, ciphertext := data[:nonceSize], data[nonceSize:]
+ // use the aes gcm to open the data
+ decBytes, decErr := aesgcm.Open(nil, nonce, ciphertext, nil)
+ if decErr != nil {
+ log.Errorf("Decryption error during open %s", decErr)
+ return decErr, nil
+ }
+ return decErr, decBytes
+}
+
+func GetRandom32BitKey() ([]byte, error) {
+ key := make([]byte, 32)
+
+ _, err := rand.Read(key)
+ if err != nil {
+ log.Errorf("Error generating random 32 bit key %s", err)
+ return nil, err
+ }
+ log.Infof("Generated random 32 bit key")
+ return key, nil
+}
+
+
+
+
+
// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+ "context"
+ "crypto/md5"
+ "encoding/hex"
+ "github.com/opensds/multi-cloud/backend/proto"
+ log "github.com/sirupsen/logrus"
+)
+
+type Database struct {
+ Credential string `conf:"credential,username:password@tcp(ip:port)/dbname"`
+ Driver string `conf:"driver,mongodb"`
+ Endpoint string `conf:"endpoint,localhost:27017"`
+}
+
+// Tier1, Tier99 and Tier999 just like the tiers of hot, warm, cold.
+// In the future, we will provide the ability for users to add new storage tiers, if we use 1, 2 and 3, then no space for new storage tiers.
+const (
+ Tier1 = 1
+ Tier99 = 99
+ Tier999 = 999
+)
+
+const (
+ AWS_STANDARD = "STANDARD"
+ AWS_STANDARD_IA = "STANDARD_IA"
+ AWS_GLACIER = "GLACIER"
+)
+
+const (
+ CEPH_STANDARD = "STDANDARD"
+)
+
+const (
+ GCS_MULTI_REGIONAL = "MULTI_REGIONAL"
+ GCS_REGIONAL = "REGIONAL"
+ GCS_NEARLINE = "NEARLINE"
+ GCS_COLDLINE = "COLDLINE"
+)
+
+//Object Storage Type
+const (
+ OSTYPE_OPENSDS = "OpenSDS"
+ OSTYPE_AWS = "aws-s3"
+ OSTYPE_Azure = "azure-blob"
+ OSTYPE_OBS = "hw-obs"
+ OSTYPE_GCS = "gcp-s3"
+ OSTYPE_CEPH = "ceph-s3"
+ OSTYPE_FUSIONSTORAGE = "fusionstorage-object"
+)
+
+const (
+ DBKEY_DELETEMARKER = "isdeletemarker"
+ DBKEY_INITFLAG = "initflag"
+ DBKEY_OBJECTKEY = "objectkey"
+ DBKEY_UPLOADID = "uploadid"
+ DBKEY_LASTMODIFIED = "lastmodified"
+ DBKEY_SUPPOSEDSTATUS = "supposedstatus"
+ DBKEY_LOCKOBJ_OBJKEY = "objkey"
+ DBKEY_BUCKET = "bucket"
+ DBKEY_INITTIME = "inittime"
+ DBKEY_NAME = "name"
+ DBKEY_LIFECYCLE = "lifecycleconfiguration"
+ DBKEY_ID = "id"
+)
+
+type ObjsCountInfo struct {
+ Size int64
+ Count int64
+}
+
+const (
+ MaxObjectList = 1000 // Limit number of objects in a listObjectsResponse.
+ MaxUploadsList = 1000 // Limit number of uploads in a listUploadsResponse.
+ MaxPartsList = 1000 // Limit number of parts in a listPartsResponse.
+)
+
+const (
+ VersioningEnabled = "Enabled"
+ VersioningDisabled = "Disabled"
+ VersioningSuspended = "Suspended"
+)
+
+type ListObjsAppendInfo struct {
+ Prefixes []string
+ Truncated bool
+ NextMarker string
+}
+
+const (
+ MoveType_Invalid = iota
+ MoveType_MoveCrossBuckets
+ MoveType_ChangeLocation
+ MoveType_ChangeStorageTier
+)
+
+const (
+ RequestType_Lifecycle = "lifecycle"
+)
+
+func Md5Content(data []byte) string {
+ md5Ctx := md5.New()
+ md5Ctx.Write(data)
+ cipherStr := md5Ctx.Sum(nil)
+ //value := base64.StdEncoding.EncodeToString(cipherStr)
+ value := hex.EncodeToString(cipherStr)
+ return value
+}
+
+func GetBackend(ctx context.Context, backedClient backend.BackendService, backendName string) (*backend.BackendDetail,
+ error) {
+ log.Infof("backendName is %v:\n", backendName)
+ backendRep, backendErr := backedClient.ListBackend(ctx, &backend.ListBackendRequest{
+ Offset: 0,
+ Limit: 1,
+ Filter: map[string]string{"name": backendName}})
+ log.Infof("backendErr is %v:", backendErr)
+ if backendErr != nil {
+ log.Errorf("get backend %s failed.", backendName)
+ return nil, backendErr
+ }
+ log.Infof("backendRep is %v:", backendRep)
+ backend := backendRep.Backends[0]
+ return backend, nil
+}
+
+
+
// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: s3.proto
+
+package s3
+
+import (
+ fmt "fmt"
+ proto "github.com/golang/protobuf/proto"
+ math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+type ListObjectPartsRequest struct {
+ BucketName string `protobuf:"bytes,1,opt,name=bucketName,proto3" json:"bucketName,omitempty"`
+ ObjectKey string `protobuf:"bytes,2,opt,name=objectKey,proto3" json:"objectKey,omitempty"`
+ EncodingType string `protobuf:"bytes,3,opt,name=encodingType,proto3" json:"encodingType,omitempty"`
+ UploadId string `protobuf:"bytes,4,opt,name=uploadId,proto3" json:"uploadId,omitempty"`
+ MaxParts int64 `protobuf:"varint,5,opt,name=maxParts,proto3" json:"maxParts,omitempty"`
+ PartNumberMarker int64 `protobuf:"varint,6,opt,name=partNumberMarker,proto3" json:"partNumberMarker,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ListObjectPartsRequest) Reset() { *m = ListObjectPartsRequest{} }
+func (m *ListObjectPartsRequest) String() string { return proto.CompactTextString(m) }
+func (*ListObjectPartsRequest) ProtoMessage() {}
+func (*ListObjectPartsRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{0}
+}
+
+func (m *ListObjectPartsRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ListObjectPartsRequest.Unmarshal(m, b)
+}
+func (m *ListObjectPartsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ListObjectPartsRequest.Marshal(b, m, deterministic)
+}
+func (m *ListObjectPartsRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ListObjectPartsRequest.Merge(m, src)
+}
+func (m *ListObjectPartsRequest) XXX_Size() int {
+ return xxx_messageInfo_ListObjectPartsRequest.Size(m)
+}
+func (m *ListObjectPartsRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_ListObjectPartsRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListObjectPartsRequest proto.InternalMessageInfo
+
+func (m *ListObjectPartsRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *ListObjectPartsRequest) GetObjectKey() string {
+ if m != nil {
+ return m.ObjectKey
+ }
+ return ""
+}
+
+func (m *ListObjectPartsRequest) GetEncodingType() string {
+ if m != nil {
+ return m.EncodingType
+ }
+ return ""
+}
+
+func (m *ListObjectPartsRequest) GetUploadId() string {
+ if m != nil {
+ return m.UploadId
+ }
+ return ""
+}
+
+func (m *ListObjectPartsRequest) GetMaxParts() int64 {
+ if m != nil {
+ return m.MaxParts
+ }
+ return 0
+}
+
+func (m *ListObjectPartsRequest) GetPartNumberMarker() int64 {
+ if m != nil {
+ return m.PartNumberMarker
+ }
+ return 0
+}
+
+type Part struct {
+ PartNumber int64 `protobuf:"varint,1,opt,name=PartNumber,proto3" json:"PartNumber,omitempty"`
+ ETag string `protobuf:"bytes,2,opt,name=ETag,proto3" json:"ETag,omitempty"`
+ LastModified string `protobuf:"bytes,3,opt,name=LastModified,proto3" json:"LastModified,omitempty"`
+ Size int64 `protobuf:"varint,4,opt,name=Size,proto3" json:"Size,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Part) Reset() { *m = Part{} }
+func (m *Part) String() string { return proto.CompactTextString(m) }
+func (*Part) ProtoMessage() {}
+func (*Part) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{1}
+}
+
+func (m *Part) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Part.Unmarshal(m, b)
+}
+func (m *Part) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Part.Marshal(b, m, deterministic)
+}
+func (m *Part) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Part.Merge(m, src)
+}
+func (m *Part) XXX_Size() int {
+ return xxx_messageInfo_Part.Size(m)
+}
+func (m *Part) XXX_DiscardUnknown() {
+ xxx_messageInfo_Part.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Part proto.InternalMessageInfo
+
+func (m *Part) GetPartNumber() int64 {
+ if m != nil {
+ return m.PartNumber
+ }
+ return 0
+}
+
+func (m *Part) GetETag() string {
+ if m != nil {
+ return m.ETag
+ }
+ return ""
+}
+
+func (m *Part) GetLastModified() string {
+ if m != nil {
+ return m.LastModified
+ }
+ return ""
+}
+
+func (m *Part) GetSize() int64 {
+ if m != nil {
+ return m.Size
+ }
+ return 0
+}
+
+type ListObjectPartsResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=ErrorCode,proto3" json:"ErrorCode,omitempty"`
+ Initiator *Owner `protobuf:"bytes,2,opt,name=Initiator,proto3" json:"Initiator,omitempty"`
+ Owner *Owner `protobuf:"bytes,3,opt,name=Owner,proto3" json:"Owner,omitempty"`
+ StorageClass string `protobuf:"bytes,4,opt,name=StorageClass,proto3" json:"StorageClass,omitempty"`
+ PartNumberMarker int64 `protobuf:"varint,5,opt,name=PartNumberMarker,proto3" json:"PartNumberMarker,omitempty"`
+ NextPartNumberMarker int64 `protobuf:"varint,6,opt,name=NextPartNumberMarker,proto3" json:"NextPartNumberMarker,omitempty"`
+ MaxParts int64 `protobuf:"varint,7,opt,name=MaxParts,proto3" json:"MaxParts,omitempty"`
+ IsTruncated bool `protobuf:"varint,8,opt,name=IsTruncated,proto3" json:"IsTruncated,omitempty"`
+ Parts []*Part `protobuf:"bytes,9,rep,name=Parts,proto3" json:"Parts,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ListObjectPartsResponse) Reset() { *m = ListObjectPartsResponse{} }
+func (m *ListObjectPartsResponse) String() string { return proto.CompactTextString(m) }
+func (*ListObjectPartsResponse) ProtoMessage() {}
+func (*ListObjectPartsResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{2}
+}
+
+func (m *ListObjectPartsResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ListObjectPartsResponse.Unmarshal(m, b)
+}
+func (m *ListObjectPartsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ListObjectPartsResponse.Marshal(b, m, deterministic)
+}
+func (m *ListObjectPartsResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ListObjectPartsResponse.Merge(m, src)
+}
+func (m *ListObjectPartsResponse) XXX_Size() int {
+ return xxx_messageInfo_ListObjectPartsResponse.Size(m)
+}
+func (m *ListObjectPartsResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_ListObjectPartsResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListObjectPartsResponse proto.InternalMessageInfo
+
+func (m *ListObjectPartsResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *ListObjectPartsResponse) GetInitiator() *Owner {
+ if m != nil {
+ return m.Initiator
+ }
+ return nil
+}
+
+func (m *ListObjectPartsResponse) GetOwner() *Owner {
+ if m != nil {
+ return m.Owner
+ }
+ return nil
+}
+
+func (m *ListObjectPartsResponse) GetStorageClass() string {
+ if m != nil {
+ return m.StorageClass
+ }
+ return ""
+}
+
+func (m *ListObjectPartsResponse) GetPartNumberMarker() int64 {
+ if m != nil {
+ return m.PartNumberMarker
+ }
+ return 0
+}
+
+func (m *ListObjectPartsResponse) GetNextPartNumberMarker() int64 {
+ if m != nil {
+ return m.NextPartNumberMarker
+ }
+ return 0
+}
+
+func (m *ListObjectPartsResponse) GetMaxParts() int64 {
+ if m != nil {
+ return m.MaxParts
+ }
+ return 0
+}
+
+func (m *ListObjectPartsResponse) GetIsTruncated() bool {
+ if m != nil {
+ return m.IsTruncated
+ }
+ return false
+}
+
+func (m *ListObjectPartsResponse) GetParts() []*Part {
+ if m != nil {
+ return m.Parts
+ }
+ return nil
+}
+
+type AbortMultipartRequest struct {
+ BucketName string `protobuf:"bytes,1,opt,name=bucketName,proto3" json:"bucketName,omitempty"`
+ ObjectKey string `protobuf:"bytes,2,opt,name=objectKey,proto3" json:"objectKey,omitempty"`
+ UploadId string `protobuf:"bytes,3,opt,name=uploadId,proto3" json:"uploadId,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *AbortMultipartRequest) Reset() { *m = AbortMultipartRequest{} }
+func (m *AbortMultipartRequest) String() string { return proto.CompactTextString(m) }
+func (*AbortMultipartRequest) ProtoMessage() {}
+func (*AbortMultipartRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{3}
+}
+
+func (m *AbortMultipartRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_AbortMultipartRequest.Unmarshal(m, b)
+}
+func (m *AbortMultipartRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_AbortMultipartRequest.Marshal(b, m, deterministic)
+}
+func (m *AbortMultipartRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_AbortMultipartRequest.Merge(m, src)
+}
+func (m *AbortMultipartRequest) XXX_Size() int {
+ return xxx_messageInfo_AbortMultipartRequest.Size(m)
+}
+func (m *AbortMultipartRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_AbortMultipartRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_AbortMultipartRequest proto.InternalMessageInfo
+
+func (m *AbortMultipartRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *AbortMultipartRequest) GetObjectKey() string {
+ if m != nil {
+ return m.ObjectKey
+ }
+ return ""
+}
+
+func (m *AbortMultipartRequest) GetUploadId() string {
+ if m != nil {
+ return m.UploadId
+ }
+ return ""
+}
+
+type CompletePart struct {
+ PartNumber int64 `protobuf:"varint,1,opt,name=partNumber,proto3" json:"partNumber,omitempty"`
+ ETag string `protobuf:"bytes,2,opt,name=eTag,proto3" json:"eTag,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *CompletePart) Reset() { *m = CompletePart{} }
+func (m *CompletePart) String() string { return proto.CompactTextString(m) }
+func (*CompletePart) ProtoMessage() {}
+func (*CompletePart) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{4}
+}
+
+func (m *CompletePart) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_CompletePart.Unmarshal(m, b)
+}
+func (m *CompletePart) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_CompletePart.Marshal(b, m, deterministic)
+}
+func (m *CompletePart) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_CompletePart.Merge(m, src)
+}
+func (m *CompletePart) XXX_Size() int {
+ return xxx_messageInfo_CompletePart.Size(m)
+}
+func (m *CompletePart) XXX_DiscardUnknown() {
+ xxx_messageInfo_CompletePart.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CompletePart proto.InternalMessageInfo
+
+func (m *CompletePart) GetPartNumber() int64 {
+ if m != nil {
+ return m.PartNumber
+ }
+ return 0
+}
+
+func (m *CompletePart) GetETag() string {
+ if m != nil {
+ return m.ETag
+ }
+ return ""
+}
+
+type CompleteMultipartRequest struct {
+ BucketName string `protobuf:"bytes,1,opt,name=bucketName,proto3" json:"bucketName,omitempty"`
+ ObjectKey string `protobuf:"bytes,2,opt,name=objectKey,proto3" json:"objectKey,omitempty"`
+ UploadId string `protobuf:"bytes,3,opt,name=uploadId,proto3" json:"uploadId,omitempty"`
+ CompleteParts []*CompletePart `protobuf:"bytes,4,rep,name=completeParts,proto3" json:"completeParts,omitempty"`
+ SourceVersionID string `protobuf:"bytes,5,opt,name=sourceVersionID,proto3" json:"sourceVersionID,omitempty"`
+ RequestType string `protobuf:"bytes,6,opt,name=requestType,proto3" json:"requestType,omitempty"`
+ Tier int32 `protobuf:"varint,7,opt,name=tier,proto3" json:"tier,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *CompleteMultipartRequest) Reset() { *m = CompleteMultipartRequest{} }
+func (m *CompleteMultipartRequest) String() string { return proto.CompactTextString(m) }
+func (*CompleteMultipartRequest) ProtoMessage() {}
+func (*CompleteMultipartRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{5}
+}
+
+func (m *CompleteMultipartRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_CompleteMultipartRequest.Unmarshal(m, b)
+}
+func (m *CompleteMultipartRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_CompleteMultipartRequest.Marshal(b, m, deterministic)
+}
+func (m *CompleteMultipartRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_CompleteMultipartRequest.Merge(m, src)
+}
+func (m *CompleteMultipartRequest) XXX_Size() int {
+ return xxx_messageInfo_CompleteMultipartRequest.Size(m)
+}
+func (m *CompleteMultipartRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_CompleteMultipartRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CompleteMultipartRequest proto.InternalMessageInfo
+
+func (m *CompleteMultipartRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *CompleteMultipartRequest) GetObjectKey() string {
+ if m != nil {
+ return m.ObjectKey
+ }
+ return ""
+}
+
+func (m *CompleteMultipartRequest) GetUploadId() string {
+ if m != nil {
+ return m.UploadId
+ }
+ return ""
+}
+
+func (m *CompleteMultipartRequest) GetCompleteParts() []*CompletePart {
+ if m != nil {
+ return m.CompleteParts
+ }
+ return nil
+}
+
+func (m *CompleteMultipartRequest) GetSourceVersionID() string {
+ if m != nil {
+ return m.SourceVersionID
+ }
+ return ""
+}
+
+func (m *CompleteMultipartRequest) GetRequestType() string {
+ if m != nil {
+ return m.RequestType
+ }
+ return ""
+}
+
+func (m *CompleteMultipartRequest) GetTier() int32 {
+ if m != nil {
+ return m.Tier
+ }
+ return 0
+}
+
+type CompleteMultipartResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ UploadID string `protobuf:"bytes,2,opt,name=uploadID,proto3" json:"uploadID,omitempty"`
+ ETag string `protobuf:"bytes,3,opt,name=eTag,proto3" json:"eTag,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *CompleteMultipartResponse) Reset() { *m = CompleteMultipartResponse{} }
+func (m *CompleteMultipartResponse) String() string { return proto.CompactTextString(m) }
+func (*CompleteMultipartResponse) ProtoMessage() {}
+func (*CompleteMultipartResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{6}
+}
+
+func (m *CompleteMultipartResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_CompleteMultipartResponse.Unmarshal(m, b)
+}
+func (m *CompleteMultipartResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_CompleteMultipartResponse.Marshal(b, m, deterministic)
+}
+func (m *CompleteMultipartResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_CompleteMultipartResponse.Merge(m, src)
+}
+func (m *CompleteMultipartResponse) XXX_Size() int {
+ return xxx_messageInfo_CompleteMultipartResponse.Size(m)
+}
+func (m *CompleteMultipartResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_CompleteMultipartResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CompleteMultipartResponse proto.InternalMessageInfo
+
+func (m *CompleteMultipartResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *CompleteMultipartResponse) GetUploadID() string {
+ if m != nil {
+ return m.UploadID
+ }
+ return ""
+}
+
+func (m *CompleteMultipartResponse) GetETag() string {
+ if m != nil {
+ return m.ETag
+ }
+ return ""
+}
+
+type InitMultiPartRequest struct {
+ BucketName string `protobuf:"bytes,1,opt,name=bucketName,proto3" json:"bucketName,omitempty"`
+ ObjectKey string `protobuf:"bytes,2,opt,name=objectKey,proto3" json:"objectKey,omitempty"`
+ Acl *Acl `protobuf:"bytes,3,opt,name=acl,proto3" json:"acl,omitempty"`
+ Tier int32 `protobuf:"varint,4,opt,name=tier,proto3" json:"tier,omitempty"`
+ Location string `protobuf:"bytes,5,opt,name=location,proto3" json:"location,omitempty"`
+ Attrs map[string]string `protobuf:"bytes,6,rep,name=attrs,proto3" json:"attrs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *InitMultiPartRequest) Reset() { *m = InitMultiPartRequest{} }
+func (m *InitMultiPartRequest) String() string { return proto.CompactTextString(m) }
+func (*InitMultiPartRequest) ProtoMessage() {}
+func (*InitMultiPartRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{7}
+}
+
+func (m *InitMultiPartRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_InitMultiPartRequest.Unmarshal(m, b)
+}
+func (m *InitMultiPartRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_InitMultiPartRequest.Marshal(b, m, deterministic)
+}
+func (m *InitMultiPartRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_InitMultiPartRequest.Merge(m, src)
+}
+func (m *InitMultiPartRequest) XXX_Size() int {
+ return xxx_messageInfo_InitMultiPartRequest.Size(m)
+}
+func (m *InitMultiPartRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_InitMultiPartRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_InitMultiPartRequest proto.InternalMessageInfo
+
+func (m *InitMultiPartRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *InitMultiPartRequest) GetObjectKey() string {
+ if m != nil {
+ return m.ObjectKey
+ }
+ return ""
+}
+
+func (m *InitMultiPartRequest) GetAcl() *Acl {
+ if m != nil {
+ return m.Acl
+ }
+ return nil
+}
+
+func (m *InitMultiPartRequest) GetTier() int32 {
+ if m != nil {
+ return m.Tier
+ }
+ return 0
+}
+
+func (m *InitMultiPartRequest) GetLocation() string {
+ if m != nil {
+ return m.Location
+ }
+ return ""
+}
+
+func (m *InitMultiPartRequest) GetAttrs() map[string]string {
+ if m != nil {
+ return m.Attrs
+ }
+ return nil
+}
+
+type InitMultiPartResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ UploadID string `protobuf:"bytes,2,opt,name=uploadID,proto3" json:"uploadID,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *InitMultiPartResponse) Reset() { *m = InitMultiPartResponse{} }
+func (m *InitMultiPartResponse) String() string { return proto.CompactTextString(m) }
+func (*InitMultiPartResponse) ProtoMessage() {}
+func (*InitMultiPartResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{8}
+}
+
+func (m *InitMultiPartResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_InitMultiPartResponse.Unmarshal(m, b)
+}
+func (m *InitMultiPartResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_InitMultiPartResponse.Marshal(b, m, deterministic)
+}
+func (m *InitMultiPartResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_InitMultiPartResponse.Merge(m, src)
+}
+func (m *InitMultiPartResponse) XXX_Size() int {
+ return xxx_messageInfo_InitMultiPartResponse.Size(m)
+}
+func (m *InitMultiPartResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_InitMultiPartResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_InitMultiPartResponse proto.InternalMessageInfo
+
+func (m *InitMultiPartResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *InitMultiPartResponse) GetUploadID() string {
+ if m != nil {
+ return m.UploadID
+ }
+ return ""
+}
+
+type PutDataStream struct {
+ Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *PutDataStream) Reset() { *m = PutDataStream{} }
+func (m *PutDataStream) String() string { return proto.CompactTextString(m) }
+func (*PutDataStream) ProtoMessage() {}
+func (*PutDataStream) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{9}
+}
+
+func (m *PutDataStream) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_PutDataStream.Unmarshal(m, b)
+}
+func (m *PutDataStream) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_PutDataStream.Marshal(b, m, deterministic)
+}
+func (m *PutDataStream) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_PutDataStream.Merge(m, src)
+}
+func (m *PutDataStream) XXX_Size() int {
+ return xxx_messageInfo_PutDataStream.Size(m)
+}
+func (m *PutDataStream) XXX_DiscardUnknown() {
+ xxx_messageInfo_PutDataStream.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PutDataStream proto.InternalMessageInfo
+
+func (m *PutDataStream) GetData() []byte {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
+type UploadPartRequest struct {
+ BucketName string `protobuf:"bytes,1,opt,name=bucketName,proto3" json:"bucketName,omitempty"`
+ ObjectKey string `protobuf:"bytes,2,opt,name=objectKey,proto3" json:"objectKey,omitempty"`
+ UploadId string `protobuf:"bytes,3,opt,name=uploadId,proto3" json:"uploadId,omitempty"`
+ PartId int32 `protobuf:"varint,4,opt,name=partId,proto3" json:"partId,omitempty"`
+ Size int64 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
+ Md5Hex string `protobuf:"bytes,6,opt,name=md5Hex,proto3" json:"md5Hex,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *UploadPartRequest) Reset() { *m = UploadPartRequest{} }
+func (m *UploadPartRequest) String() string { return proto.CompactTextString(m) }
+func (*UploadPartRequest) ProtoMessage() {}
+func (*UploadPartRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{10}
+}
+
+func (m *UploadPartRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_UploadPartRequest.Unmarshal(m, b)
+}
+func (m *UploadPartRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_UploadPartRequest.Marshal(b, m, deterministic)
+}
+func (m *UploadPartRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_UploadPartRequest.Merge(m, src)
+}
+func (m *UploadPartRequest) XXX_Size() int {
+ return xxx_messageInfo_UploadPartRequest.Size(m)
+}
+func (m *UploadPartRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_UploadPartRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_UploadPartRequest proto.InternalMessageInfo
+
+func (m *UploadPartRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *UploadPartRequest) GetObjectKey() string {
+ if m != nil {
+ return m.ObjectKey
+ }
+ return ""
+}
+
+func (m *UploadPartRequest) GetUploadId() string {
+ if m != nil {
+ return m.UploadId
+ }
+ return ""
+}
+
+func (m *UploadPartRequest) GetPartId() int32 {
+ if m != nil {
+ return m.PartId
+ }
+ return 0
+}
+
+func (m *UploadPartRequest) GetSize() int64 {
+ if m != nil {
+ return m.Size
+ }
+ return 0
+}
+
+func (m *UploadPartRequest) GetMd5Hex() string {
+ if m != nil {
+ return m.Md5Hex
+ }
+ return ""
+}
+
+type UploadPartResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ ETag string `protobuf:"bytes,2,opt,name=eTag,proto3" json:"eTag,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *UploadPartResponse) Reset() { *m = UploadPartResponse{} }
+func (m *UploadPartResponse) String() string { return proto.CompactTextString(m) }
+func (*UploadPartResponse) ProtoMessage() {}
+func (*UploadPartResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{11}
+}
+
+func (m *UploadPartResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_UploadPartResponse.Unmarshal(m, b)
+}
+func (m *UploadPartResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_UploadPartResponse.Marshal(b, m, deterministic)
+}
+func (m *UploadPartResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_UploadPartResponse.Merge(m, src)
+}
+func (m *UploadPartResponse) XXX_Size() int {
+ return xxx_messageInfo_UploadPartResponse.Size(m)
+}
+func (m *UploadPartResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_UploadPartResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_UploadPartResponse proto.InternalMessageInfo
+
+func (m *UploadPartResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *UploadPartResponse) GetETag() string {
+ if m != nil {
+ return m.ETag
+ }
+ return ""
+}
+
+type CopyObjectRequest struct {
+ SrcBucketName string `protobuf:"bytes,1,opt,name=srcBucketName,proto3" json:"srcBucketName,omitempty"`
+ TargetBucketName string `protobuf:"bytes,2,opt,name=targetBucketName,proto3" json:"targetBucketName,omitempty"`
+ SrcObjectName string `protobuf:"bytes,3,opt,name=srcObjectName,proto3" json:"srcObjectName,omitempty"`
+ TargetObjectName string `protobuf:"bytes,4,opt,name=targetObjectName,proto3" json:"targetObjectName,omitempty"`
+ TargetBackend string `protobuf:"bytes,5,opt,name=targetBackend,proto3" json:"targetBackend,omitempty"`
+ TargetTier int32 `protobuf:"varint,6,opt,name=targetTier,proto3" json:"targetTier,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *CopyObjectRequest) Reset() { *m = CopyObjectRequest{} }
+func (m *CopyObjectRequest) String() string { return proto.CompactTextString(m) }
+func (*CopyObjectRequest) ProtoMessage() {}
+func (*CopyObjectRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{12}
+}
+
+func (m *CopyObjectRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_CopyObjectRequest.Unmarshal(m, b)
+}
+func (m *CopyObjectRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_CopyObjectRequest.Marshal(b, m, deterministic)
+}
+func (m *CopyObjectRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_CopyObjectRequest.Merge(m, src)
+}
+func (m *CopyObjectRequest) XXX_Size() int {
+ return xxx_messageInfo_CopyObjectRequest.Size(m)
+}
+func (m *CopyObjectRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_CopyObjectRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CopyObjectRequest proto.InternalMessageInfo
+
+func (m *CopyObjectRequest) GetSrcBucketName() string {
+ if m != nil {
+ return m.SrcBucketName
+ }
+ return ""
+}
+
+func (m *CopyObjectRequest) GetTargetBucketName() string {
+ if m != nil {
+ return m.TargetBucketName
+ }
+ return ""
+}
+
+func (m *CopyObjectRequest) GetSrcObjectName() string {
+ if m != nil {
+ return m.SrcObjectName
+ }
+ return ""
+}
+
+func (m *CopyObjectRequest) GetTargetObjectName() string {
+ if m != nil {
+ return m.TargetObjectName
+ }
+ return ""
+}
+
+func (m *CopyObjectRequest) GetTargetBackend() string {
+ if m != nil {
+ return m.TargetBackend
+ }
+ return ""
+}
+
+func (m *CopyObjectRequest) GetTargetTier() int32 {
+ if m != nil {
+ return m.TargetTier
+ }
+ return 0
+}
+
+type CopyObjectResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ Md5 string `protobuf:"bytes,2,opt,name=md5,proto3" json:"md5,omitempty"`
+ LastModified int64 `protobuf:"varint,3,opt,name=lastModified,proto3" json:"lastModified,omitempty"`
+ VersionId string `protobuf:"bytes,4,opt,name=versionId,proto3" json:"versionId,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *CopyObjectResponse) Reset() { *m = CopyObjectResponse{} }
+func (m *CopyObjectResponse) String() string { return proto.CompactTextString(m) }
+func (*CopyObjectResponse) ProtoMessage() {}
+func (*CopyObjectResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{13}
+}
+
+func (m *CopyObjectResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_CopyObjectResponse.Unmarshal(m, b)
+}
+func (m *CopyObjectResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_CopyObjectResponse.Marshal(b, m, deterministic)
+}
+func (m *CopyObjectResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_CopyObjectResponse.Merge(m, src)
+}
+func (m *CopyObjectResponse) XXX_Size() int {
+ return xxx_messageInfo_CopyObjectResponse.Size(m)
+}
+func (m *CopyObjectResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_CopyObjectResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CopyObjectResponse proto.InternalMessageInfo
+
+func (m *CopyObjectResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *CopyObjectResponse) GetMd5() string {
+ if m != nil {
+ return m.Md5
+ }
+ return ""
+}
+
+func (m *CopyObjectResponse) GetLastModified() int64 {
+ if m != nil {
+ return m.LastModified
+ }
+ return 0
+}
+
+func (m *CopyObjectResponse) GetVersionId() string {
+ if m != nil {
+ return m.VersionId
+ }
+ return ""
+}
+
+type MoveObjectRequest struct {
+ SrcObject string `protobuf:"bytes,1,opt,name=srcObject,proto3" json:"srcObject,omitempty"`
+ TargetObject string `protobuf:"bytes,2,opt,name=targetObject,proto3" json:"targetObject,omitempty"`
+ SrcBucket string `protobuf:"bytes,3,opt,name=srcBucket,proto3" json:"srcBucket,omitempty"`
+ TargetBucket string `protobuf:"bytes,4,opt,name=targetBucket,proto3" json:"targetBucket,omitempty"`
+ TargetLocation string `protobuf:"bytes,5,opt,name=targetLocation,proto3" json:"targetLocation,omitempty"`
+ TargetTier int32 `protobuf:"varint,6,opt,name=targetTier,proto3" json:"targetTier,omitempty"`
+ MoveType int32 `protobuf:"varint,7,opt,name=moveType,proto3" json:"moveType,omitempty"`
+ SrcObjectVersion string `protobuf:"bytes,8,opt,name=srcObjectVersion,proto3" json:"srcObjectVersion,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *MoveObjectRequest) Reset() { *m = MoveObjectRequest{} }
+func (m *MoveObjectRequest) String() string { return proto.CompactTextString(m) }
+func (*MoveObjectRequest) ProtoMessage() {}
+func (*MoveObjectRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{14}
+}
+
+func (m *MoveObjectRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_MoveObjectRequest.Unmarshal(m, b)
+}
+func (m *MoveObjectRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_MoveObjectRequest.Marshal(b, m, deterministic)
+}
+func (m *MoveObjectRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_MoveObjectRequest.Merge(m, src)
+}
+func (m *MoveObjectRequest) XXX_Size() int {
+ return xxx_messageInfo_MoveObjectRequest.Size(m)
+}
+func (m *MoveObjectRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_MoveObjectRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MoveObjectRequest proto.InternalMessageInfo
+
+func (m *MoveObjectRequest) GetSrcObject() string {
+ if m != nil {
+ return m.SrcObject
+ }
+ return ""
+}
+
+func (m *MoveObjectRequest) GetTargetObject() string {
+ if m != nil {
+ return m.TargetObject
+ }
+ return ""
+}
+
+func (m *MoveObjectRequest) GetSrcBucket() string {
+ if m != nil {
+ return m.SrcBucket
+ }
+ return ""
+}
+
+func (m *MoveObjectRequest) GetTargetBucket() string {
+ if m != nil {
+ return m.TargetBucket
+ }
+ return ""
+}
+
+func (m *MoveObjectRequest) GetTargetLocation() string {
+ if m != nil {
+ return m.TargetLocation
+ }
+ return ""
+}
+
+func (m *MoveObjectRequest) GetTargetTier() int32 {
+ if m != nil {
+ return m.TargetTier
+ }
+ return 0
+}
+
+func (m *MoveObjectRequest) GetMoveType() int32 {
+ if m != nil {
+ return m.MoveType
+ }
+ return 0
+}
+
+func (m *MoveObjectRequest) GetSrcObjectVersion() string {
+ if m != nil {
+ return m.SrcObjectVersion
+ }
+ return ""
+}
+
+type MoveObjectResponse struct {
+ ObjMeta *Object `protobuf:"bytes,1,opt,name=objMeta,proto3" json:"objMeta,omitempty"`
+ Status string `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"`
+ Md5 string `protobuf:"bytes,3,opt,name=md5,proto3" json:"md5,omitempty"`
+ VersionId string `protobuf:"bytes,4,opt,name=versionId,proto3" json:"versionId,omitempty"`
+ LastModified int64 `protobuf:"varint,5,opt,name=lastModified,proto3" json:"lastModified,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *MoveObjectResponse) Reset() { *m = MoveObjectResponse{} }
+func (m *MoveObjectResponse) String() string { return proto.CompactTextString(m) }
+func (*MoveObjectResponse) ProtoMessage() {}
+func (*MoveObjectResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{15}
+}
+
+func (m *MoveObjectResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_MoveObjectResponse.Unmarshal(m, b)
+}
+func (m *MoveObjectResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_MoveObjectResponse.Marshal(b, m, deterministic)
+}
+func (m *MoveObjectResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_MoveObjectResponse.Merge(m, src)
+}
+func (m *MoveObjectResponse) XXX_Size() int {
+ return xxx_messageInfo_MoveObjectResponse.Size(m)
+}
+func (m *MoveObjectResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_MoveObjectResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MoveObjectResponse proto.InternalMessageInfo
+
+func (m *MoveObjectResponse) GetObjMeta() *Object {
+ if m != nil {
+ return m.ObjMeta
+ }
+ return nil
+}
+
+func (m *MoveObjectResponse) GetStatus() string {
+ if m != nil {
+ return m.Status
+ }
+ return ""
+}
+
+func (m *MoveObjectResponse) GetMd5() string {
+ if m != nil {
+ return m.Md5
+ }
+ return ""
+}
+
+func (m *MoveObjectResponse) GetVersionId() string {
+ if m != nil {
+ return m.VersionId
+ }
+ return ""
+}
+
+func (m *MoveObjectResponse) GetLastModified() int64 {
+ if m != nil {
+ return m.LastModified
+ }
+ return 0
+}
+
+type PutObjectRequest struct {
+ BucketName string `protobuf:"bytes,1,opt,name=BucketName,proto3" json:"BucketName,omitempty"`
+ ObjectKey string `protobuf:"bytes,2,opt,name=ObjectKey,proto3" json:"ObjectKey,omitempty"`
+ Acl *Acl `protobuf:"bytes,3,opt,name=Acl,proto3" json:"Acl,omitempty"`
+ Attrs map[string]string `protobuf:"bytes,4,rep,name=attrs,proto3" json:"attrs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+ Location string `protobuf:"bytes,5,opt,name=location,proto3" json:"location,omitempty"`
+ Size int64 `protobuf:"varint,6,opt,name=size,proto3" json:"size,omitempty"`
+ ContentType string `protobuf:"bytes,7,opt,name=contentType,proto3" json:"contentType,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *PutObjectRequest) Reset() { *m = PutObjectRequest{} }
+func (m *PutObjectRequest) String() string { return proto.CompactTextString(m) }
+func (*PutObjectRequest) ProtoMessage() {}
+func (*PutObjectRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{16}
+}
+
+func (m *PutObjectRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_PutObjectRequest.Unmarshal(m, b)
+}
+func (m *PutObjectRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_PutObjectRequest.Marshal(b, m, deterministic)
+}
+func (m *PutObjectRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_PutObjectRequest.Merge(m, src)
+}
+func (m *PutObjectRequest) XXX_Size() int {
+ return xxx_messageInfo_PutObjectRequest.Size(m)
+}
+func (m *PutObjectRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_PutObjectRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PutObjectRequest proto.InternalMessageInfo
+
+func (m *PutObjectRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *PutObjectRequest) GetObjectKey() string {
+ if m != nil {
+ return m.ObjectKey
+ }
+ return ""
+}
+
+func (m *PutObjectRequest) GetAcl() *Acl {
+ if m != nil {
+ return m.Acl
+ }
+ return nil
+}
+
+func (m *PutObjectRequest) GetAttrs() map[string]string {
+ if m != nil {
+ return m.Attrs
+ }
+ return nil
+}
+
+func (m *PutObjectRequest) GetLocation() string {
+ if m != nil {
+ return m.Location
+ }
+ return ""
+}
+
+func (m *PutObjectRequest) GetSize() int64 {
+ if m != nil {
+ return m.Size
+ }
+ return 0
+}
+
+func (m *PutObjectRequest) GetContentType() string {
+ if m != nil {
+ return m.ContentType
+ }
+ return ""
+}
+
+type GetObjectResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *GetObjectResponse) Reset() { *m = GetObjectResponse{} }
+func (m *GetObjectResponse) String() string { return proto.CompactTextString(m) }
+func (*GetObjectResponse) ProtoMessage() {}
+func (*GetObjectResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{17}
+}
+
+func (m *GetObjectResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_GetObjectResponse.Unmarshal(m, b)
+}
+func (m *GetObjectResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_GetObjectResponse.Marshal(b, m, deterministic)
+}
+func (m *GetObjectResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_GetObjectResponse.Merge(m, src)
+}
+func (m *GetObjectResponse) XXX_Size() int {
+ return xxx_messageInfo_GetObjectResponse.Size(m)
+}
+func (m *GetObjectResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_GetObjectResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetObjectResponse proto.InternalMessageInfo
+
+func (m *GetObjectResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *GetObjectResponse) GetData() []byte {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
+type GetObjectMetaResult struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ Object *Object `protobuf:"bytes,2,opt,name=object,proto3" json:"object,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *GetObjectMetaResult) Reset() { *m = GetObjectMetaResult{} }
+func (m *GetObjectMetaResult) String() string { return proto.CompactTextString(m) }
+func (*GetObjectMetaResult) ProtoMessage() {}
+func (*GetObjectMetaResult) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{18}
+}
+
+func (m *GetObjectMetaResult) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_GetObjectMetaResult.Unmarshal(m, b)
+}
+func (m *GetObjectMetaResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_GetObjectMetaResult.Marshal(b, m, deterministic)
+}
+func (m *GetObjectMetaResult) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_GetObjectMetaResult.Merge(m, src)
+}
+func (m *GetObjectMetaResult) XXX_Size() int {
+ return xxx_messageInfo_GetObjectMetaResult.Size(m)
+}
+func (m *GetObjectMetaResult) XXX_DiscardUnknown() {
+ xxx_messageInfo_GetObjectMetaResult.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetObjectMetaResult proto.InternalMessageInfo
+
+func (m *GetObjectMetaResult) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *GetObjectMetaResult) GetObject() *Object {
+ if m != nil {
+ return m.Object
+ }
+ return nil
+}
+
+type PutObjectResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ ObjMeta *Object `protobuf:"bytes,2,opt,name=objMeta,proto3" json:"objMeta,omitempty"`
+ Status string `protobuf:"bytes,3,opt,name=status,proto3" json:"status,omitempty"`
+ Md5 string `protobuf:"bytes,4,opt,name=md5,proto3" json:"md5,omitempty"`
+ VersionId string `protobuf:"bytes,5,opt,name=versionId,proto3" json:"versionId,omitempty"`
+ LastModified int64 `protobuf:"varint,6,opt,name=lastModified,proto3" json:"lastModified,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *PutObjectResponse) Reset() { *m = PutObjectResponse{} }
+func (m *PutObjectResponse) String() string { return proto.CompactTextString(m) }
+func (*PutObjectResponse) ProtoMessage() {}
+func (*PutObjectResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{19}
+}
+
+func (m *PutObjectResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_PutObjectResponse.Unmarshal(m, b)
+}
+func (m *PutObjectResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_PutObjectResponse.Marshal(b, m, deterministic)
+}
+func (m *PutObjectResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_PutObjectResponse.Merge(m, src)
+}
+func (m *PutObjectResponse) XXX_Size() int {
+ return xxx_messageInfo_PutObjectResponse.Size(m)
+}
+func (m *PutObjectResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_PutObjectResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PutObjectResponse proto.InternalMessageInfo
+
+func (m *PutObjectResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *PutObjectResponse) GetObjMeta() *Object {
+ if m != nil {
+ return m.ObjMeta
+ }
+ return nil
+}
+
+func (m *PutObjectResponse) GetStatus() string {
+ if m != nil {
+ return m.Status
+ }
+ return ""
+}
+
+func (m *PutObjectResponse) GetMd5() string {
+ if m != nil {
+ return m.Md5
+ }
+ return ""
+}
+
+func (m *PutObjectResponse) GetVersionId() string {
+ if m != nil {
+ return m.VersionId
+ }
+ return ""
+}
+
+func (m *PutObjectResponse) GetLastModified() int64 {
+ if m != nil {
+ return m.LastModified
+ }
+ return 0
+}
+
+type PutBucketACLRequest struct {
+ ACLConfig *BucketACL `protobuf:"bytes,1,opt,name=ACLConfig,proto3" json:"ACLConfig,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *PutBucketACLRequest) Reset() { *m = PutBucketACLRequest{} }
+func (m *PutBucketACLRequest) String() string { return proto.CompactTextString(m) }
+func (*PutBucketACLRequest) ProtoMessage() {}
+func (*PutBucketACLRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{20}
+}
+
+func (m *PutBucketACLRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_PutBucketACLRequest.Unmarshal(m, b)
+}
+func (m *PutBucketACLRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_PutBucketACLRequest.Marshal(b, m, deterministic)
+}
+func (m *PutBucketACLRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_PutBucketACLRequest.Merge(m, src)
+}
+func (m *PutBucketACLRequest) XXX_Size() int {
+ return xxx_messageInfo_PutBucketACLRequest.Size(m)
+}
+func (m *PutBucketACLRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_PutBucketACLRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PutBucketACLRequest proto.InternalMessageInfo
+
+func (m *PutBucketACLRequest) GetACLConfig() *BucketACL {
+ if m != nil {
+ return m.ACLConfig
+ }
+ return nil
+}
+
+type BucketACL struct {
+ BucketName string `protobuf:"bytes,1,opt,name=BucketName,proto3" json:"BucketName,omitempty"`
+ CannedAcl string `protobuf:"bytes,2,opt,name=CannedAcl,proto3" json:"CannedAcl,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *BucketACL) Reset() { *m = BucketACL{} }
+func (m *BucketACL) String() string { return proto.CompactTextString(m) }
+func (*BucketACL) ProtoMessage() {}
+func (*BucketACL) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{21}
+}
+
+func (m *BucketACL) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_BucketACL.Unmarshal(m, b)
+}
+func (m *BucketACL) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_BucketACL.Marshal(b, m, deterministic)
+}
+func (m *BucketACL) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_BucketACL.Merge(m, src)
+}
+func (m *BucketACL) XXX_Size() int {
+ return xxx_messageInfo_BucketACL.Size(m)
+}
+func (m *BucketACL) XXX_DiscardUnknown() {
+ xxx_messageInfo_BucketACL.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_BucketACL proto.InternalMessageInfo
+
+func (m *BucketACL) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *BucketACL) GetCannedAcl() string {
+ if m != nil {
+ return m.CannedAcl
+ }
+ return ""
+}
+
+type BucketVersioning struct {
+ Status string `protobuf:"bytes,1,opt,name=Status,proto3" json:"Status,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *BucketVersioning) Reset() { *m = BucketVersioning{} }
+func (m *BucketVersioning) String() string { return proto.CompactTextString(m) }
+func (*BucketVersioning) ProtoMessage() {}
+func (*BucketVersioning) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{22}
+}
+
+func (m *BucketVersioning) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_BucketVersioning.Unmarshal(m, b)
+}
+func (m *BucketVersioning) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_BucketVersioning.Marshal(b, m, deterministic)
+}
+func (m *BucketVersioning) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_BucketVersioning.Merge(m, src)
+}
+func (m *BucketVersioning) XXX_Size() int {
+ return xxx_messageInfo_BucketVersioning.Size(m)
+}
+func (m *BucketVersioning) XXX_DiscardUnknown() {
+ xxx_messageInfo_BucketVersioning.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_BucketVersioning proto.InternalMessageInfo
+
+func (m *BucketVersioning) GetStatus() string {
+ if m != nil {
+ return m.Status
+ }
+ return ""
+}
+
+type ObjACL struct {
+ BucketName string `protobuf:"bytes,1,opt,name=BucketName,proto3" json:"BucketName,omitempty"`
+ ObjectKey string `protobuf:"bytes,2,opt,name=ObjectKey,proto3" json:"ObjectKey,omitempty"`
+ CannedAcl string `protobuf:"bytes,3,opt,name=CannedAcl,proto3" json:"CannedAcl,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ObjACL) Reset() { *m = ObjACL{} }
+func (m *ObjACL) String() string { return proto.CompactTextString(m) }
+func (*ObjACL) ProtoMessage() {}
+func (*ObjACL) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{23}
+}
+
+func (m *ObjACL) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ObjACL.Unmarshal(m, b)
+}
+func (m *ObjACL) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ObjACL.Marshal(b, m, deterministic)
+}
+func (m *ObjACL) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ObjACL.Merge(m, src)
+}
+func (m *ObjACL) XXX_Size() int {
+ return xxx_messageInfo_ObjACL.Size(m)
+}
+func (m *ObjACL) XXX_DiscardUnknown() {
+ xxx_messageInfo_ObjACL.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ObjACL proto.InternalMessageInfo
+
+func (m *ObjACL) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *ObjACL) GetObjectKey() string {
+ if m != nil {
+ return m.ObjectKey
+ }
+ return ""
+}
+
+func (m *ObjACL) GetCannedAcl() string {
+ if m != nil {
+ return m.CannedAcl
+ }
+ return ""
+}
+
+type PutObjACLRequest struct {
+ Context string `protobuf:"bytes,1,opt,name=Context,proto3" json:"Context,omitempty"`
+ ACLConfig *ObjACL `protobuf:"bytes,2,opt,name=ACLConfig,proto3" json:"ACLConfig,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *PutObjACLRequest) Reset() { *m = PutObjACLRequest{} }
+func (m *PutObjACLRequest) String() string { return proto.CompactTextString(m) }
+func (*PutObjACLRequest) ProtoMessage() {}
+func (*PutObjACLRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{24}
+}
+
+func (m *PutObjACLRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_PutObjACLRequest.Unmarshal(m, b)
+}
+func (m *PutObjACLRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_PutObjACLRequest.Marshal(b, m, deterministic)
+}
+func (m *PutObjACLRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_PutObjACLRequest.Merge(m, src)
+}
+func (m *PutObjACLRequest) XXX_Size() int {
+ return xxx_messageInfo_PutObjACLRequest.Size(m)
+}
+func (m *PutObjACLRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_PutObjACLRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PutObjACLRequest proto.InternalMessageInfo
+
+func (m *PutObjACLRequest) GetContext() string {
+ if m != nil {
+ return m.Context
+ }
+ return ""
+}
+
+func (m *PutObjACLRequest) GetACLConfig() *ObjACL {
+ if m != nil {
+ return m.ACLConfig
+ }
+ return nil
+}
+
+type BaseBucketRequest struct {
+ Context string `protobuf:"bytes,1,opt,name=Context,proto3" json:"Context,omitempty"`
+ BucketName string `protobuf:"bytes,2,opt,name=BucketName,proto3" json:"BucketName,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *BaseBucketRequest) Reset() { *m = BaseBucketRequest{} }
+func (m *BaseBucketRequest) String() string { return proto.CompactTextString(m) }
+func (*BaseBucketRequest) ProtoMessage() {}
+func (*BaseBucketRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{25}
+}
+
+func (m *BaseBucketRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_BaseBucketRequest.Unmarshal(m, b)
+}
+func (m *BaseBucketRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_BaseBucketRequest.Marshal(b, m, deterministic)
+}
+func (m *BaseBucketRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_BaseBucketRequest.Merge(m, src)
+}
+func (m *BaseBucketRequest) XXX_Size() int {
+ return xxx_messageInfo_BaseBucketRequest.Size(m)
+}
+func (m *BaseBucketRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_BaseBucketRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_BaseBucketRequest proto.InternalMessageInfo
+
+func (m *BaseBucketRequest) GetContext() string {
+ if m != nil {
+ return m.Context
+ }
+ return ""
+}
+
+func (m *BaseBucketRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+type BaseObjRequest struct {
+ Context string `protobuf:"bytes,1,opt,name=Context,proto3" json:"Context,omitempty"`
+ BucketName string `protobuf:"bytes,2,opt,name=BucketName,proto3" json:"BucketName,omitempty"`
+ ObjectKey string `protobuf:"bytes,3,opt,name=ObjectKey,proto3" json:"ObjectKey,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *BaseObjRequest) Reset() { *m = BaseObjRequest{} }
+func (m *BaseObjRequest) String() string { return proto.CompactTextString(m) }
+func (*BaseObjRequest) ProtoMessage() {}
+func (*BaseObjRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{26}
+}
+
+func (m *BaseObjRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_BaseObjRequest.Unmarshal(m, b)
+}
+func (m *BaseObjRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_BaseObjRequest.Marshal(b, m, deterministic)
+}
+func (m *BaseObjRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_BaseObjRequest.Merge(m, src)
+}
+func (m *BaseObjRequest) XXX_Size() int {
+ return xxx_messageInfo_BaseObjRequest.Size(m)
+}
+func (m *BaseObjRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_BaseObjRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_BaseObjRequest proto.InternalMessageInfo
+
+func (m *BaseObjRequest) GetContext() string {
+ if m != nil {
+ return m.Context
+ }
+ return ""
+}
+
+func (m *BaseObjRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *BaseObjRequest) GetObjectKey() string {
+ if m != nil {
+ return m.ObjectKey
+ }
+ return ""
+}
+
+type CopyObjPartRequest struct {
+ SourceBucket string `protobuf:"bytes,1,opt,name=SourceBucket,proto3" json:"SourceBucket,omitempty"`
+ SourceObject string `protobuf:"bytes,2,opt,name=SourceObject,proto3" json:"SourceObject,omitempty"`
+ TargetBucket string `protobuf:"bytes,3,opt,name=TargetBucket,proto3" json:"TargetBucket,omitempty"`
+ TargetObject string `protobuf:"bytes,4,opt,name=TargetObject,proto3" json:"TargetObject,omitempty"`
+ TargetLocation string `protobuf:"bytes,5,opt,name=TargetLocation,proto3" json:"TargetLocation,omitempty"`
+ UploadID string `protobuf:"bytes,6,opt,name=UploadID,proto3" json:"UploadID,omitempty"`
+ PartID int64 `protobuf:"varint,7,opt,name=PartID,proto3" json:"PartID,omitempty"`
+ ReadOffset int64 `protobuf:"varint,8,opt,name=ReadOffset,proto3" json:"ReadOffset,omitempty"`
+ ReadLength int64 `protobuf:"varint,9,opt,name=ReadLength,proto3" json:"ReadLength,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *CopyObjPartRequest) Reset() { *m = CopyObjPartRequest{} }
+func (m *CopyObjPartRequest) String() string { return proto.CompactTextString(m) }
+func (*CopyObjPartRequest) ProtoMessage() {}
+func (*CopyObjPartRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{27}
+}
+
+func (m *CopyObjPartRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_CopyObjPartRequest.Unmarshal(m, b)
+}
+func (m *CopyObjPartRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_CopyObjPartRequest.Marshal(b, m, deterministic)
+}
+func (m *CopyObjPartRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_CopyObjPartRequest.Merge(m, src)
+}
+func (m *CopyObjPartRequest) XXX_Size() int {
+ return xxx_messageInfo_CopyObjPartRequest.Size(m)
+}
+func (m *CopyObjPartRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_CopyObjPartRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CopyObjPartRequest proto.InternalMessageInfo
+
+func (m *CopyObjPartRequest) GetSourceBucket() string {
+ if m != nil {
+ return m.SourceBucket
+ }
+ return ""
+}
+
+func (m *CopyObjPartRequest) GetSourceObject() string {
+ if m != nil {
+ return m.SourceObject
+ }
+ return ""
+}
+
+func (m *CopyObjPartRequest) GetTargetBucket() string {
+ if m != nil {
+ return m.TargetBucket
+ }
+ return ""
+}
+
+func (m *CopyObjPartRequest) GetTargetObject() string {
+ if m != nil {
+ return m.TargetObject
+ }
+ return ""
+}
+
+func (m *CopyObjPartRequest) GetTargetLocation() string {
+ if m != nil {
+ return m.TargetLocation
+ }
+ return ""
+}
+
+func (m *CopyObjPartRequest) GetUploadID() string {
+ if m != nil {
+ return m.UploadID
+ }
+ return ""
+}
+
+func (m *CopyObjPartRequest) GetPartID() int64 {
+ if m != nil {
+ return m.PartID
+ }
+ return 0
+}
+
+func (m *CopyObjPartRequest) GetReadOffset() int64 {
+ if m != nil {
+ return m.ReadOffset
+ }
+ return 0
+}
+
+func (m *CopyObjPartRequest) GetReadLength() int64 {
+ if m != nil {
+ return m.ReadLength
+ }
+ return 0
+}
+
+type CopyObjPartResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=ErrorCode,proto3" json:"ErrorCode,omitempty"`
+ LastModified int64 `protobuf:"varint,2,opt,name=LastModified,proto3" json:"LastModified,omitempty"`
+ Etag string `protobuf:"bytes,3,opt,name=Etag,proto3" json:"Etag,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *CopyObjPartResponse) Reset() { *m = CopyObjPartResponse{} }
+func (m *CopyObjPartResponse) String() string { return proto.CompactTextString(m) }
+func (*CopyObjPartResponse) ProtoMessage() {}
+func (*CopyObjPartResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{28}
+}
+
+func (m *CopyObjPartResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_CopyObjPartResponse.Unmarshal(m, b)
+}
+func (m *CopyObjPartResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_CopyObjPartResponse.Marshal(b, m, deterministic)
+}
+func (m *CopyObjPartResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_CopyObjPartResponse.Merge(m, src)
+}
+func (m *CopyObjPartResponse) XXX_Size() int {
+ return xxx_messageInfo_CopyObjPartResponse.Size(m)
+}
+func (m *CopyObjPartResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_CopyObjPartResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CopyObjPartResponse proto.InternalMessageInfo
+
+func (m *CopyObjPartResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *CopyObjPartResponse) GetLastModified() int64 {
+ if m != nil {
+ return m.LastModified
+ }
+ return 0
+}
+
+func (m *CopyObjPartResponse) GetEtag() string {
+ if m != nil {
+ return m.Etag
+ }
+ return ""
+}
+
+type ServerSideEncryption struct {
+ SseType string `protobuf:"bytes,1,opt,name=sseType,proto3" json:"sseType,omitempty"`
+ EncryptionKey []byte `protobuf:"bytes,2,opt,name=encryptionKey,proto3" json:"encryptionKey,omitempty"`
+ InitilizationVector []byte `protobuf:"bytes,3,opt,name=initilizationVector,proto3" json:"initilizationVector,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ServerSideEncryption) Reset() { *m = ServerSideEncryption{} }
+func (m *ServerSideEncryption) String() string { return proto.CompactTextString(m) }
+func (*ServerSideEncryption) ProtoMessage() {}
+func (*ServerSideEncryption) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{29}
+}
+
+func (m *ServerSideEncryption) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ServerSideEncryption.Unmarshal(m, b)
+}
+func (m *ServerSideEncryption) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ServerSideEncryption.Marshal(b, m, deterministic)
+}
+func (m *ServerSideEncryption) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ServerSideEncryption.Merge(m, src)
+}
+func (m *ServerSideEncryption) XXX_Size() int {
+ return xxx_messageInfo_ServerSideEncryption.Size(m)
+}
+func (m *ServerSideEncryption) XXX_DiscardUnknown() {
+ xxx_messageInfo_ServerSideEncryption.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ServerSideEncryption proto.InternalMessageInfo
+
+func (m *ServerSideEncryption) GetSseType() string {
+ if m != nil {
+ return m.SseType
+ }
+ return ""
+}
+
+func (m *ServerSideEncryption) GetEncryptionKey() []byte {
+ if m != nil {
+ return m.EncryptionKey
+ }
+ return nil
+}
+
+func (m *ServerSideEncryption) GetInitilizationVector() []byte {
+ if m != nil {
+ return m.InitilizationVector
+ }
+ return nil
+}
+
+type RedirectAllRequestsTo struct {
+ HostName string `protobuf:"bytes,1,opt,name=hostName,proto3" json:"hostName,omitempty"`
+ Protocol []string `protobuf:"bytes,2,rep,name=protocol,proto3" json:"protocol,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *RedirectAllRequestsTo) Reset() { *m = RedirectAllRequestsTo{} }
+func (m *RedirectAllRequestsTo) String() string { return proto.CompactTextString(m) }
+func (*RedirectAllRequestsTo) ProtoMessage() {}
+func (*RedirectAllRequestsTo) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{30}
+}
+
+func (m *RedirectAllRequestsTo) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_RedirectAllRequestsTo.Unmarshal(m, b)
+}
+func (m *RedirectAllRequestsTo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_RedirectAllRequestsTo.Marshal(b, m, deterministic)
+}
+func (m *RedirectAllRequestsTo) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_RedirectAllRequestsTo.Merge(m, src)
+}
+func (m *RedirectAllRequestsTo) XXX_Size() int {
+ return xxx_messageInfo_RedirectAllRequestsTo.Size(m)
+}
+func (m *RedirectAllRequestsTo) XXX_DiscardUnknown() {
+ xxx_messageInfo_RedirectAllRequestsTo.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_RedirectAllRequestsTo proto.InternalMessageInfo
+
+func (m *RedirectAllRequestsTo) GetHostName() string {
+ if m != nil {
+ return m.HostName
+ }
+ return ""
+}
+
+func (m *RedirectAllRequestsTo) GetProtocol() []string {
+ if m != nil {
+ return m.Protocol
+ }
+ return nil
+}
+
+type Redirect struct {
+ Protocol string `protobuf:"bytes,1,opt,name=protocol,proto3" json:"protocol,omitempty"`
+ HostName string `protobuf:"bytes,2,opt,name=hostName,proto3" json:"hostName,omitempty"`
+ ReplaceKeyPrefixWith string `protobuf:"bytes,3,opt,name=replaceKeyPrefixWith,proto3" json:"replaceKeyPrefixWith,omitempty"`
+ ReplaceKeyWith string `protobuf:"bytes,4,opt,name=replaceKeyWith,proto3" json:"replaceKeyWith,omitempty"`
+ HttpRedirectCode string `protobuf:"bytes,5,opt,name=httpRedirectCode,proto3" json:"httpRedirectCode,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Redirect) Reset() { *m = Redirect{} }
+func (m *Redirect) String() string { return proto.CompactTextString(m) }
+func (*Redirect) ProtoMessage() {}
+func (*Redirect) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{31}
+}
+
+func (m *Redirect) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Redirect.Unmarshal(m, b)
+}
+func (m *Redirect) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Redirect.Marshal(b, m, deterministic)
+}
+func (m *Redirect) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Redirect.Merge(m, src)
+}
+func (m *Redirect) XXX_Size() int {
+ return xxx_messageInfo_Redirect.Size(m)
+}
+func (m *Redirect) XXX_DiscardUnknown() {
+ xxx_messageInfo_Redirect.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Redirect proto.InternalMessageInfo
+
+func (m *Redirect) GetProtocol() string {
+ if m != nil {
+ return m.Protocol
+ }
+ return ""
+}
+
+func (m *Redirect) GetHostName() string {
+ if m != nil {
+ return m.HostName
+ }
+ return ""
+}
+
+func (m *Redirect) GetReplaceKeyPrefixWith() string {
+ if m != nil {
+ return m.ReplaceKeyPrefixWith
+ }
+ return ""
+}
+
+func (m *Redirect) GetReplaceKeyWith() string {
+ if m != nil {
+ return m.ReplaceKeyWith
+ }
+ return ""
+}
+
+func (m *Redirect) GetHttpRedirectCode() string {
+ if m != nil {
+ return m.HttpRedirectCode
+ }
+ return ""
+}
+
+type Condition struct {
+ KeyPrefixEquals string `protobuf:"bytes,1,opt,name=keyPrefixEquals,proto3" json:"keyPrefixEquals,omitempty"`
+ HttpErrorCodeReturnedEquals string `protobuf:"bytes,2,opt,name=httpErrorCodeReturnedEquals,proto3" json:"httpErrorCodeReturnedEquals,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Condition) Reset() { *m = Condition{} }
+func (m *Condition) String() string { return proto.CompactTextString(m) }
+func (*Condition) ProtoMessage() {}
+func (*Condition) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{32}
+}
+
+func (m *Condition) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Condition.Unmarshal(m, b)
+}
+func (m *Condition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Condition.Marshal(b, m, deterministic)
+}
+func (m *Condition) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Condition.Merge(m, src)
+}
+func (m *Condition) XXX_Size() int {
+ return xxx_messageInfo_Condition.Size(m)
+}
+func (m *Condition) XXX_DiscardUnknown() {
+ xxx_messageInfo_Condition.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Condition proto.InternalMessageInfo
+
+func (m *Condition) GetKeyPrefixEquals() string {
+ if m != nil {
+ return m.KeyPrefixEquals
+ }
+ return ""
+}
+
+func (m *Condition) GetHttpErrorCodeReturnedEquals() string {
+ if m != nil {
+ return m.HttpErrorCodeReturnedEquals
+ }
+ return ""
+}
+
+type RoutingRules struct {
+ Redirect *Redirect `protobuf:"bytes,1,opt,name=redirect,proto3" json:"redirect,omitempty"`
+ Condition *Condition `protobuf:"bytes,2,opt,name=condition,proto3" json:"condition,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *RoutingRules) Reset() { *m = RoutingRules{} }
+func (m *RoutingRules) String() string { return proto.CompactTextString(m) }
+func (*RoutingRules) ProtoMessage() {}
+func (*RoutingRules) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{33}
+}
+
+func (m *RoutingRules) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_RoutingRules.Unmarshal(m, b)
+}
+func (m *RoutingRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_RoutingRules.Marshal(b, m, deterministic)
+}
+func (m *RoutingRules) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_RoutingRules.Merge(m, src)
+}
+func (m *RoutingRules) XXX_Size() int {
+ return xxx_messageInfo_RoutingRules.Size(m)
+}
+func (m *RoutingRules) XXX_DiscardUnknown() {
+ xxx_messageInfo_RoutingRules.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_RoutingRules proto.InternalMessageInfo
+
+func (m *RoutingRules) GetRedirect() *Redirect {
+ if m != nil {
+ return m.Redirect
+ }
+ return nil
+}
+
+func (m *RoutingRules) GetCondition() *Condition {
+ if m != nil {
+ return m.Condition
+ }
+ return nil
+}
+
+type WebsiteConfiguration struct {
+ IndexDocument string `protobuf:"bytes,1,opt,name=indexDocument,proto3" json:"indexDocument,omitempty"`
+ ErrorDocument string `protobuf:"bytes,2,opt,name=errorDocument,proto3" json:"errorDocument,omitempty"`
+ RedirectAllRequestsTo *RedirectAllRequestsTo `protobuf:"bytes,3,opt,name=redirectAllRequestsTo,proto3" json:"redirectAllRequestsTo,omitempty"`
+ RoutingRules []*RoutingRules `protobuf:"bytes,4,rep,name=routingRules,proto3" json:"routingRules,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *WebsiteConfiguration) Reset() { *m = WebsiteConfiguration{} }
+func (m *WebsiteConfiguration) String() string { return proto.CompactTextString(m) }
+func (*WebsiteConfiguration) ProtoMessage() {}
+func (*WebsiteConfiguration) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{34}
+}
+
+func (m *WebsiteConfiguration) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_WebsiteConfiguration.Unmarshal(m, b)
+}
+func (m *WebsiteConfiguration) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_WebsiteConfiguration.Marshal(b, m, deterministic)
+}
+func (m *WebsiteConfiguration) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_WebsiteConfiguration.Merge(m, src)
+}
+func (m *WebsiteConfiguration) XXX_Size() int {
+ return xxx_messageInfo_WebsiteConfiguration.Size(m)
+}
+func (m *WebsiteConfiguration) XXX_DiscardUnknown() {
+ xxx_messageInfo_WebsiteConfiguration.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_WebsiteConfiguration proto.InternalMessageInfo
+
+func (m *WebsiteConfiguration) GetIndexDocument() string {
+ if m != nil {
+ return m.IndexDocument
+ }
+ return ""
+}
+
+func (m *WebsiteConfiguration) GetErrorDocument() string {
+ if m != nil {
+ return m.ErrorDocument
+ }
+ return ""
+}
+
+func (m *WebsiteConfiguration) GetRedirectAllRequestsTo() *RedirectAllRequestsTo {
+ if m != nil {
+ return m.RedirectAllRequestsTo
+ }
+ return nil
+}
+
+func (m *WebsiteConfiguration) GetRoutingRules() []*RoutingRules {
+ if m != nil {
+ return m.RoutingRules
+ }
+ return nil
+}
+
+type CORSConfiguration struct {
+ Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+ AllowedMethods string `protobuf:"bytes,2,opt,name=allowedMethods,proto3" json:"allowedMethods,omitempty"`
+ AllowedOrigins string `protobuf:"bytes,3,opt,name=allowedOrigins,proto3" json:"allowedOrigins,omitempty"`
+ AllowedHeaders string `protobuf:"bytes,4,opt,name=allowedHeaders,proto3" json:"allowedHeaders,omitempty"`
+ MaxAgeSeconds string `protobuf:"bytes,5,opt,name=maxAgeSeconds,proto3" json:"maxAgeSeconds,omitempty"`
+ ExposedHeaders string `protobuf:"bytes,6,opt,name=exposedHeaders,proto3" json:"exposedHeaders,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *CORSConfiguration) Reset() { *m = CORSConfiguration{} }
+func (m *CORSConfiguration) String() string { return proto.CompactTextString(m) }
+func (*CORSConfiguration) ProtoMessage() {}
+func (*CORSConfiguration) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{35}
+}
+
+func (m *CORSConfiguration) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_CORSConfiguration.Unmarshal(m, b)
+}
+func (m *CORSConfiguration) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_CORSConfiguration.Marshal(b, m, deterministic)
+}
+func (m *CORSConfiguration) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_CORSConfiguration.Merge(m, src)
+}
+func (m *CORSConfiguration) XXX_Size() int {
+ return xxx_messageInfo_CORSConfiguration.Size(m)
+}
+func (m *CORSConfiguration) XXX_DiscardUnknown() {
+ xxx_messageInfo_CORSConfiguration.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CORSConfiguration proto.InternalMessageInfo
+
+func (m *CORSConfiguration) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *CORSConfiguration) GetAllowedMethods() string {
+ if m != nil {
+ return m.AllowedMethods
+ }
+ return ""
+}
+
+func (m *CORSConfiguration) GetAllowedOrigins() string {
+ if m != nil {
+ return m.AllowedOrigins
+ }
+ return ""
+}
+
+func (m *CORSConfiguration) GetAllowedHeaders() string {
+ if m != nil {
+ return m.AllowedHeaders
+ }
+ return ""
+}
+
+func (m *CORSConfiguration) GetMaxAgeSeconds() string {
+ if m != nil {
+ return m.MaxAgeSeconds
+ }
+ return ""
+}
+
+func (m *CORSConfiguration) GetExposedHeaders() string {
+ if m != nil {
+ return m.ExposedHeaders
+ }
+ return ""
+}
+
+type Destination struct {
+ Bucket string `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"`
+ StorageClass string `protobuf:"bytes,2,opt,name=storageClass,proto3" json:"storageClass,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Destination) Reset() { *m = Destination{} }
+func (m *Destination) String() string { return proto.CompactTextString(m) }
+func (*Destination) ProtoMessage() {}
+func (*Destination) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{36}
+}
+
+func (m *Destination) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Destination.Unmarshal(m, b)
+}
+func (m *Destination) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Destination.Marshal(b, m, deterministic)
+}
+func (m *Destination) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Destination.Merge(m, src)
+}
+func (m *Destination) XXX_Size() int {
+ return xxx_messageInfo_Destination.Size(m)
+}
+func (m *Destination) XXX_DiscardUnknown() {
+ xxx_messageInfo_Destination.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Destination proto.InternalMessageInfo
+
+func (m *Destination) GetBucket() string {
+ if m != nil {
+ return m.Bucket
+ }
+ return ""
+}
+
+func (m *Destination) GetStorageClass() string {
+ if m != nil {
+ return m.StorageClass
+ }
+ return ""
+}
+
+type ReplicationRole struct {
+ Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+ Prefix string `protobuf:"bytes,2,opt,name=prefix,proto3" json:"prefix,omitempty"`
+ Enabled bool `protobuf:"varint,3,opt,name=enabled,proto3" json:"enabled,omitempty"`
+ Destination *Destination `protobuf:"bytes,4,opt,name=destination,proto3" json:"destination,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ReplicationRole) Reset() { *m = ReplicationRole{} }
+func (m *ReplicationRole) String() string { return proto.CompactTextString(m) }
+func (*ReplicationRole) ProtoMessage() {}
+func (*ReplicationRole) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{37}
+}
+
+func (m *ReplicationRole) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ReplicationRole.Unmarshal(m, b)
+}
+func (m *ReplicationRole) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ReplicationRole.Marshal(b, m, deterministic)
+}
+func (m *ReplicationRole) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ReplicationRole.Merge(m, src)
+}
+func (m *ReplicationRole) XXX_Size() int {
+ return xxx_messageInfo_ReplicationRole.Size(m)
+}
+func (m *ReplicationRole) XXX_DiscardUnknown() {
+ xxx_messageInfo_ReplicationRole.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ReplicationRole proto.InternalMessageInfo
+
+func (m *ReplicationRole) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *ReplicationRole) GetPrefix() string {
+ if m != nil {
+ return m.Prefix
+ }
+ return ""
+}
+
+func (m *ReplicationRole) GetEnabled() bool {
+ if m != nil {
+ return m.Enabled
+ }
+ return false
+}
+
+func (m *ReplicationRole) GetDestination() *Destination {
+ if m != nil {
+ return m.Destination
+ }
+ return nil
+}
+
+type ReplicationConfiguration struct {
+ Role string `protobuf:"bytes,1,opt,name=role,proto3" json:"role,omitempty"`
+ Rules []*ReplicationRole `protobuf:"bytes,2,rep,name=rules,proto3" json:"rules,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ReplicationConfiguration) Reset() { *m = ReplicationConfiguration{} }
+func (m *ReplicationConfiguration) String() string { return proto.CompactTextString(m) }
+func (*ReplicationConfiguration) ProtoMessage() {}
+func (*ReplicationConfiguration) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{38}
+}
+
+func (m *ReplicationConfiguration) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ReplicationConfiguration.Unmarshal(m, b)
+}
+func (m *ReplicationConfiguration) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ReplicationConfiguration.Marshal(b, m, deterministic)
+}
+func (m *ReplicationConfiguration) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ReplicationConfiguration.Merge(m, src)
+}
+func (m *ReplicationConfiguration) XXX_Size() int {
+ return xxx_messageInfo_ReplicationConfiguration.Size(m)
+}
+func (m *ReplicationConfiguration) XXX_DiscardUnknown() {
+ xxx_messageInfo_ReplicationConfiguration.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ReplicationConfiguration proto.InternalMessageInfo
+
+func (m *ReplicationConfiguration) GetRole() string {
+ if m != nil {
+ return m.Role
+ }
+ return ""
+}
+
+func (m *ReplicationConfiguration) GetRules() []*ReplicationRole {
+ if m != nil {
+ return m.Rules
+ }
+ return nil
+}
+
+type Tag struct {
+ Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+ Val string `protobuf:"bytes,2,opt,name=val,proto3" json:"val,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Tag) Reset() { *m = Tag{} }
+func (m *Tag) String() string { return proto.CompactTextString(m) }
+func (*Tag) ProtoMessage() {}
+func (*Tag) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{39}
+}
+
+func (m *Tag) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Tag.Unmarshal(m, b)
+}
+func (m *Tag) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Tag.Marshal(b, m, deterministic)
+}
+func (m *Tag) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Tag.Merge(m, src)
+}
+func (m *Tag) XXX_Size() int {
+ return xxx_messageInfo_Tag.Size(m)
+}
+func (m *Tag) XXX_DiscardUnknown() {
+ xxx_messageInfo_Tag.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Tag proto.InternalMessageInfo
+
+func (m *Tag) GetKey() string {
+ if m != nil {
+ return m.Key
+ }
+ return ""
+}
+
+func (m *Tag) GetVal() string {
+ if m != nil {
+ return m.Val
+ }
+ return ""
+}
+
+type LifecycleFilter struct {
+ //Object prefix for lifecycle filter
+ Prefix string `protobuf:"bytes,1,opt,name=prefix,proto3" json:"prefix,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *LifecycleFilter) Reset() { *m = LifecycleFilter{} }
+func (m *LifecycleFilter) String() string { return proto.CompactTextString(m) }
+func (*LifecycleFilter) ProtoMessage() {}
+func (*LifecycleFilter) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{40}
+}
+
+func (m *LifecycleFilter) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_LifecycleFilter.Unmarshal(m, b)
+}
+func (m *LifecycleFilter) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_LifecycleFilter.Marshal(b, m, deterministic)
+}
+func (m *LifecycleFilter) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_LifecycleFilter.Merge(m, src)
+}
+func (m *LifecycleFilter) XXX_Size() int {
+ return xxx_messageInfo_LifecycleFilter.Size(m)
+}
+func (m *LifecycleFilter) XXX_DiscardUnknown() {
+ xxx_messageInfo_LifecycleFilter.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_LifecycleFilter proto.InternalMessageInfo
+
+func (m *LifecycleFilter) GetPrefix() string {
+ if m != nil {
+ return m.Prefix
+ }
+ return ""
+}
+
+type Action struct {
+ //Name of the action transition/expiration
+ Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+ //Days after creation of object
+ Days int32 `protobuf:"varint,2,opt,name=days,proto3" json:"days,omitempty"`
+ //Delete marker in case of expiration for versioned bucket
+ DeleteMarker string `protobuf:"bytes,3,opt,name=deleteMarker,proto3" json:"deleteMarker,omitempty"`
+ //Storage class tier of the object where object is to be transitioned
+ Tier int32 `protobuf:"varint,4,opt,name=tier,proto3" json:"tier,omitempty"`
+ // Destination backend of the object/bucket for Cross-cloud transition
+ Backend string `protobuf:"bytes,5,opt,name=backend,proto3" json:"backend,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Action) Reset() { *m = Action{} }
+func (m *Action) String() string { return proto.CompactTextString(m) }
+func (*Action) ProtoMessage() {}
+func (*Action) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{41}
+}
+
+func (m *Action) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Action.Unmarshal(m, b)
+}
+func (m *Action) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Action.Marshal(b, m, deterministic)
+}
+func (m *Action) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Action.Merge(m, src)
+}
+func (m *Action) XXX_Size() int {
+ return xxx_messageInfo_Action.Size(m)
+}
+func (m *Action) XXX_DiscardUnknown() {
+ xxx_messageInfo_Action.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Action proto.InternalMessageInfo
+
+func (m *Action) GetName() string {
+ if m != nil {
+ return m.Name
+ }
+ return ""
+}
+
+func (m *Action) GetDays() int32 {
+ if m != nil {
+ return m.Days
+ }
+ return 0
+}
+
+func (m *Action) GetDeleteMarker() string {
+ if m != nil {
+ return m.DeleteMarker
+ }
+ return ""
+}
+
+func (m *Action) GetTier() int32 {
+ if m != nil {
+ return m.Tier
+ }
+ return 0
+}
+
+func (m *Action) GetBackend() string {
+ if m != nil {
+ return m.Backend
+ }
+ return ""
+}
+
+type AbortMultipartUpload struct {
+ //Days after which the abort operation will be performed on incomplete upload
+ DaysAfterInitiation int32 `protobuf:"varint,1,opt,name=daysAfterInitiation,proto3" json:"daysAfterInitiation,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *AbortMultipartUpload) Reset() { *m = AbortMultipartUpload{} }
+func (m *AbortMultipartUpload) String() string { return proto.CompactTextString(m) }
+func (*AbortMultipartUpload) ProtoMessage() {}
+func (*AbortMultipartUpload) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{42}
+}
+
+func (m *AbortMultipartUpload) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_AbortMultipartUpload.Unmarshal(m, b)
+}
+func (m *AbortMultipartUpload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_AbortMultipartUpload.Marshal(b, m, deterministic)
+}
+func (m *AbortMultipartUpload) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_AbortMultipartUpload.Merge(m, src)
+}
+func (m *AbortMultipartUpload) XXX_Size() int {
+ return xxx_messageInfo_AbortMultipartUpload.Size(m)
+}
+func (m *AbortMultipartUpload) XXX_DiscardUnknown() {
+ xxx_messageInfo_AbortMultipartUpload.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_AbortMultipartUpload proto.InternalMessageInfo
+
+func (m *AbortMultipartUpload) GetDaysAfterInitiation() int32 {
+ if m != nil {
+ return m.DaysAfterInitiation
+ }
+ return 0
+}
+
+type LifecycleRule struct {
+ Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+ Status string `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"`
+ Filter *LifecycleFilter `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"`
+ Actions []*Action `protobuf:"bytes,4,rep,name=actions,proto3" json:"actions,omitempty"`
+ AbortIncompleteMultipartUpload *AbortMultipartUpload `protobuf:"bytes,5,opt,name=abortIncompleteMultipartUpload,proto3" json:"abortIncompleteMultipartUpload,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *LifecycleRule) Reset() { *m = LifecycleRule{} }
+func (m *LifecycleRule) String() string { return proto.CompactTextString(m) }
+func (*LifecycleRule) ProtoMessage() {}
+func (*LifecycleRule) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{43}
+}
+
+func (m *LifecycleRule) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_LifecycleRule.Unmarshal(m, b)
+}
+func (m *LifecycleRule) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_LifecycleRule.Marshal(b, m, deterministic)
+}
+func (m *LifecycleRule) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_LifecycleRule.Merge(m, src)
+}
+func (m *LifecycleRule) XXX_Size() int {
+ return xxx_messageInfo_LifecycleRule.Size(m)
+}
+func (m *LifecycleRule) XXX_DiscardUnknown() {
+ xxx_messageInfo_LifecycleRule.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_LifecycleRule proto.InternalMessageInfo
+
+func (m *LifecycleRule) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *LifecycleRule) GetStatus() string {
+ if m != nil {
+ return m.Status
+ }
+ return ""
+}
+
+func (m *LifecycleRule) GetFilter() *LifecycleFilter {
+ if m != nil {
+ return m.Filter
+ }
+ return nil
+}
+
+func (m *LifecycleRule) GetActions() []*Action {
+ if m != nil {
+ return m.Actions
+ }
+ return nil
+}
+
+func (m *LifecycleRule) GetAbortIncompleteMultipartUpload() *AbortMultipartUpload {
+ if m != nil {
+ return m.AbortIncompleteMultipartUpload
+ }
+ return nil
+}
+
+type PutBucketLifecycleRequest struct {
+ BucketName string `protobuf:"bytes,1,opt,name=bucketName,proto3" json:"bucketName,omitempty"`
+ Lc []*LifecycleRule `protobuf:"bytes,2,rep,name=lc,proto3" json:"lc,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *PutBucketLifecycleRequest) Reset() { *m = PutBucketLifecycleRequest{} }
+func (m *PutBucketLifecycleRequest) String() string { return proto.CompactTextString(m) }
+func (*PutBucketLifecycleRequest) ProtoMessage() {}
+func (*PutBucketLifecycleRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{44}
+}
+
+func (m *PutBucketLifecycleRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_PutBucketLifecycleRequest.Unmarshal(m, b)
+}
+func (m *PutBucketLifecycleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_PutBucketLifecycleRequest.Marshal(b, m, deterministic)
+}
+func (m *PutBucketLifecycleRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_PutBucketLifecycleRequest.Merge(m, src)
+}
+func (m *PutBucketLifecycleRequest) XXX_Size() int {
+ return xxx_messageInfo_PutBucketLifecycleRequest.Size(m)
+}
+func (m *PutBucketLifecycleRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_PutBucketLifecycleRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PutBucketLifecycleRequest proto.InternalMessageInfo
+
+func (m *PutBucketLifecycleRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *PutBucketLifecycleRequest) GetLc() []*LifecycleRule {
+ if m != nil {
+ return m.Lc
+ }
+ return nil
+}
+
+type GetBucketLifecycleResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ Lc []*LifecycleRule `protobuf:"bytes,2,rep,name=lc,proto3" json:"lc,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *GetBucketLifecycleResponse) Reset() { *m = GetBucketLifecycleResponse{} }
+func (m *GetBucketLifecycleResponse) String() string { return proto.CompactTextString(m) }
+func (*GetBucketLifecycleResponse) ProtoMessage() {}
+func (*GetBucketLifecycleResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{45}
+}
+
+func (m *GetBucketLifecycleResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_GetBucketLifecycleResponse.Unmarshal(m, b)
+}
+func (m *GetBucketLifecycleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_GetBucketLifecycleResponse.Marshal(b, m, deterministic)
+}
+func (m *GetBucketLifecycleResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_GetBucketLifecycleResponse.Merge(m, src)
+}
+func (m *GetBucketLifecycleResponse) XXX_Size() int {
+ return xxx_messageInfo_GetBucketLifecycleResponse.Size(m)
+}
+func (m *GetBucketLifecycleResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_GetBucketLifecycleResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetBucketLifecycleResponse proto.InternalMessageInfo
+
+func (m *GetBucketLifecycleResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *GetBucketLifecycleResponse) GetLc() []*LifecycleRule {
+ if m != nil {
+ return m.Lc
+ }
+ return nil
+}
+
+type ReplicationInfo struct {
+ Status string `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"`
+ Backend string `protobuf:"bytes,2,opt,name=backend,proto3" json:"backend,omitempty"`
+ StorageClass string `protobuf:"bytes,3,opt,name=storageClass,proto3" json:"storageClass,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ReplicationInfo) Reset() { *m = ReplicationInfo{} }
+func (m *ReplicationInfo) String() string { return proto.CompactTextString(m) }
+func (*ReplicationInfo) ProtoMessage() {}
+func (*ReplicationInfo) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{46}
+}
+
+func (m *ReplicationInfo) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ReplicationInfo.Unmarshal(m, b)
+}
+func (m *ReplicationInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ReplicationInfo.Marshal(b, m, deterministic)
+}
+func (m *ReplicationInfo) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ReplicationInfo.Merge(m, src)
+}
+func (m *ReplicationInfo) XXX_Size() int {
+ return xxx_messageInfo_ReplicationInfo.Size(m)
+}
+func (m *ReplicationInfo) XXX_DiscardUnknown() {
+ xxx_messageInfo_ReplicationInfo.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ReplicationInfo proto.InternalMessageInfo
+
+func (m *ReplicationInfo) GetStatus() string {
+ if m != nil {
+ return m.Status
+ }
+ return ""
+}
+
+func (m *ReplicationInfo) GetBackend() string {
+ if m != nil {
+ return m.Backend
+ }
+ return ""
+}
+
+func (m *ReplicationInfo) GetStorageClass() string {
+ if m != nil {
+ return m.StorageClass
+ }
+ return ""
+}
+
+type Acl struct {
+ CannedAcl string `protobuf:"bytes,1,opt,name=cannedAcl,proto3" json:"cannedAcl,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Acl) Reset() { *m = Acl{} }
+func (m *Acl) String() string { return proto.CompactTextString(m) }
+func (*Acl) ProtoMessage() {}
+func (*Acl) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{47}
+}
+
+func (m *Acl) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Acl.Unmarshal(m, b)
+}
+func (m *Acl) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Acl.Marshal(b, m, deterministic)
+}
+func (m *Acl) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Acl.Merge(m, src)
+}
+func (m *Acl) XXX_Size() int {
+ return xxx_messageInfo_Acl.Size(m)
+}
+func (m *Acl) XXX_DiscardUnknown() {
+ xxx_messageInfo_Acl.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Acl proto.InternalMessageInfo
+
+func (m *Acl) GetCannedAcl() string {
+ if m != nil {
+ return m.CannedAcl
+ }
+ return ""
+}
+
+type GetBucketResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ BucketMeta *Bucket `protobuf:"bytes,2,opt,name=bucketMeta,proto3" json:"bucketMeta,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *GetBucketResponse) Reset() { *m = GetBucketResponse{} }
+func (m *GetBucketResponse) String() string { return proto.CompactTextString(m) }
+func (*GetBucketResponse) ProtoMessage() {}
+func (*GetBucketResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{48}
+}
+
+func (m *GetBucketResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_GetBucketResponse.Unmarshal(m, b)
+}
+func (m *GetBucketResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_GetBucketResponse.Marshal(b, m, deterministic)
+}
+func (m *GetBucketResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_GetBucketResponse.Merge(m, src)
+}
+func (m *GetBucketResponse) XXX_Size() int {
+ return xxx_messageInfo_GetBucketResponse.Size(m)
+}
+func (m *GetBucketResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_GetBucketResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetBucketResponse proto.InternalMessageInfo
+
+func (m *GetBucketResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *GetBucketResponse) GetBucketMeta() *Bucket {
+ if m != nil {
+ return m.BucketMeta
+ }
+ return nil
+}
+
+type Bucket struct {
+ Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+ TenantId string `protobuf:"bytes,2,opt,name=tenantId,proto3" json:"tenantId,omitempty"`
+ UserId string `protobuf:"bytes,3,opt,name=userId,proto3" json:"userId,omitempty"`
+ Acl *Acl `protobuf:"bytes,4,opt,name=acl,proto3" json:"acl,omitempty"`
+ CreateTime int64 `protobuf:"varint,5,opt,name=createTime,proto3" json:"createTime,omitempty"`
+ Id string `protobuf:"bytes,6,opt,name=id,proto3" json:"id,omitempty"`
+ Deleted bool `protobuf:"varint,7,opt,name=deleted,proto3" json:"deleted,omitempty"`
+ ServerSideEncryption *ServerSideEncryption `protobuf:"bytes,8,opt,name=serverSideEncryption,proto3" json:"serverSideEncryption,omitempty"`
+ Versioning *BucketVersioning `protobuf:"bytes,9,opt,name=versioning,proto3" json:"versioning,omitempty"`
+ DefaultLocation string `protobuf:"bytes,10,opt,name=defaultLocation,proto3" json:"defaultLocation,omitempty"`
+ WebsiteConfiguration *WebsiteConfiguration `protobuf:"bytes,11,opt,name=websiteConfiguration,proto3" json:"websiteConfiguration,omitempty"`
+ Cors *CORSConfiguration `protobuf:"bytes,12,opt,name=cors,proto3" json:"cors,omitempty"`
+ ReplicationConfiguration *ReplicationConfiguration `protobuf:"bytes,13,opt,name=replicationConfiguration,proto3" json:"replicationConfiguration,omitempty"`
+ LifecycleConfiguration []*LifecycleRule `protobuf:"bytes,14,rep,name=lifecycleConfiguration,proto3" json:"lifecycleConfiguration,omitempty"`
+ BucketPolicy string `protobuf:"bytes,15,opt,name=bucketPolicy,proto3" json:"bucketPolicy,omitempty"`
+ Usages int64 `protobuf:"varint,16,opt,name=usages,proto3" json:"usages,omitempty"`
+ Tier int32 `protobuf:"varint,17,opt,name=tier,proto3" json:"tier,omitempty"`
+ ReplicationInfo []*ReplicationInfo `protobuf:"bytes,18,rep,name=replicationInfo,proto3" json:"replicationInfo,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Bucket) Reset() { *m = Bucket{} }
+func (m *Bucket) String() string { return proto.CompactTextString(m) }
+func (*Bucket) ProtoMessage() {}
+func (*Bucket) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{49}
+}
+
+func (m *Bucket) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Bucket.Unmarshal(m, b)
+}
+func (m *Bucket) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Bucket.Marshal(b, m, deterministic)
+}
+func (m *Bucket) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Bucket.Merge(m, src)
+}
+func (m *Bucket) XXX_Size() int {
+ return xxx_messageInfo_Bucket.Size(m)
+}
+func (m *Bucket) XXX_DiscardUnknown() {
+ xxx_messageInfo_Bucket.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Bucket proto.InternalMessageInfo
+
+func (m *Bucket) GetName() string {
+ if m != nil {
+ return m.Name
+ }
+ return ""
+}
+
+func (m *Bucket) GetTenantId() string {
+ if m != nil {
+ return m.TenantId
+ }
+ return ""
+}
+
+func (m *Bucket) GetUserId() string {
+ if m != nil {
+ return m.UserId
+ }
+ return ""
+}
+
+func (m *Bucket) GetAcl() *Acl {
+ if m != nil {
+ return m.Acl
+ }
+ return nil
+}
+
+func (m *Bucket) GetCreateTime() int64 {
+ if m != nil {
+ return m.CreateTime
+ }
+ return 0
+}
+
+func (m *Bucket) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *Bucket) GetDeleted() bool {
+ if m != nil {
+ return m.Deleted
+ }
+ return false
+}
+
+func (m *Bucket) GetServerSideEncryption() *ServerSideEncryption {
+ if m != nil {
+ return m.ServerSideEncryption
+ }
+ return nil
+}
+
+func (m *Bucket) GetVersioning() *BucketVersioning {
+ if m != nil {
+ return m.Versioning
+ }
+ return nil
+}
+
+func (m *Bucket) GetDefaultLocation() string {
+ if m != nil {
+ return m.DefaultLocation
+ }
+ return ""
+}
+
+func (m *Bucket) GetWebsiteConfiguration() *WebsiteConfiguration {
+ if m != nil {
+ return m.WebsiteConfiguration
+ }
+ return nil
+}
+
+func (m *Bucket) GetCors() *CORSConfiguration {
+ if m != nil {
+ return m.Cors
+ }
+ return nil
+}
+
+func (m *Bucket) GetReplicationConfiguration() *ReplicationConfiguration {
+ if m != nil {
+ return m.ReplicationConfiguration
+ }
+ return nil
+}
+
+func (m *Bucket) GetLifecycleConfiguration() []*LifecycleRule {
+ if m != nil {
+ return m.LifecycleConfiguration
+ }
+ return nil
+}
+
+func (m *Bucket) GetBucketPolicy() string {
+ if m != nil {
+ return m.BucketPolicy
+ }
+ return ""
+}
+
+func (m *Bucket) GetUsages() int64 {
+ if m != nil {
+ return m.Usages
+ }
+ return 0
+}
+
+func (m *Bucket) GetTier() int32 {
+ if m != nil {
+ return m.Tier
+ }
+ return 0
+}
+
+func (m *Bucket) GetReplicationInfo() []*ReplicationInfo {
+ if m != nil {
+ return m.ReplicationInfo
+ }
+ return nil
+}
+
+type Partion struct {
+ Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+ PartNumber int64 `protobuf:"varint,2,opt,name=partNumber,proto3" json:"partNumber,omitempty"`
+ Size int64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
+ Etag string `protobuf:"bytes,4,opt,name=etag,proto3" json:"etag,omitempty"`
+ LastModified int64 `protobuf:"varint,5,opt,name=lastModified,proto3" json:"lastModified,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Partion) Reset() { *m = Partion{} }
+func (m *Partion) String() string { return proto.CompactTextString(m) }
+func (*Partion) ProtoMessage() {}
+func (*Partion) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{50}
+}
+
+func (m *Partion) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Partion.Unmarshal(m, b)
+}
+func (m *Partion) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Partion.Marshal(b, m, deterministic)
+}
+func (m *Partion) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Partion.Merge(m, src)
+}
+func (m *Partion) XXX_Size() int {
+ return xxx_messageInfo_Partion.Size(m)
+}
+func (m *Partion) XXX_DiscardUnknown() {
+ xxx_messageInfo_Partion.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Partion proto.InternalMessageInfo
+
+func (m *Partion) GetKey() string {
+ if m != nil {
+ return m.Key
+ }
+ return ""
+}
+
+func (m *Partion) GetPartNumber() int64 {
+ if m != nil {
+ return m.PartNumber
+ }
+ return 0
+}
+
+func (m *Partion) GetSize() int64 {
+ if m != nil {
+ return m.Size
+ }
+ return 0
+}
+
+func (m *Partion) GetEtag() string {
+ if m != nil {
+ return m.Etag
+ }
+ return ""
+}
+
+func (m *Partion) GetLastModified() int64 {
+ if m != nil {
+ return m.LastModified
+ }
+ return 0
+}
+
+type Version struct {
+ Versionid string `protobuf:"bytes,1,opt,name=versionid,proto3" json:"versionid,omitempty"`
+ Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"`
+ IsLatest string `protobuf:"bytes,3,opt,name=isLatest,proto3" json:"isLatest,omitempty"`
+ LastModified int64 `protobuf:"varint,4,opt,name=lastModified,proto3" json:"lastModified,omitempty"`
+ Size int64 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Version) Reset() { *m = Version{} }
+func (m *Version) String() string { return proto.CompactTextString(m) }
+func (*Version) ProtoMessage() {}
+func (*Version) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{51}
+}
+
+func (m *Version) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Version.Unmarshal(m, b)
+}
+func (m *Version) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Version.Marshal(b, m, deterministic)
+}
+func (m *Version) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Version.Merge(m, src)
+}
+func (m *Version) XXX_Size() int {
+ return xxx_messageInfo_Version.Size(m)
+}
+func (m *Version) XXX_DiscardUnknown() {
+ xxx_messageInfo_Version.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Version proto.InternalMessageInfo
+
+func (m *Version) GetVersionid() string {
+ if m != nil {
+ return m.Versionid
+ }
+ return ""
+}
+
+func (m *Version) GetEtag() string {
+ if m != nil {
+ return m.Etag
+ }
+ return ""
+}
+
+func (m *Version) GetIsLatest() string {
+ if m != nil {
+ return m.IsLatest
+ }
+ return ""
+}
+
+func (m *Version) GetLastModified() int64 {
+ if m != nil {
+ return m.LastModified
+ }
+ return 0
+}
+
+func (m *Version) GetSize() int64 {
+ if m != nil {
+ return m.Size
+ }
+ return 0
+}
+
+type Object struct {
+ ObjectKey string `protobuf:"bytes,1,opt,name=objectKey,proto3" json:"objectKey,omitempty"`
+ BucketName string `protobuf:"bytes,2,opt,name=bucketName,proto3" json:"bucketName,omitempty"`
+ TenantId string `protobuf:"bytes,3,opt,name=tenantId,proto3" json:"tenantId,omitempty"`
+ UserId string `protobuf:"bytes,4,opt,name=userId,proto3" json:"userId,omitempty"`
+ VersionId string `protobuf:"bytes,5,opt,name=versionId,proto3" json:"versionId,omitempty"`
+ ObjectId string `protobuf:"bytes,6,opt,name=objectId,proto3" json:"objectId,omitempty"`
+ ContentType string `protobuf:"bytes,7,opt,name=contentType,proto3" json:"contentType,omitempty"`
+ ContentMd5 string `protobuf:"bytes,8,opt,name=contentMd5,proto3" json:"contentMd5,omitempty"`
+ Versions []*Version `protobuf:"bytes,9,rep,name=versions,proto3" json:"versions,omitempty"`
+ StorageClass string `protobuf:"bytes,10,opt,name=storageClass,proto3" json:"storageClass,omitempty"`
+ ServerSideEncryption *ServerSideEncryption `protobuf:"bytes,11,opt,name=serverSideEncryption,proto3" json:"serverSideEncryption,omitempty"`
+ WebsiteRedirectLocation string `protobuf:"bytes,12,opt,name=websiteRedirectLocation,proto3" json:"websiteRedirectLocation,omitempty"`
+ Acl *Acl `protobuf:"bytes,13,opt,name=acl,proto3" json:"acl,omitempty"`
+ Location string `protobuf:"bytes,14,opt,name=location,proto3" json:"location,omitempty"`
+ IsNull bool `protobuf:"varint,15,opt,name=isNull,proto3" json:"isNull,omitempty"`
+ DeleteMarker bool `protobuf:"varint,16,opt,name=deleteMarker,proto3" json:"deleteMarker,omitempty"`
+ Size int64 `protobuf:"varint,17,opt,name=size,proto3" json:"size,omitempty"`
+ Tags []*Tag `protobuf:"bytes,18,rep,name=tags,proto3" json:"tags,omitempty"`
+ LastModified int64 `protobuf:"varint,19,opt,name=lastModified,proto3" json:"lastModified,omitempty"`
+ Etag string `protobuf:"bytes,20,opt,name=etag,proto3" json:"etag,omitempty"`
+ Tier int32 `protobuf:"varint,21,opt,name=tier,proto3" json:"tier,omitempty"`
+ Type int32 `protobuf:"varint,22,opt,name=type,proto3" json:"type,omitempty"`
+ NullVersion bool `protobuf:"varint,23,opt,name=nullVersion,proto3" json:"nullVersion,omitempty"`
+ CustomAttributes map[string]string `protobuf:"bytes,24,rep,name=customAttributes,proto3" json:"customAttributes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+ StorageMeta string `protobuf:"bytes,25,opt,name=storageMeta,proto3" json:"storageMeta,omitempty"`
+ EncSize int64 `protobuf:"varint,26,opt,name=encSize,proto3" json:"encSize,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Object) Reset() { *m = Object{} }
+func (m *Object) String() string { return proto.CompactTextString(m) }
+func (*Object) ProtoMessage() {}
+func (*Object) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{52}
+}
+
+func (m *Object) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Object.Unmarshal(m, b)
+}
+func (m *Object) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Object.Marshal(b, m, deterministic)
+}
+func (m *Object) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Object.Merge(m, src)
+}
+func (m *Object) XXX_Size() int {
+ return xxx_messageInfo_Object.Size(m)
+}
+func (m *Object) XXX_DiscardUnknown() {
+ xxx_messageInfo_Object.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Object proto.InternalMessageInfo
+
+func (m *Object) GetObjectKey() string {
+ if m != nil {
+ return m.ObjectKey
+ }
+ return ""
+}
+
+func (m *Object) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *Object) GetTenantId() string {
+ if m != nil {
+ return m.TenantId
+ }
+ return ""
+}
+
+func (m *Object) GetUserId() string {
+ if m != nil {
+ return m.UserId
+ }
+ return ""
+}
+
+func (m *Object) GetVersionId() string {
+ if m != nil {
+ return m.VersionId
+ }
+ return ""
+}
+
+func (m *Object) GetObjectId() string {
+ if m != nil {
+ return m.ObjectId
+ }
+ return ""
+}
+
+func (m *Object) GetContentType() string {
+ if m != nil {
+ return m.ContentType
+ }
+ return ""
+}
+
+func (m *Object) GetContentMd5() string {
+ if m != nil {
+ return m.ContentMd5
+ }
+ return ""
+}
+
+func (m *Object) GetVersions() []*Version {
+ if m != nil {
+ return m.Versions
+ }
+ return nil
+}
+
+func (m *Object) GetStorageClass() string {
+ if m != nil {
+ return m.StorageClass
+ }
+ return ""
+}
+
+func (m *Object) GetServerSideEncryption() *ServerSideEncryption {
+ if m != nil {
+ return m.ServerSideEncryption
+ }
+ return nil
+}
+
+func (m *Object) GetWebsiteRedirectLocation() string {
+ if m != nil {
+ return m.WebsiteRedirectLocation
+ }
+ return ""
+}
+
+func (m *Object) GetAcl() *Acl {
+ if m != nil {
+ return m.Acl
+ }
+ return nil
+}
+
+func (m *Object) GetLocation() string {
+ if m != nil {
+ return m.Location
+ }
+ return ""
+}
+
+func (m *Object) GetIsNull() bool {
+ if m != nil {
+ return m.IsNull
+ }
+ return false
+}
+
+func (m *Object) GetDeleteMarker() bool {
+ if m != nil {
+ return m.DeleteMarker
+ }
+ return false
+}
+
+func (m *Object) GetSize() int64 {
+ if m != nil {
+ return m.Size
+ }
+ return 0
+}
+
+func (m *Object) GetTags() []*Tag {
+ if m != nil {
+ return m.Tags
+ }
+ return nil
+}
+
+func (m *Object) GetLastModified() int64 {
+ if m != nil {
+ return m.LastModified
+ }
+ return 0
+}
+
+func (m *Object) GetEtag() string {
+ if m != nil {
+ return m.Etag
+ }
+ return ""
+}
+
+func (m *Object) GetTier() int32 {
+ if m != nil {
+ return m.Tier
+ }
+ return 0
+}
+
+func (m *Object) GetType() int32 {
+ if m != nil {
+ return m.Type
+ }
+ return 0
+}
+
+func (m *Object) GetNullVersion() bool {
+ if m != nil {
+ return m.NullVersion
+ }
+ return false
+}
+
+func (m *Object) GetCustomAttributes() map[string]string {
+ if m != nil {
+ return m.CustomAttributes
+ }
+ return nil
+}
+
+func (m *Object) GetStorageMeta() string {
+ if m != nil {
+ return m.StorageMeta
+ }
+ return ""
+}
+
+func (m *Object) GetEncSize() int64 {
+ if m != nil {
+ return m.EncSize
+ }
+ return 0
+}
+
+type ListBucketsResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ Buckets []*Bucket `protobuf:"bytes,2,rep,name=buckets,proto3" json:"buckets,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ListBucketsResponse) Reset() { *m = ListBucketsResponse{} }
+func (m *ListBucketsResponse) String() string { return proto.CompactTextString(m) }
+func (*ListBucketsResponse) ProtoMessage() {}
+func (*ListBucketsResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{53}
+}
+
+func (m *ListBucketsResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ListBucketsResponse.Unmarshal(m, b)
+}
+func (m *ListBucketsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ListBucketsResponse.Marshal(b, m, deterministic)
+}
+func (m *ListBucketsResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ListBucketsResponse.Merge(m, src)
+}
+func (m *ListBucketsResponse) XXX_Size() int {
+ return xxx_messageInfo_ListBucketsResponse.Size(m)
+}
+func (m *ListBucketsResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_ListBucketsResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListBucketsResponse proto.InternalMessageInfo
+
+func (m *ListBucketsResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *ListBucketsResponse) GetBuckets() []*Bucket {
+ if m != nil {
+ return m.Buckets
+ }
+ return nil
+}
+
+type BaseResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ Msg string `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *BaseResponse) Reset() { *m = BaseResponse{} }
+func (m *BaseResponse) String() string { return proto.CompactTextString(m) }
+func (*BaseResponse) ProtoMessage() {}
+func (*BaseResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{54}
+}
+
+func (m *BaseResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_BaseResponse.Unmarshal(m, b)
+}
+func (m *BaseResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_BaseResponse.Marshal(b, m, deterministic)
+}
+func (m *BaseResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_BaseResponse.Merge(m, src)
+}
+func (m *BaseResponse) XXX_Size() int {
+ return xxx_messageInfo_BaseResponse.Size(m)
+}
+func (m *BaseResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_BaseResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_BaseResponse proto.InternalMessageInfo
+
+func (m *BaseResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *BaseResponse) GetMsg() string {
+ if m != nil {
+ return m.Msg
+ }
+ return ""
+}
+
+type BaseRequest struct {
+ Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *BaseRequest) Reset() { *m = BaseRequest{} }
+func (m *BaseRequest) String() string { return proto.CompactTextString(m) }
+func (*BaseRequest) ProtoMessage() {}
+func (*BaseRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{55}
+}
+
+func (m *BaseRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_BaseRequest.Unmarshal(m, b)
+}
+func (m *BaseRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_BaseRequest.Marshal(b, m, deterministic)
+}
+func (m *BaseRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_BaseRequest.Merge(m, src)
+}
+func (m *BaseRequest) XXX_Size() int {
+ return xxx_messageInfo_BaseRequest.Size(m)
+}
+func (m *BaseRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_BaseRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_BaseRequest proto.InternalMessageInfo
+
+func (m *BaseRequest) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+type ListObjectsRequest struct {
+ Bucket string `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"`
+ Versioned bool `protobuf:"varint,2,opt,name=versioned,proto3" json:"versioned,omitempty"`
+ Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"`
+ Delimiter string `protobuf:"bytes,4,opt,name=delimiter,proto3" json:"delimiter,omitempty"`
+ EncodingType string `protobuf:"bytes,5,opt,name=encodingType,proto3" json:"encodingType,omitempty"`
+ MaxKeys int32 `protobuf:"varint,6,opt,name=maxKeys,proto3" json:"maxKeys,omitempty"`
+ Prefix string `protobuf:"bytes,7,opt,name=prefix,proto3" json:"prefix,omitempty"`
+ // v1 specific
+ Marker string `protobuf:"bytes,8,opt,name=marker,proto3" json:"marker,omitempty"`
+ // v2 specific
+ ContinuationToken string `protobuf:"bytes,9,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"`
+ StartAfter string `protobuf:"bytes,10,opt,name=startAfter,proto3" json:"startAfter,omitempty"`
+ FetchOwner bool `protobuf:"varint,11,opt,name=fetchOwner,proto3" json:"fetchOwner,omitempty"`
+ // versioned specific
+ KeyMarker string `protobuf:"bytes,12,opt,name=keyMarker,proto3" json:"keyMarker,omitempty"`
+ VersionIdMarker string `protobuf:"bytes,13,opt,name=versionIdMarker,proto3" json:"versionIdMarker,omitempty"`
+ // for internal use, for example, lifecycle management may need to filter objects by tier, or create time.
+ Filter map[string]string `protobuf:"bytes,14,rep,name=filter,proto3" json:"filter,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ListObjectsRequest) Reset() { *m = ListObjectsRequest{} }
+func (m *ListObjectsRequest) String() string { return proto.CompactTextString(m) }
+func (*ListObjectsRequest) ProtoMessage() {}
+func (*ListObjectsRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{56}
+}
+
+func (m *ListObjectsRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ListObjectsRequest.Unmarshal(m, b)
+}
+func (m *ListObjectsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ListObjectsRequest.Marshal(b, m, deterministic)
+}
+func (m *ListObjectsRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ListObjectsRequest.Merge(m, src)
+}
+func (m *ListObjectsRequest) XXX_Size() int {
+ return xxx_messageInfo_ListObjectsRequest.Size(m)
+}
+func (m *ListObjectsRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_ListObjectsRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListObjectsRequest proto.InternalMessageInfo
+
+func (m *ListObjectsRequest) GetBucket() string {
+ if m != nil {
+ return m.Bucket
+ }
+ return ""
+}
+
+func (m *ListObjectsRequest) GetVersioned() bool {
+ if m != nil {
+ return m.Versioned
+ }
+ return false
+}
+
+func (m *ListObjectsRequest) GetVersion() int32 {
+ if m != nil {
+ return m.Version
+ }
+ return 0
+}
+
+func (m *ListObjectsRequest) GetDelimiter() string {
+ if m != nil {
+ return m.Delimiter
+ }
+ return ""
+}
+
+func (m *ListObjectsRequest) GetEncodingType() string {
+ if m != nil {
+ return m.EncodingType
+ }
+ return ""
+}
+
+func (m *ListObjectsRequest) GetMaxKeys() int32 {
+ if m != nil {
+ return m.MaxKeys
+ }
+ return 0
+}
+
+func (m *ListObjectsRequest) GetPrefix() string {
+ if m != nil {
+ return m.Prefix
+ }
+ return ""
+}
+
+func (m *ListObjectsRequest) GetMarker() string {
+ if m != nil {
+ return m.Marker
+ }
+ return ""
+}
+
+func (m *ListObjectsRequest) GetContinuationToken() string {
+ if m != nil {
+ return m.ContinuationToken
+ }
+ return ""
+}
+
+func (m *ListObjectsRequest) GetStartAfter() string {
+ if m != nil {
+ return m.StartAfter
+ }
+ return ""
+}
+
+func (m *ListObjectsRequest) GetFetchOwner() bool {
+ if m != nil {
+ return m.FetchOwner
+ }
+ return false
+}
+
+func (m *ListObjectsRequest) GetKeyMarker() string {
+ if m != nil {
+ return m.KeyMarker
+ }
+ return ""
+}
+
+func (m *ListObjectsRequest) GetVersionIdMarker() string {
+ if m != nil {
+ return m.VersionIdMarker
+ }
+ return ""
+}
+
+func (m *ListObjectsRequest) GetFilter() map[string]string {
+ if m != nil {
+ return m.Filter
+ }
+ return nil
+}
+
+type ListObjectsResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ // Indicates whether the returned list objects response is truncated. A
+ // value of true indicates that the list was truncated. The list can be truncated
+ // if the number of objects exceeds the limit allowed or specified
+ // by max keys.
+ IsTruncated bool `protobuf:"varint,2,opt,name=isTruncated,proto3" json:"isTruncated,omitempty"`
+ // When response is truncated (the IsTruncated element value in the response
+ // is true), you can use the key name in this field as marker in the subsequent
+ // request to get next set of objects.
+ //
+ // NOTE: This element is returned only if you have delimiter request parameter
+ // specified.
+ NextMarker string `protobuf:"bytes,3,opt,name=nextMarker,proto3" json:"nextMarker,omitempty"`
+ // List of prefixes for this request.
+ Prefixes []string `protobuf:"bytes,4,rep,name=Prefixes,proto3" json:"Prefixes,omitempty"`
+ // List of objects info for this request.
+ Objects []*Object `protobuf:"bytes,5,rep,name=objects,proto3" json:"objects,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ListObjectsResponse) Reset() { *m = ListObjectsResponse{} }
+func (m *ListObjectsResponse) String() string { return proto.CompactTextString(m) }
+func (*ListObjectsResponse) ProtoMessage() {}
+func (*ListObjectsResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{57}
+}
+
+func (m *ListObjectsResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ListObjectsResponse.Unmarshal(m, b)
+}
+func (m *ListObjectsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ListObjectsResponse.Marshal(b, m, deterministic)
+}
+func (m *ListObjectsResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ListObjectsResponse.Merge(m, src)
+}
+func (m *ListObjectsResponse) XXX_Size() int {
+ return xxx_messageInfo_ListObjectsResponse.Size(m)
+}
+func (m *ListObjectsResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_ListObjectsResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListObjectsResponse proto.InternalMessageInfo
+
+func (m *ListObjectsResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *ListObjectsResponse) GetIsTruncated() bool {
+ if m != nil {
+ return m.IsTruncated
+ }
+ return false
+}
+
+func (m *ListObjectsResponse) GetNextMarker() string {
+ if m != nil {
+ return m.NextMarker
+ }
+ return ""
+}
+
+func (m *ListObjectsResponse) GetPrefixes() []string {
+ if m != nil {
+ return m.Prefixes
+ }
+ return nil
+}
+
+func (m *ListObjectsResponse) GetObjects() []*Object {
+ if m != nil {
+ return m.Objects
+ }
+ return nil
+}
+
+type CountObjectsResponse struct {
+ Count int64 `protobuf:"varint,1,opt,name=Count,proto3" json:"Count,omitempty"`
+ Size int64 `protobuf:"varint,2,opt,name=Size,proto3" json:"Size,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *CountObjectsResponse) Reset() { *m = CountObjectsResponse{} }
+func (m *CountObjectsResponse) String() string { return proto.CompactTextString(m) }
+func (*CountObjectsResponse) ProtoMessage() {}
+func (*CountObjectsResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{58}
+}
+
+func (m *CountObjectsResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_CountObjectsResponse.Unmarshal(m, b)
+}
+func (m *CountObjectsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_CountObjectsResponse.Marshal(b, m, deterministic)
+}
+func (m *CountObjectsResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_CountObjectsResponse.Merge(m, src)
+}
+func (m *CountObjectsResponse) XXX_Size() int {
+ return xxx_messageInfo_CountObjectsResponse.Size(m)
+}
+func (m *CountObjectsResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_CountObjectsResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CountObjectsResponse proto.InternalMessageInfo
+
+func (m *CountObjectsResponse) GetCount() int64 {
+ if m != nil {
+ return m.Count
+ }
+ return 0
+}
+
+func (m *CountObjectsResponse) GetSize() int64 {
+ if m != nil {
+ return m.Size
+ }
+ return 0
+}
+
+type DeleteObjectInput struct {
+ Bucket string `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"`
+ Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
+ VersioId string `protobuf:"bytes,3,opt,name=versioId,proto3" json:"versioId,omitempty"`
+ StorageMeta string `protobuf:"bytes,4,opt,name=storageMeta,proto3" json:"storageMeta,omitempty"`
+ ETag string `protobuf:"bytes,5,opt,name=eTag,proto3" json:"eTag,omitempty"`
+ ObjectId string `protobuf:"bytes,6,opt,name=objectId,proto3" json:"objectId,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *DeleteObjectInput) Reset() { *m = DeleteObjectInput{} }
+func (m *DeleteObjectInput) String() string { return proto.CompactTextString(m) }
+func (*DeleteObjectInput) ProtoMessage() {}
+func (*DeleteObjectInput) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{59}
+}
+
+func (m *DeleteObjectInput) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_DeleteObjectInput.Unmarshal(m, b)
+}
+func (m *DeleteObjectInput) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_DeleteObjectInput.Marshal(b, m, deterministic)
+}
+func (m *DeleteObjectInput) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_DeleteObjectInput.Merge(m, src)
+}
+func (m *DeleteObjectInput) XXX_Size() int {
+ return xxx_messageInfo_DeleteObjectInput.Size(m)
+}
+func (m *DeleteObjectInput) XXX_DiscardUnknown() {
+ xxx_messageInfo_DeleteObjectInput.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DeleteObjectInput proto.InternalMessageInfo
+
+func (m *DeleteObjectInput) GetBucket() string {
+ if m != nil {
+ return m.Bucket
+ }
+ return ""
+}
+
+func (m *DeleteObjectInput) GetKey() string {
+ if m != nil {
+ return m.Key
+ }
+ return ""
+}
+
+func (m *DeleteObjectInput) GetVersioId() string {
+ if m != nil {
+ return m.VersioId
+ }
+ return ""
+}
+
+func (m *DeleteObjectInput) GetStorageMeta() string {
+ if m != nil {
+ return m.StorageMeta
+ }
+ return ""
+}
+
+func (m *DeleteObjectInput) GetETag() string {
+ if m != nil {
+ return m.ETag
+ }
+ return ""
+}
+
+func (m *DeleteObjectInput) GetObjectId() string {
+ if m != nil {
+ return m.ObjectId
+ }
+ return ""
+}
+
+type DeleteObjectOutput struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ DeleteMarker bool `protobuf:"varint,2,opt,name=deleteMarker,proto3" json:"deleteMarker,omitempty"`
+ VersionId string `protobuf:"bytes,3,opt,name=versionId,proto3" json:"versionId,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *DeleteObjectOutput) Reset() { *m = DeleteObjectOutput{} }
+func (m *DeleteObjectOutput) String() string { return proto.CompactTextString(m) }
+func (*DeleteObjectOutput) ProtoMessage() {}
+func (*DeleteObjectOutput) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{60}
+}
+
+func (m *DeleteObjectOutput) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_DeleteObjectOutput.Unmarshal(m, b)
+}
+func (m *DeleteObjectOutput) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_DeleteObjectOutput.Marshal(b, m, deterministic)
+}
+func (m *DeleteObjectOutput) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_DeleteObjectOutput.Merge(m, src)
+}
+func (m *DeleteObjectOutput) XXX_Size() int {
+ return xxx_messageInfo_DeleteObjectOutput.Size(m)
+}
+func (m *DeleteObjectOutput) XXX_DiscardUnknown() {
+ xxx_messageInfo_DeleteObjectOutput.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DeleteObjectOutput proto.InternalMessageInfo
+
+func (m *DeleteObjectOutput) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *DeleteObjectOutput) GetDeleteMarker() bool {
+ if m != nil {
+ return m.DeleteMarker
+ }
+ return false
+}
+
+func (m *DeleteObjectOutput) GetVersionId() string {
+ if m != nil {
+ return m.VersionId
+ }
+ return ""
+}
+
+type GetObjectInput struct {
+ Bucket string `protobuf:"bytes,1,opt,name=Bucket,proto3" json:"Bucket,omitempty"`
+ Key string `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"`
+ VersionId string `protobuf:"bytes,3,opt,name=VersionId,proto3" json:"VersionId,omitempty"`
+ Offset int64 `protobuf:"varint,4,opt,name=Offset,proto3" json:"Offset,omitempty"`
+ Length int64 `protobuf:"varint,5,opt,name=Length,proto3" json:"Length,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *GetObjectInput) Reset() { *m = GetObjectInput{} }
+func (m *GetObjectInput) String() string { return proto.CompactTextString(m) }
+func (*GetObjectInput) ProtoMessage() {}
+func (*GetObjectInput) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{61}
+}
+
+func (m *GetObjectInput) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_GetObjectInput.Unmarshal(m, b)
+}
+func (m *GetObjectInput) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_GetObjectInput.Marshal(b, m, deterministic)
+}
+func (m *GetObjectInput) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_GetObjectInput.Merge(m, src)
+}
+func (m *GetObjectInput) XXX_Size() int {
+ return xxx_messageInfo_GetObjectInput.Size(m)
+}
+func (m *GetObjectInput) XXX_DiscardUnknown() {
+ xxx_messageInfo_GetObjectInput.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetObjectInput proto.InternalMessageInfo
+
+func (m *GetObjectInput) GetBucket() string {
+ if m != nil {
+ return m.Bucket
+ }
+ return ""
+}
+
+func (m *GetObjectInput) GetKey() string {
+ if m != nil {
+ return m.Key
+ }
+ return ""
+}
+
+func (m *GetObjectInput) GetVersionId() string {
+ if m != nil {
+ return m.VersionId
+ }
+ return ""
+}
+
+func (m *GetObjectInput) GetOffset() int64 {
+ if m != nil {
+ return m.Offset
+ }
+ return 0
+}
+
+func (m *GetObjectInput) GetLength() int64 {
+ if m != nil {
+ return m.Length
+ }
+ return 0
+}
+
+type MultipartUpload struct {
+ Bucket string `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"`
+ Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
+ UploadId string `protobuf:"bytes,3,opt,name=uploadId,proto3" json:"uploadId,omitempty"`
+ ObjectId string `protobuf:"bytes,4,opt,name=objectId,proto3" json:"objectId,omitempty"`
+ Location string `protobuf:"bytes,5,opt,name=location,proto3" json:"location,omitempty"`
+ Tier int32 `protobuf:"varint,6,opt,name=tier,proto3" json:"tier,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *MultipartUpload) Reset() { *m = MultipartUpload{} }
+func (m *MultipartUpload) String() string { return proto.CompactTextString(m) }
+func (*MultipartUpload) ProtoMessage() {}
+func (*MultipartUpload) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{62}
+}
+
+func (m *MultipartUpload) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_MultipartUpload.Unmarshal(m, b)
+}
+func (m *MultipartUpload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_MultipartUpload.Marshal(b, m, deterministic)
+}
+func (m *MultipartUpload) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_MultipartUpload.Merge(m, src)
+}
+func (m *MultipartUpload) XXX_Size() int {
+ return xxx_messageInfo_MultipartUpload.Size(m)
+}
+func (m *MultipartUpload) XXX_DiscardUnknown() {
+ xxx_messageInfo_MultipartUpload.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MultipartUpload proto.InternalMessageInfo
+
+func (m *MultipartUpload) GetBucket() string {
+ if m != nil {
+ return m.Bucket
+ }
+ return ""
+}
+
+func (m *MultipartUpload) GetKey() string {
+ if m != nil {
+ return m.Key
+ }
+ return ""
+}
+
+func (m *MultipartUpload) GetUploadId() string {
+ if m != nil {
+ return m.UploadId
+ }
+ return ""
+}
+
+func (m *MultipartUpload) GetObjectId() string {
+ if m != nil {
+ return m.ObjectId
+ }
+ return ""
+}
+
+func (m *MultipartUpload) GetLocation() string {
+ if m != nil {
+ return m.Location
+ }
+ return ""
+}
+
+func (m *MultipartUpload) GetTier() int32 {
+ if m != nil {
+ return m.Tier
+ }
+ return 0
+}
+
+type ListParts struct {
+ Bucket string `protobuf:"bytes,1,opt,name=Bucket,proto3" json:"Bucket,omitempty"`
+ Key string `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"`
+ UploadId string `protobuf:"bytes,3,opt,name=uploadId,proto3" json:"uploadId,omitempty"`
+ MaxParts int64 `protobuf:"varint,4,opt,name=MaxParts,proto3" json:"MaxParts,omitempty"`
+ PartNumberMarker int64 `protobuf:"varint,5,opt,name=PartNumberMarker,proto3" json:"PartNumberMarker,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ListParts) Reset() { *m = ListParts{} }
+func (m *ListParts) String() string { return proto.CompactTextString(m) }
+func (*ListParts) ProtoMessage() {}
+func (*ListParts) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{63}
+}
+
+func (m *ListParts) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ListParts.Unmarshal(m, b)
+}
+func (m *ListParts) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ListParts.Marshal(b, m, deterministic)
+}
+func (m *ListParts) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ListParts.Merge(m, src)
+}
+func (m *ListParts) XXX_Size() int {
+ return xxx_messageInfo_ListParts.Size(m)
+}
+func (m *ListParts) XXX_DiscardUnknown() {
+ xxx_messageInfo_ListParts.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListParts proto.InternalMessageInfo
+
+func (m *ListParts) GetBucket() string {
+ if m != nil {
+ return m.Bucket
+ }
+ return ""
+}
+
+func (m *ListParts) GetKey() string {
+ if m != nil {
+ return m.Key
+ }
+ return ""
+}
+
+func (m *ListParts) GetUploadId() string {
+ if m != nil {
+ return m.UploadId
+ }
+ return ""
+}
+
+func (m *ListParts) GetMaxParts() int64 {
+ if m != nil {
+ return m.MaxParts
+ }
+ return 0
+}
+
+func (m *ListParts) GetPartNumberMarker() int64 {
+ if m != nil {
+ return m.PartNumberMarker
+ }
+ return 0
+}
+
+type TList struct {
+ Tier []int32 `protobuf:"varint,1,rep,packed,name=Tier,proto3" json:"Tier,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *TList) Reset() { *m = TList{} }
+func (m *TList) String() string { return proto.CompactTextString(m) }
+func (*TList) ProtoMessage() {}
+func (*TList) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{64}
+}
+
+func (m *TList) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_TList.Unmarshal(m, b)
+}
+func (m *TList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_TList.Marshal(b, m, deterministic)
+}
+func (m *TList) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_TList.Merge(m, src)
+}
+func (m *TList) XXX_Size() int {
+ return xxx_messageInfo_TList.Size(m)
+}
+func (m *TList) XXX_DiscardUnknown() {
+ xxx_messageInfo_TList.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_TList proto.InternalMessageInfo
+
+func (m *TList) GetTier() []int32 {
+ if m != nil {
+ return m.Tier
+ }
+ return nil
+}
+
+type Tier2ClassName struct {
+ Lst map[int32]string `protobuf:"bytes,1,rep,name=lst,proto3" json:"lst,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Tier2ClassName) Reset() { *m = Tier2ClassName{} }
+func (m *Tier2ClassName) String() string { return proto.CompactTextString(m) }
+func (*Tier2ClassName) ProtoMessage() {}
+func (*Tier2ClassName) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{65}
+}
+
+func (m *Tier2ClassName) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Tier2ClassName.Unmarshal(m, b)
+}
+func (m *Tier2ClassName) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Tier2ClassName.Marshal(b, m, deterministic)
+}
+func (m *Tier2ClassName) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Tier2ClassName.Merge(m, src)
+}
+func (m *Tier2ClassName) XXX_Size() int {
+ return xxx_messageInfo_Tier2ClassName.Size(m)
+}
+func (m *Tier2ClassName) XXX_DiscardUnknown() {
+ xxx_messageInfo_Tier2ClassName.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Tier2ClassName proto.InternalMessageInfo
+
+func (m *Tier2ClassName) GetLst() map[int32]string {
+ if m != nil {
+ return m.Lst
+ }
+ return nil
+}
+
+type GetTierMapResponse struct {
+ Transition []string `protobuf:"bytes,1,rep,name=Transition,proto3" json:"Transition,omitempty"`
+ Tier2Name map[string]*Tier2ClassName `protobuf:"bytes,2,rep,name=Tier2Name,proto3" json:"Tier2Name,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *GetTierMapResponse) Reset() { *m = GetTierMapResponse{} }
+func (m *GetTierMapResponse) String() string { return proto.CompactTextString(m) }
+func (*GetTierMapResponse) ProtoMessage() {}
+func (*GetTierMapResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{66}
+}
+
+func (m *GetTierMapResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_GetTierMapResponse.Unmarshal(m, b)
+}
+func (m *GetTierMapResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_GetTierMapResponse.Marshal(b, m, deterministic)
+}
+func (m *GetTierMapResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_GetTierMapResponse.Merge(m, src)
+}
+func (m *GetTierMapResponse) XXX_Size() int {
+ return xxx_messageInfo_GetTierMapResponse.Size(m)
+}
+func (m *GetTierMapResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_GetTierMapResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetTierMapResponse proto.InternalMessageInfo
+
+func (m *GetTierMapResponse) GetTransition() []string {
+ if m != nil {
+ return m.Transition
+ }
+ return nil
+}
+
+func (m *GetTierMapResponse) GetTier2Name() map[string]*Tier2ClassName {
+ if m != nil {
+ return m.Tier2Name
+ }
+ return nil
+}
+
+type UpdateObjMetaRequest struct {
+ ObjKey string `protobuf:"bytes,1,opt,name=ObjKey,proto3" json:"ObjKey,omitempty"`
+ BucketName string `protobuf:"bytes,2,opt,name=BucketName,proto3" json:"BucketName,omitempty"`
+ LastModified int64 `protobuf:"varint,3,opt,name=LastModified,proto3" json:"LastModified,omitempty"`
+ Setting map[string]string `protobuf:"bytes,4,rep,name=Setting,proto3" json:"Setting,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *UpdateObjMetaRequest) Reset() { *m = UpdateObjMetaRequest{} }
+func (m *UpdateObjMetaRequest) String() string { return proto.CompactTextString(m) }
+func (*UpdateObjMetaRequest) ProtoMessage() {}
+func (*UpdateObjMetaRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{67}
+}
+
+func (m *UpdateObjMetaRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_UpdateObjMetaRequest.Unmarshal(m, b)
+}
+func (m *UpdateObjMetaRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_UpdateObjMetaRequest.Marshal(b, m, deterministic)
+}
+func (m *UpdateObjMetaRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_UpdateObjMetaRequest.Merge(m, src)
+}
+func (m *UpdateObjMetaRequest) XXX_Size() int {
+ return xxx_messageInfo_UpdateObjMetaRequest.Size(m)
+}
+func (m *UpdateObjMetaRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_UpdateObjMetaRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_UpdateObjMetaRequest proto.InternalMessageInfo
+
+func (m *UpdateObjMetaRequest) GetObjKey() string {
+ if m != nil {
+ return m.ObjKey
+ }
+ return ""
+}
+
+func (m *UpdateObjMetaRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *UpdateObjMetaRequest) GetLastModified() int64 {
+ if m != nil {
+ return m.LastModified
+ }
+ return 0
+}
+
+func (m *UpdateObjMetaRequest) GetSetting() map[string]string {
+ if m != nil {
+ return m.Setting
+ }
+ return nil
+}
+
+type StorageClass struct {
+ Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
+ Tier int32 `protobuf:"varint,2,opt,name=Tier,proto3" json:"Tier,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *StorageClass) Reset() { *m = StorageClass{} }
+func (m *StorageClass) String() string { return proto.CompactTextString(m) }
+func (*StorageClass) ProtoMessage() {}
+func (*StorageClass) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{68}
+}
+
+func (m *StorageClass) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_StorageClass.Unmarshal(m, b)
+}
+func (m *StorageClass) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_StorageClass.Marshal(b, m, deterministic)
+}
+func (m *StorageClass) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_StorageClass.Merge(m, src)
+}
+func (m *StorageClass) XXX_Size() int {
+ return xxx_messageInfo_StorageClass.Size(m)
+}
+func (m *StorageClass) XXX_DiscardUnknown() {
+ xxx_messageInfo_StorageClass.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_StorageClass proto.InternalMessageInfo
+
+func (m *StorageClass) GetName() string {
+ if m != nil {
+ return m.Name
+ }
+ return ""
+}
+
+func (m *StorageClass) GetTier() int32 {
+ if m != nil {
+ return m.Tier
+ }
+ return 0
+}
+
+type GetStorageClassesResponse struct {
+ Classes []*StorageClass `protobuf:"bytes,1,rep,name=classes,proto3" json:"classes,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *GetStorageClassesResponse) Reset() { *m = GetStorageClassesResponse{} }
+func (m *GetStorageClassesResponse) String() string { return proto.CompactTextString(m) }
+func (*GetStorageClassesResponse) ProtoMessage() {}
+func (*GetStorageClassesResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{69}
+}
+
+func (m *GetStorageClassesResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_GetStorageClassesResponse.Unmarshal(m, b)
+}
+func (m *GetStorageClassesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_GetStorageClassesResponse.Marshal(b, m, deterministic)
+}
+func (m *GetStorageClassesResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_GetStorageClassesResponse.Merge(m, src)
+}
+func (m *GetStorageClassesResponse) XXX_Size() int {
+ return xxx_messageInfo_GetStorageClassesResponse.Size(m)
+}
+func (m *GetStorageClassesResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_GetStorageClassesResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetStorageClassesResponse proto.InternalMessageInfo
+
+func (m *GetStorageClassesResponse) GetClasses() []*StorageClass {
+ if m != nil {
+ return m.Classes
+ }
+ return nil
+}
+
+type GetBackendTypeByTierRequest struct {
+ Tier int32 `protobuf:"varint,1,opt,name=Tier,proto3" json:"Tier,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *GetBackendTypeByTierRequest) Reset() { *m = GetBackendTypeByTierRequest{} }
+func (m *GetBackendTypeByTierRequest) String() string { return proto.CompactTextString(m) }
+func (*GetBackendTypeByTierRequest) ProtoMessage() {}
+func (*GetBackendTypeByTierRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{70}
+}
+
+func (m *GetBackendTypeByTierRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_GetBackendTypeByTierRequest.Unmarshal(m, b)
+}
+func (m *GetBackendTypeByTierRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_GetBackendTypeByTierRequest.Marshal(b, m, deterministic)
+}
+func (m *GetBackendTypeByTierRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_GetBackendTypeByTierRequest.Merge(m, src)
+}
+func (m *GetBackendTypeByTierRequest) XXX_Size() int {
+ return xxx_messageInfo_GetBackendTypeByTierRequest.Size(m)
+}
+func (m *GetBackendTypeByTierRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_GetBackendTypeByTierRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetBackendTypeByTierRequest proto.InternalMessageInfo
+
+func (m *GetBackendTypeByTierRequest) GetTier() int32 {
+ if m != nil {
+ return m.Tier
+ }
+ return 0
+}
+
+type GetBackendTypeByTierResponse struct {
+ Types []string `protobuf:"bytes,1,rep,name=Types,proto3" json:"Types,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *GetBackendTypeByTierResponse) Reset() { *m = GetBackendTypeByTierResponse{} }
+func (m *GetBackendTypeByTierResponse) String() string { return proto.CompactTextString(m) }
+func (*GetBackendTypeByTierResponse) ProtoMessage() {}
+func (*GetBackendTypeByTierResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{71}
+}
+
+func (m *GetBackendTypeByTierResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_GetBackendTypeByTierResponse.Unmarshal(m, b)
+}
+func (m *GetBackendTypeByTierResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_GetBackendTypeByTierResponse.Marshal(b, m, deterministic)
+}
+func (m *GetBackendTypeByTierResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_GetBackendTypeByTierResponse.Merge(m, src)
+}
+func (m *GetBackendTypeByTierResponse) XXX_Size() int {
+ return xxx_messageInfo_GetBackendTypeByTierResponse.Size(m)
+}
+func (m *GetBackendTypeByTierResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_GetBackendTypeByTierResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetBackendTypeByTierResponse proto.InternalMessageInfo
+
+func (m *GetBackendTypeByTierResponse) GetTypes() []string {
+ if m != nil {
+ return m.Types
+ }
+ return nil
+}
+
+type MultipartUploadRecord struct {
+ ObjectKey string `protobuf:"bytes,1,opt,name=ObjectKey,proto3" json:"ObjectKey,omitempty"`
+ Bucket string `protobuf:"bytes,2,opt,name=Bucket,proto3" json:"Bucket,omitempty"`
+ Backend string `protobuf:"bytes,3,opt,name=Backend,proto3" json:"Backend,omitempty"`
+ UploadId string `protobuf:"bytes,4,opt,name=UploadId,proto3" json:"UploadId,omitempty"`
+ InitTime int64 `protobuf:"varint,5,opt,name=InitTime,proto3" json:"InitTime,omitempty"`
+ TenantId string `protobuf:"bytes,6,opt,name=tenantId,proto3" json:"tenantId,omitempty"`
+ UserId string `protobuf:"bytes,7,opt,name=userId,proto3" json:"userId,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *MultipartUploadRecord) Reset() { *m = MultipartUploadRecord{} }
+func (m *MultipartUploadRecord) String() string { return proto.CompactTextString(m) }
+func (*MultipartUploadRecord) ProtoMessage() {}
+func (*MultipartUploadRecord) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{72}
+}
+
+func (m *MultipartUploadRecord) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_MultipartUploadRecord.Unmarshal(m, b)
+}
+func (m *MultipartUploadRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_MultipartUploadRecord.Marshal(b, m, deterministic)
+}
+func (m *MultipartUploadRecord) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_MultipartUploadRecord.Merge(m, src)
+}
+func (m *MultipartUploadRecord) XXX_Size() int {
+ return xxx_messageInfo_MultipartUploadRecord.Size(m)
+}
+func (m *MultipartUploadRecord) XXX_DiscardUnknown() {
+ xxx_messageInfo_MultipartUploadRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MultipartUploadRecord proto.InternalMessageInfo
+
+func (m *MultipartUploadRecord) GetObjectKey() string {
+ if m != nil {
+ return m.ObjectKey
+ }
+ return ""
+}
+
+func (m *MultipartUploadRecord) GetBucket() string {
+ if m != nil {
+ return m.Bucket
+ }
+ return ""
+}
+
+func (m *MultipartUploadRecord) GetBackend() string {
+ if m != nil {
+ return m.Backend
+ }
+ return ""
+}
+
+func (m *MultipartUploadRecord) GetUploadId() string {
+ if m != nil {
+ return m.UploadId
+ }
+ return ""
+}
+
+func (m *MultipartUploadRecord) GetInitTime() int64 {
+ if m != nil {
+ return m.InitTime
+ }
+ return 0
+}
+
+func (m *MultipartUploadRecord) GetTenantId() string {
+ if m != nil {
+ return m.TenantId
+ }
+ return ""
+}
+
+func (m *MultipartUploadRecord) GetUserId() string {
+ if m != nil {
+ return m.UserId
+ }
+ return ""
+}
+
+type ListBucketUploadRequest struct {
+ BucketName string `protobuf:"bytes,1,opt,name=bucketName,proto3" json:"bucketName,omitempty"`
+ Delimiter string `protobuf:"bytes,2,opt,name=delimiter,proto3" json:"delimiter,omitempty"`
+ EncodingType string `protobuf:"bytes,3,opt,name=encodingType,proto3" json:"encodingType,omitempty"`
+ MaxUploads int32 `protobuf:"varint,4,opt,name=maxUploads,proto3" json:"maxUploads,omitempty"`
+ KeyMarker string `protobuf:"bytes,5,opt,name=keyMarker,proto3" json:"keyMarker,omitempty"`
+ Prefix string `protobuf:"bytes,6,opt,name=prefix,proto3" json:"prefix,omitempty"`
+ UploadIdMarker string `protobuf:"bytes,7,opt,name=uploadIdMarker,proto3" json:"uploadIdMarker,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ListBucketUploadRequest) Reset() { *m = ListBucketUploadRequest{} }
+func (m *ListBucketUploadRequest) String() string { return proto.CompactTextString(m) }
+func (*ListBucketUploadRequest) ProtoMessage() {}
+func (*ListBucketUploadRequest) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{73}
+}
+
+func (m *ListBucketUploadRequest) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ListBucketUploadRequest.Unmarshal(m, b)
+}
+func (m *ListBucketUploadRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ListBucketUploadRequest.Marshal(b, m, deterministic)
+}
+func (m *ListBucketUploadRequest) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ListBucketUploadRequest.Merge(m, src)
+}
+func (m *ListBucketUploadRequest) XXX_Size() int {
+ return xxx_messageInfo_ListBucketUploadRequest.Size(m)
+}
+func (m *ListBucketUploadRequest) XXX_DiscardUnknown() {
+ xxx_messageInfo_ListBucketUploadRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListBucketUploadRequest proto.InternalMessageInfo
+
+func (m *ListBucketUploadRequest) GetBucketName() string {
+ if m != nil {
+ return m.BucketName
+ }
+ return ""
+}
+
+func (m *ListBucketUploadRequest) GetDelimiter() string {
+ if m != nil {
+ return m.Delimiter
+ }
+ return ""
+}
+
+func (m *ListBucketUploadRequest) GetEncodingType() string {
+ if m != nil {
+ return m.EncodingType
+ }
+ return ""
+}
+
+func (m *ListBucketUploadRequest) GetMaxUploads() int32 {
+ if m != nil {
+ return m.MaxUploads
+ }
+ return 0
+}
+
+func (m *ListBucketUploadRequest) GetKeyMarker() string {
+ if m != nil {
+ return m.KeyMarker
+ }
+ return ""
+}
+
+func (m *ListBucketUploadRequest) GetPrefix() string {
+ if m != nil {
+ return m.Prefix
+ }
+ return ""
+}
+
+func (m *ListBucketUploadRequest) GetUploadIdMarker() string {
+ if m != nil {
+ return m.UploadIdMarker
+ }
+ return ""
+}
+
+type Owner struct {
+ Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+ DisplayName string `protobuf:"bytes,2,opt,name=displayName,proto3" json:"displayName,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Owner) Reset() { *m = Owner{} }
+func (m *Owner) String() string { return proto.CompactTextString(m) }
+func (*Owner) ProtoMessage() {}
+func (*Owner) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{74}
+}
+
+func (m *Owner) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Owner.Unmarshal(m, b)
+}
+func (m *Owner) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Owner.Marshal(b, m, deterministic)
+}
+func (m *Owner) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Owner.Merge(m, src)
+}
+func (m *Owner) XXX_Size() int {
+ return xxx_messageInfo_Owner.Size(m)
+}
+func (m *Owner) XXX_DiscardUnknown() {
+ xxx_messageInfo_Owner.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Owner proto.InternalMessageInfo
+
+func (m *Owner) GetId() string {
+ if m != nil {
+ return m.Id
+ }
+ return ""
+}
+
+func (m *Owner) GetDisplayName() string {
+ if m != nil {
+ return m.DisplayName
+ }
+ return ""
+}
+
+type Upload struct {
+ Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+ UploadId string `protobuf:"bytes,2,opt,name=uploadId,proto3" json:"uploadId,omitempty"`
+ Initiator *Owner `protobuf:"bytes,3,opt,name=initiator,proto3" json:"initiator,omitempty"`
+ Owner *Owner `protobuf:"bytes,4,opt,name=owner,proto3" json:"owner,omitempty"`
+ StorageClass string `protobuf:"bytes,5,opt,name=storageClass,proto3" json:"storageClass,omitempty"`
+ Initiated string `protobuf:"bytes,6,opt,name=initiated,proto3" json:"initiated,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Upload) Reset() { *m = Upload{} }
+func (m *Upload) String() string { return proto.CompactTextString(m) }
+func (*Upload) ProtoMessage() {}
+func (*Upload) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{75}
+}
+
+func (m *Upload) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Upload.Unmarshal(m, b)
+}
+func (m *Upload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Upload.Marshal(b, m, deterministic)
+}
+func (m *Upload) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Upload.Merge(m, src)
+}
+func (m *Upload) XXX_Size() int {
+ return xxx_messageInfo_Upload.Size(m)
+}
+func (m *Upload) XXX_DiscardUnknown() {
+ xxx_messageInfo_Upload.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Upload proto.InternalMessageInfo
+
+func (m *Upload) GetKey() string {
+ if m != nil {
+ return m.Key
+ }
+ return ""
+}
+
+func (m *Upload) GetUploadId() string {
+ if m != nil {
+ return m.UploadId
+ }
+ return ""
+}
+
+func (m *Upload) GetInitiator() *Owner {
+ if m != nil {
+ return m.Initiator
+ }
+ return nil
+}
+
+func (m *Upload) GetOwner() *Owner {
+ if m != nil {
+ return m.Owner
+ }
+ return nil
+}
+
+func (m *Upload) GetStorageClass() string {
+ if m != nil {
+ return m.StorageClass
+ }
+ return ""
+}
+
+func (m *Upload) GetInitiated() string {
+ if m != nil {
+ return m.Initiated
+ }
+ return ""
+}
+
+type ListBucketUploadResult struct {
+ IsTruncated bool `protobuf:"varint,1,opt,name=isTruncated,proto3" json:"isTruncated,omitempty"`
+ NextKeyMarker string `protobuf:"bytes,2,opt,name=nextKeyMarker,proto3" json:"nextKeyMarker,omitempty"`
+ NextUploadIdMarker string `protobuf:"bytes,3,opt,name=nextUploadIdMarker,proto3" json:"nextUploadIdMarker,omitempty"`
+ CommonPrefix []string `protobuf:"bytes,4,rep,name=commonPrefix,proto3" json:"commonPrefix,omitempty"`
+ Uploads []*Upload `protobuf:"bytes,5,rep,name=uploads,proto3" json:"uploads,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ListBucketUploadResult) Reset() { *m = ListBucketUploadResult{} }
+func (m *ListBucketUploadResult) String() string { return proto.CompactTextString(m) }
+func (*ListBucketUploadResult) ProtoMessage() {}
+func (*ListBucketUploadResult) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{76}
+}
+
+func (m *ListBucketUploadResult) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ListBucketUploadResult.Unmarshal(m, b)
+}
+func (m *ListBucketUploadResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ListBucketUploadResult.Marshal(b, m, deterministic)
+}
+func (m *ListBucketUploadResult) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ListBucketUploadResult.Merge(m, src)
+}
+func (m *ListBucketUploadResult) XXX_Size() int {
+ return xxx_messageInfo_ListBucketUploadResult.Size(m)
+}
+func (m *ListBucketUploadResult) XXX_DiscardUnknown() {
+ xxx_messageInfo_ListBucketUploadResult.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListBucketUploadResult proto.InternalMessageInfo
+
+func (m *ListBucketUploadResult) GetIsTruncated() bool {
+ if m != nil {
+ return m.IsTruncated
+ }
+ return false
+}
+
+func (m *ListBucketUploadResult) GetNextKeyMarker() string {
+ if m != nil {
+ return m.NextKeyMarker
+ }
+ return ""
+}
+
+func (m *ListBucketUploadResult) GetNextUploadIdMarker() string {
+ if m != nil {
+ return m.NextUploadIdMarker
+ }
+ return ""
+}
+
+func (m *ListBucketUploadResult) GetCommonPrefix() []string {
+ if m != nil {
+ return m.CommonPrefix
+ }
+ return nil
+}
+
+func (m *ListBucketUploadResult) GetUploads() []*Upload {
+ if m != nil {
+ return m.Uploads
+ }
+ return nil
+}
+
+type ListBucketUploadResponse struct {
+ ErrorCode int32 `protobuf:"varint,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+ Result *ListBucketUploadResult `protobuf:"bytes,2,opt,name=result,proto3" json:"result,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *ListBucketUploadResponse) Reset() { *m = ListBucketUploadResponse{} }
+func (m *ListBucketUploadResponse) String() string { return proto.CompactTextString(m) }
+func (*ListBucketUploadResponse) ProtoMessage() {}
+func (*ListBucketUploadResponse) Descriptor() ([]byte, []int) {
+ return fileDescriptor_005e34be4304e022, []int{77}
+}
+
+func (m *ListBucketUploadResponse) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_ListBucketUploadResponse.Unmarshal(m, b)
+}
+func (m *ListBucketUploadResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_ListBucketUploadResponse.Marshal(b, m, deterministic)
+}
+func (m *ListBucketUploadResponse) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_ListBucketUploadResponse.Merge(m, src)
+}
+func (m *ListBucketUploadResponse) XXX_Size() int {
+ return xxx_messageInfo_ListBucketUploadResponse.Size(m)
+}
+func (m *ListBucketUploadResponse) XXX_DiscardUnknown() {
+ xxx_messageInfo_ListBucketUploadResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListBucketUploadResponse proto.InternalMessageInfo
+
+func (m *ListBucketUploadResponse) GetErrorCode() int32 {
+ if m != nil {
+ return m.ErrorCode
+ }
+ return 0
+}
+
+func (m *ListBucketUploadResponse) GetResult() *ListBucketUploadResult {
+ if m != nil {
+ return m.Result
+ }
+ return nil
+}
+
+func init() {
+ proto.RegisterType((*ListObjectPartsRequest)(nil), "ListObjectPartsRequest")
+ proto.RegisterType((*Part)(nil), "Part")
+ proto.RegisterType((*ListObjectPartsResponse)(nil), "ListObjectPartsResponse")
+ proto.RegisterType((*AbortMultipartRequest)(nil), "AbortMultipartRequest")
+ proto.RegisterType((*CompletePart)(nil), "CompletePart")
+ proto.RegisterType((*CompleteMultipartRequest)(nil), "CompleteMultipartRequest")
+ proto.RegisterType((*CompleteMultipartResponse)(nil), "CompleteMultipartResponse")
+ proto.RegisterType((*InitMultiPartRequest)(nil), "InitMultiPartRequest")
+ proto.RegisterMapType((map[string]string)(nil), "InitMultiPartRequest.AttrsEntry")
+ proto.RegisterType((*InitMultiPartResponse)(nil), "InitMultiPartResponse")
+ proto.RegisterType((*PutDataStream)(nil), "PutDataStream")
+ proto.RegisterType((*UploadPartRequest)(nil), "UploadPartRequest")
+ proto.RegisterType((*UploadPartResponse)(nil), "UploadPartResponse")
+ proto.RegisterType((*CopyObjectRequest)(nil), "CopyObjectRequest")
+ proto.RegisterType((*CopyObjectResponse)(nil), "CopyObjectResponse")
+ proto.RegisterType((*MoveObjectRequest)(nil), "MoveObjectRequest")
+ proto.RegisterType((*MoveObjectResponse)(nil), "MoveObjectResponse")
+ proto.RegisterType((*PutObjectRequest)(nil), "PutObjectRequest")
+ proto.RegisterMapType((map[string]string)(nil), "PutObjectRequest.AttrsEntry")
+ proto.RegisterType((*GetObjectResponse)(nil), "GetObjectResponse")
+ proto.RegisterType((*GetObjectMetaResult)(nil), "GetObjectMetaResult")
+ proto.RegisterType((*PutObjectResponse)(nil), "PutObjectResponse")
+ proto.RegisterType((*PutBucketACLRequest)(nil), "PutBucketACLRequest")
+ proto.RegisterType((*BucketACL)(nil), "BucketACL")
+ proto.RegisterType((*BucketVersioning)(nil), "BucketVersioning")
+ proto.RegisterType((*ObjACL)(nil), "ObjACL")
+ proto.RegisterType((*PutObjACLRequest)(nil), "PutObjACLRequest")
+ proto.RegisterType((*BaseBucketRequest)(nil), "BaseBucketRequest")
+ proto.RegisterType((*BaseObjRequest)(nil), "BaseObjRequest")
+ proto.RegisterType((*CopyObjPartRequest)(nil), "CopyObjPartRequest")
+ proto.RegisterType((*CopyObjPartResponse)(nil), "CopyObjPartResponse")
+ proto.RegisterType((*ServerSideEncryption)(nil), "ServerSideEncryption")
+ proto.RegisterType((*RedirectAllRequestsTo)(nil), "RedirectAllRequestsTo")
+ proto.RegisterType((*Redirect)(nil), "Redirect")
+ proto.RegisterType((*Condition)(nil), "Condition")
+ proto.RegisterType((*RoutingRules)(nil), "RoutingRules")
+ proto.RegisterType((*WebsiteConfiguration)(nil), "WebsiteConfiguration")
+ proto.RegisterType((*CORSConfiguration)(nil), "CORSConfiguration")
+ proto.RegisterType((*Destination)(nil), "Destination")
+ proto.RegisterType((*ReplicationRole)(nil), "ReplicationRole")
+ proto.RegisterType((*ReplicationConfiguration)(nil), "ReplicationConfiguration")
+ proto.RegisterType((*Tag)(nil), "Tag")
+ proto.RegisterType((*LifecycleFilter)(nil), "LifecycleFilter")
+ proto.RegisterType((*Action)(nil), "Action")
+ proto.RegisterType((*AbortMultipartUpload)(nil), "AbortMultipartUpload")
+ proto.RegisterType((*LifecycleRule)(nil), "LifecycleRule")
+ proto.RegisterType((*PutBucketLifecycleRequest)(nil), "PutBucketLifecycleRequest")
+ proto.RegisterType((*GetBucketLifecycleResponse)(nil), "GetBucketLifecycleResponse")
+ proto.RegisterType((*ReplicationInfo)(nil), "ReplicationInfo")
+ proto.RegisterType((*Acl)(nil), "Acl")
+ proto.RegisterType((*GetBucketResponse)(nil), "GetBucketResponse")
+ proto.RegisterType((*Bucket)(nil), "Bucket")
+ proto.RegisterType((*Partion)(nil), "Partion")
+ proto.RegisterType((*Version)(nil), "Version")
+ proto.RegisterType((*Object)(nil), "Object")
+ proto.RegisterMapType((map[string]string)(nil), "Object.CustomAttributesEntry")
+ proto.RegisterType((*ListBucketsResponse)(nil), "ListBucketsResponse")
+ proto.RegisterType((*BaseResponse)(nil), "BaseResponse")
+ proto.RegisterType((*BaseRequest)(nil), "BaseRequest")
+ proto.RegisterType((*ListObjectsRequest)(nil), "ListObjectsRequest")
+ proto.RegisterMapType((map[string]string)(nil), "ListObjectsRequest.FilterEntry")
+ proto.RegisterType((*ListObjectsResponse)(nil), "ListObjectsResponse")
+ proto.RegisterType((*CountObjectsResponse)(nil), "CountObjectsResponse")
+ proto.RegisterType((*DeleteObjectInput)(nil), "DeleteObjectInput")
+ proto.RegisterType((*DeleteObjectOutput)(nil), "DeleteObjectOutput")
+ proto.RegisterType((*GetObjectInput)(nil), "GetObjectInput")
+ proto.RegisterType((*MultipartUpload)(nil), "MultipartUpload")
+ proto.RegisterType((*ListParts)(nil), "ListParts")
+ proto.RegisterType((*TList)(nil), "TList")
+ proto.RegisterType((*Tier2ClassName)(nil), "Tier2ClassName")
+ proto.RegisterMapType((map[int32]string)(nil), "Tier2ClassName.LstEntry")
+ proto.RegisterType((*GetTierMapResponse)(nil), "GetTierMapResponse")
+ proto.RegisterMapType((map[string]*Tier2ClassName)(nil), "GetTierMapResponse.Tier2NameEntry")
+ proto.RegisterType((*UpdateObjMetaRequest)(nil), "UpdateObjMetaRequest")
+ proto.RegisterMapType((map[string]string)(nil), "UpdateObjMetaRequest.SettingEntry")
+ proto.RegisterType((*StorageClass)(nil), "StorageClass")
+ proto.RegisterType((*GetStorageClassesResponse)(nil), "GetStorageClassesResponse")
+ proto.RegisterType((*GetBackendTypeByTierRequest)(nil), "GetBackendTypeByTierRequest")
+ proto.RegisterType((*GetBackendTypeByTierResponse)(nil), "GetBackendTypeByTierResponse")
+ proto.RegisterType((*MultipartUploadRecord)(nil), "MultipartUploadRecord")
+ proto.RegisterType((*ListBucketUploadRequest)(nil), "ListBucketUploadRequest")
+ proto.RegisterType((*Owner)(nil), "Owner")
+ proto.RegisterType((*Upload)(nil), "Upload")
+ proto.RegisterType((*ListBucketUploadResult)(nil), "ListBucketUploadResult")
+ proto.RegisterType((*ListBucketUploadResponse)(nil), "ListBucketUploadResponse")
+}
+
+func init() { proto.RegisterFile("s3.proto", fileDescriptor_005e34be4304e022) }
+
+var fileDescriptor_005e34be4304e022 = []byte{
+ // 4309 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x3b, 0x4d, 0x6f, 0x1c, 0xc9,
+ 0x75, 0xec, 0xf9, 0x22, 0xe7, 0x0d, 0x3f, 0x8b, 0x1f, 0x1a, 0x8d, 0xb4, 0x32, 0x53, 0xd6, 0x6a,
+ 0xe9, 0x4d, 0x52, 0x96, 0x28, 0xef, 0x7a, 0x57, 0x71, 0xec, 0xa5, 0x46, 0x1f, 0x4b, 0x2c, 0xb9,
+ 0x64, 0x9a, 0x94, 0x0c, 0x38, 0x08, 0x82, 0x66, 0x77, 0x91, 0x6a, 0xa9, 0xd9, 0x3d, 0xdb, 0x5d,
+ 0xa3, 0x25, 0x7d, 0x4c, 0x80, 0xc4, 0xb9, 0x24, 0xe7, 0xe4, 0xea, 0x43, 0x02, 0xe4, 0x64, 0x20,
+ 0xc7, 0x1c, 0x7c, 0xc9, 0x25, 0xd7, 0x1c, 0x02, 0xe4, 0x90, 0x4b, 0x4e, 0x01, 0xf2, 0x03, 0x9c,
+ 0x43, 0x80, 0xa0, 0x3e, 0xbb, 0xfa, 0x63, 0x38, 0x94, 0xd7, 0xf0, 0x69, 0xba, 0x5e, 0xbd, 0xaa,
+ 0x7a, 0xef, 0xd5, 0x7b, 0xaf, 0x5e, 0xbd, 0x7a, 0x03, 0x73, 0xd9, 0x43, 0x32, 0x4a, 0x13, 0x96,
+ 0xe0, 0xff, 0x74, 0x60, 0x63, 0x2f, 0xcc, 0xd8, 0xc1, 0xc9, 0x6b, 0xea, 0xb3, 0x43, 0x2f, 0x65,
+ 0x99, 0x4b, 0xbf, 0x1a, 0xd3, 0x8c, 0xa1, 0x3b, 0x00, 0x27, 0x63, 0xff, 0x0d, 0x65, 0x5f, 0x7a,
+ 0xe7, 0xb4, 0xef, 0x6c, 0x3a, 0x5b, 0x5d, 0xd7, 0x82, 0xa0, 0xdb, 0xd0, 0x4d, 0xc4, 0xa8, 0x2f,
+ 0xe8, 0x65, 0xbf, 0x21, 0xba, 0x73, 0x00, 0xc2, 0x30, 0x4f, 0x63, 0x3f, 0x09, 0xc2, 0xf8, 0xec,
+ 0xf8, 0x72, 0x44, 0xfb, 0x4d, 0x81, 0x50, 0x80, 0xa1, 0x01, 0xcc, 0x8d, 0x47, 0x51, 0xe2, 0x05,
+ 0xbb, 0x41, 0xbf, 0x25, 0xfa, 0x4d, 0x9b, 0xf7, 0x9d, 0x7b, 0x17, 0x82, 0xa0, 0x7e, 0x7b, 0xd3,
+ 0xd9, 0x6a, 0xba, 0xa6, 0x8d, 0x3e, 0x84, 0xe5, 0x91, 0x97, 0xb2, 0x2f, 0xc7, 0xe7, 0x27, 0x34,
+ 0xdd, 0xf7, 0xd2, 0x37, 0x34, 0xed, 0x77, 0x04, 0x4e, 0x05, 0x8e, 0x53, 0x68, 0xf1, 0x41, 0x9c,
+ 0x9b, 0x43, 0xd3, 0x27, 0xb8, 0x69, 0xba, 0x16, 0x04, 0x21, 0x68, 0x3d, 0x3d, 0xf6, 0xce, 0x14,
+ 0x23, 0xe2, 0x9b, 0xf3, 0xb0, 0xe7, 0x65, 0x6c, 0x3f, 0x09, 0xc2, 0xd3, 0x90, 0x06, 0x9a, 0x07,
+ 0x1b, 0xc6, 0xc7, 0x1d, 0x85, 0x3f, 0xa5, 0x82, 0xfe, 0xa6, 0x2b, 0xbe, 0xf1, 0x7f, 0x34, 0xe0,
+ 0x46, 0x45, 0xa8, 0xd9, 0x28, 0x89, 0x33, 0x21, 0xb5, 0xa7, 0x69, 0x9a, 0xa4, 0xc3, 0x24, 0x90,
+ 0x42, 0x6d, 0xbb, 0x39, 0x00, 0xdd, 0x85, 0xee, 0x6e, 0x1c, 0xb2, 0xd0, 0x63, 0x49, 0x2a, 0x48,
+ 0xe9, 0x6d, 0x77, 0xc8, 0xc1, 0xd7, 0x31, 0x4d, 0xdd, 0xbc, 0x03, 0xdd, 0x86, 0xb6, 0x80, 0x09,
+ 0x82, 0x72, 0x0c, 0x09, 0xe4, 0x54, 0x1f, 0xb1, 0x24, 0xf5, 0xce, 0xe8, 0x30, 0xf2, 0xb2, 0x4c,
+ 0x49, 0xb6, 0x00, 0xe3, 0x12, 0x3c, 0x2c, 0x4b, 0x50, 0x4a, 0xb9, 0x02, 0x47, 0xdb, 0xb0, 0xf6,
+ 0x25, 0xbd, 0x60, 0x87, 0xf5, 0x12, 0xaf, 0xed, 0xe3, 0xbb, 0xb7, 0xaf, 0x77, 0x6f, 0x56, 0xee,
+ 0x9e, 0x6e, 0xa3, 0x4d, 0xe8, 0xed, 0x66, 0xc7, 0xe9, 0x38, 0xf6, 0x3d, 0x46, 0x83, 0xfe, 0xdc,
+ 0xa6, 0xb3, 0x35, 0xe7, 0xda, 0x20, 0x74, 0x0b, 0xda, 0x72, 0x68, 0x77, 0xb3, 0xb9, 0xd5, 0xdb,
+ 0x6e, 0x13, 0xde, 0x72, 0x25, 0x0c, 0x7f, 0x05, 0xeb, 0x3b, 0x27, 0x49, 0xca, 0xf6, 0xc7, 0x11,
+ 0x0b, 0xf9, 0x76, 0xff, 0x66, 0xf4, 0xd5, 0xd6, 0xc5, 0x66, 0x51, 0x17, 0xf1, 0x63, 0x98, 0x1f,
+ 0x26, 0xe7, 0xa3, 0x88, 0x32, 0xaa, 0x75, 0x69, 0x54, 0xd1, 0xa5, 0x51, 0x41, 0x97, 0xa8, 0xa5,
+ 0x4b, 0xfc, 0x1b, 0xff, 0x55, 0x03, 0xfa, 0x7a, 0x92, 0xdf, 0x1e, 0xe9, 0xe8, 0x21, 0x2c, 0xf8,
+ 0x16, 0xe9, 0x5c, 0x1b, 0xb8, 0x48, 0x17, 0x88, 0xcd, 0x90, 0x5b, 0xc4, 0x41, 0x5b, 0xb0, 0x94,
+ 0x25, 0xe3, 0xd4, 0xa7, 0x2f, 0x69, 0x9a, 0x85, 0x49, 0xbc, 0xfb, 0x44, 0x28, 0x47, 0xd7, 0x2d,
+ 0x83, 0xf9, 0x5e, 0xa6, 0x92, 0x07, 0x61, 0xe4, 0x1d, 0x81, 0x65, 0x83, 0xb8, 0x2c, 0x58, 0x48,
+ 0x53, 0xa1, 0x05, 0x6d, 0x57, 0x7c, 0xe3, 0x10, 0x6e, 0xd6, 0x88, 0x22, 0x37, 0x10, 0x5a, 0x36,
+ 0x10, 0x03, 0xb0, 0x78, 0x7d, 0xa2, 0x04, 0x61, 0xda, 0x46, 0xec, 0x4d, 0x4b, 0xec, 0x7f, 0xdd,
+ 0x80, 0x35, 0x6e, 0x38, 0x62, 0x9d, 0xc3, 0xdf, 0x98, 0xc8, 0x37, 0xa0, 0xe9, 0xf9, 0x91, 0xb2,
+ 0xbf, 0x16, 0xd9, 0xf1, 0x23, 0x97, 0x03, 0x0c, 0xb7, 0xad, 0x9c, 0x5b, 0x4e, 0x72, 0x94, 0xf8,
+ 0x1e, 0x0b, 0x93, 0x58, 0x89, 0xd1, 0xb4, 0xd1, 0xc7, 0xd0, 0xf6, 0x18, 0x4b, 0xb3, 0x7e, 0x47,
+ 0x6c, 0xcb, 0x26, 0xa9, 0xa3, 0x95, 0xec, 0x70, 0x94, 0xa7, 0x31, 0x4b, 0x2f, 0x5d, 0x89, 0x3e,
+ 0xf8, 0x04, 0x20, 0x07, 0xa2, 0x65, 0x68, 0xbe, 0xa1, 0x97, 0x8a, 0x09, 0xfe, 0x89, 0xd6, 0xa0,
+ 0xfd, 0xd6, 0x8b, 0xc6, 0x54, 0x51, 0x2e, 0x1b, 0x8f, 0x1a, 0x9f, 0x38, 0xf8, 0x8f, 0x60, 0xbd,
+ 0xb4, 0xc6, 0x37, 0x95, 0x3b, 0xfe, 0x36, 0x2c, 0x1c, 0x8e, 0xd9, 0x13, 0x8f, 0x79, 0x47, 0x2c,
+ 0xa5, 0xde, 0x39, 0x97, 0x42, 0xe0, 0x31, 0x4f, 0xcc, 0x32, 0xef, 0x8a, 0x6f, 0xfc, 0x0b, 0x07,
+ 0x56, 0x5e, 0x88, 0x11, 0x87, 0xbf, 0x15, 0xc5, 0xdf, 0x80, 0x0e, 0x57, 0x2b, 0x75, 0xb2, 0xb4,
+ 0x5d, 0xd5, 0xe2, 0xb4, 0x65, 0xdc, 0x5f, 0x4b, 0x6f, 0x27, 0xbe, 0x39, 0xee, 0x79, 0xf0, 0xd1,
+ 0xe7, 0xf4, 0x42, 0x29, 0xb0, 0x6a, 0xe1, 0x67, 0x80, 0x6c, 0x92, 0xaf, 0x25, 0xa8, 0x3a, 0xdb,
+ 0xff, 0x3f, 0x07, 0x56, 0x86, 0xc9, 0xe8, 0x52, 0x9e, 0x07, 0x9a, 0xf7, 0xbb, 0xb0, 0x90, 0xa5,
+ 0xfe, 0xe3, 0x32, 0xfb, 0x45, 0x20, 0xf7, 0xd4, 0xcc, 0x4b, 0xcf, 0x28, 0xb3, 0x10, 0xe5, 0xdc,
+ 0x15, 0xb8, 0x9a, 0x51, 0xae, 0x22, 0x10, 0x9b, 0x66, 0xc6, 0x1c, 0x98, 0xcf, 0x68, 0x21, 0xb6,
+ 0xec, 0x19, 0x2d, 0xdc, 0xbb, 0xb0, 0xa0, 0x56, 0xf1, 0xfc, 0x37, 0x34, 0x0e, 0x94, 0x02, 0x17,
+ 0x81, 0x7c, 0x17, 0x25, 0xe0, 0x38, 0x54, 0xe7, 0x42, 0xdb, 0xb5, 0x20, 0xf8, 0x2f, 0x1c, 0x40,
+ 0x36, 0xff, 0xd7, 0x12, 0xe4, 0x32, 0x34, 0xcf, 0x83, 0x8f, 0x14, 0xaf, 0xfc, 0x93, 0x1f, 0x6c,
+ 0x51, 0xf9, 0x38, 0x6e, 0xba, 0x05, 0x18, 0x9f, 0xf3, 0xad, 0xf2, 0x4e, 0x3a, 0xa6, 0xc8, 0x01,
+ 0xf8, 0xef, 0x1b, 0xb0, 0xb2, 0x9f, 0xbc, 0xa5, 0xc5, 0x8d, 0xb8, 0x0d, 0x5d, 0x23, 0x21, 0xb5,
+ 0x09, 0x39, 0x80, 0xaf, 0x6a, 0x8b, 0x45, 0x11, 0x54, 0x80, 0xa9, 0x19, 0xe4, 0x4e, 0x28, 0xa1,
+ 0xe7, 0x80, 0x7c, 0x06, 0x85, 0xd0, 0xb2, 0x67, 0x50, 0x38, 0xf7, 0x60, 0x51, 0xb6, 0xf7, 0x8a,
+ 0xae, 0xa2, 0x04, 0x9d, 0x26, 0x6a, 0x11, 0x36, 0x25, 0x6f, 0xa9, 0xf0, 0xc6, 0xd2, 0xe5, 0x9a,
+ 0x36, 0xdf, 0x78, 0xc3, 0x96, 0x72, 0xe1, 0xe2, 0xf4, 0xed, 0xba, 0x15, 0x38, 0xfe, 0xb9, 0x03,
+ 0xc8, 0x96, 0x94, 0xda, 0xb2, 0xdf, 0x81, 0xd9, 0xe4, 0xe4, 0xf5, 0x3e, 0x55, 0xc6, 0xdd, 0xdb,
+ 0x9e, 0x25, 0x0a, 0x43, 0xc3, 0xb9, 0x31, 0x65, 0xcc, 0x63, 0xe3, 0x4c, 0x49, 0x4a, 0xb5, 0xf4,
+ 0x7e, 0x36, 0xf3, 0xfd, 0xbc, 0x72, 0xaf, 0x2a, 0xbb, 0xdd, 0xae, 0xee, 0x36, 0xfe, 0x87, 0x06,
+ 0x2c, 0x1f, 0x8e, 0x59, 0x71, 0x3b, 0xef, 0x00, 0x54, 0x8c, 0xca, 0x82, 0xf0, 0x65, 0x0f, 0xca,
+ 0x3e, 0xe5, 0xc0, 0xf6, 0xec, 0x3b, 0x65, 0xcf, 0xbe, 0xe3, 0x47, 0x68, 0x5b, 0x7b, 0x6a, 0x79,
+ 0x80, 0xde, 0x26, 0xe5, 0x75, 0xab, 0x5e, 0xfa, 0x4a, 0xcf, 0xaf, 0xfd, 0x50, 0xc7, 0xf2, 0x43,
+ 0x9b, 0xd0, 0xf3, 0x93, 0x98, 0xd1, 0x98, 0x99, 0xfd, 0xeb, 0xba, 0x36, 0xe8, 0x1b, 0xf8, 0xfd,
+ 0xa7, 0xb0, 0xf2, 0x9c, 0xb2, 0x77, 0xb2, 0x40, 0xed, 0xc6, 0x1b, 0x96, 0x1b, 0x3f, 0x86, 0x55,
+ 0x33, 0x0d, 0xdf, 0x6e, 0x97, 0x66, 0xe3, 0x88, 0x4d, 0x99, 0xe8, 0x5b, 0xd0, 0x49, 0x72, 0xe3,
+ 0xb1, 0x94, 0x46, 0x81, 0xf1, 0x2f, 0x1d, 0x58, 0xb1, 0xe4, 0x79, 0x2d, 0xea, 0x2c, 0x55, 0x6c,
+ 0x4c, 0x55, 0xc5, 0x66, 0x9d, 0x2a, 0xb6, 0x26, 0xa8, 0x62, 0x7b, 0x9a, 0x2a, 0x76, 0x6a, 0x54,
+ 0xf1, 0x47, 0xb0, 0x7a, 0x38, 0x56, 0xd6, 0xbc, 0x33, 0xdc, 0xd3, 0xca, 0xb8, 0x05, 0xdd, 0x9d,
+ 0xe1, 0xde, 0x30, 0x89, 0x4f, 0xc3, 0x33, 0x65, 0x32, 0x40, 0x72, 0xac, 0xbc, 0x13, 0xef, 0x42,
+ 0xd7, 0xc0, 0xaf, 0xa3, 0xc3, 0x43, 0x2f, 0x8e, 0x69, 0xc0, 0x75, 0x55, 0xe9, 0xb0, 0x01, 0xe0,
+ 0x0f, 0x61, 0x59, 0xe2, 0x2a, 0x6b, 0x0e, 0xe3, 0x33, 0x2e, 0x8b, 0x23, 0x29, 0x0b, 0x39, 0x9b,
+ 0x6a, 0xe1, 0x00, 0x3a, 0x07, 0x27, 0xaf, 0xaf, 0xb9, 0xe6, 0x15, 0x76, 0x53, 0xa0, 0xa8, 0x59,
+ 0xa6, 0xe8, 0x48, 0xdb, 0xa9, 0x25, 0x9a, 0x3e, 0xcc, 0x0e, 0xb9, 0x6a, 0x5f, 0x68, 0xa7, 0xab,
+ 0x9b, 0xe8, 0x7d, 0x5b, 0x68, 0xd6, 0xe6, 0x96, 0x24, 0xb6, 0x0f, 0x2b, 0x8f, 0xbd, 0x8c, 0x4a,
+ 0x12, 0xa7, 0xcf, 0x5a, 0xe4, 0xaf, 0x51, 0xe6, 0x0f, 0xbf, 0x82, 0x45, 0x3e, 0xdd, 0xc1, 0xc9,
+ 0xeb, 0x6f, 0x3c, 0x57, 0x51, 0x56, 0xcd, 0x92, 0xac, 0xf0, 0x2f, 0x1b, 0xe6, 0x3c, 0xb4, 0x83,
+ 0x21, 0x7e, 0x71, 0x13, 0xf1, 0xb5, 0x3a, 0x27, 0x1c, 0x75, 0x71, 0xb3, 0x60, 0x39, 0x4e, 0xf1,
+ 0x34, 0xb2, 0x61, 0x1c, 0xe7, 0xd8, 0x3e, 0x6f, 0xd4, 0xb5, 0xd5, 0x86, 0xe5, 0x38, 0x6a, 0x9e,
+ 0x96, 0x8d, 0xa3, 0xe6, 0xb9, 0x07, 0x8b, 0xc7, 0xb5, 0x67, 0x52, 0x11, 0xca, 0xdd, 0xdc, 0x0b,
+ 0x1d, 0x1b, 0xca, 0x00, 0xca, 0xb4, 0xb9, 0xda, 0x71, 0x16, 0x77, 0x9f, 0xa8, 0x6b, 0xa0, 0x6a,
+ 0x71, 0x01, 0xba, 0xd4, 0x0b, 0x0e, 0x4e, 0x4f, 0x33, 0xca, 0xc4, 0x29, 0xd4, 0x74, 0x2d, 0x88,
+ 0xee, 0xdf, 0xa3, 0xf1, 0x19, 0x7b, 0xd5, 0xef, 0xe6, 0xfd, 0x12, 0x82, 0xdf, 0xc0, 0x6a, 0x41,
+ 0x82, 0xd7, 0xba, 0x5d, 0x97, 0xef, 0xf3, 0x0d, 0x69, 0xc7, 0xe5, 0xfb, 0xfc, 0x53, 0x96, 0x5f,
+ 0x22, 0xf8, 0x37, 0xfe, 0x99, 0x03, 0x6b, 0x47, 0x34, 0x7d, 0x4b, 0xd3, 0xa3, 0x30, 0xa0, 0x4f,
+ 0x63, 0x3f, 0xbd, 0x1c, 0x09, 0xce, 0xfb, 0x30, 0x9b, 0x65, 0xf2, 0xb0, 0x55, 0x0a, 0xa2, 0x9a,
+ 0x3c, 0x70, 0xa2, 0x06, 0x4f, 0x1b, 0xcc, 0xbc, 0x5b, 0x04, 0xa2, 0xfb, 0xb0, 0x1a, 0xf2, 0x5b,
+ 0x7d, 0x14, 0xfe, 0x54, 0x88, 0xf2, 0x25, 0xf5, 0xf9, 0xc5, 0xbf, 0x29, 0x70, 0xeb, 0xba, 0xf0,
+ 0x01, 0xac, 0xbb, 0x34, 0x08, 0x53, 0xea, 0xb3, 0x9d, 0x28, 0x52, 0x9a, 0x93, 0x1d, 0x27, 0x7c,
+ 0x13, 0x5e, 0x25, 0x99, 0x6d, 0xbb, 0xa6, 0xcd, 0xfb, 0x44, 0xb6, 0xc7, 0x4f, 0xb8, 0xb3, 0x68,
+ 0xf2, 0x3e, 0xdd, 0xc6, 0xff, 0xea, 0xc0, 0x9c, 0x9e, 0xb1, 0x80, 0xa8, 0x26, 0xd1, 0xed, 0xc2,
+ 0x02, 0x8d, 0xd2, 0x02, 0xdb, 0xb0, 0x96, 0xd2, 0x51, 0xe4, 0xf9, 0xf4, 0x0b, 0x7a, 0x79, 0x98,
+ 0xd2, 0xd3, 0xf0, 0xe2, 0xc7, 0x21, 0x7b, 0xa5, 0x84, 0x58, 0xdb, 0xc7, 0xb5, 0x2b, 0x87, 0x0b,
+ 0x6c, 0xa9, 0x83, 0x25, 0x28, 0x8f, 0x5a, 0x5e, 0x31, 0x36, 0xd2, 0x34, 0x8a, 0x9d, 0x95, 0x7a,
+ 0x58, 0x81, 0xe3, 0xaf, 0xa1, 0x3b, 0x4c, 0xe2, 0x20, 0x14, 0x9b, 0xb3, 0x05, 0x4b, 0x6f, 0xf4,
+ 0x8a, 0x4f, 0xbf, 0x1a, 0x7b, 0x91, 0x76, 0x7d, 0x65, 0x30, 0xfa, 0x0c, 0x6e, 0xf1, 0xa9, 0x8c,
+ 0xa2, 0xb8, 0x94, 0x8d, 0xd3, 0x98, 0x06, 0x6a, 0x94, 0xe4, 0xf6, 0x2a, 0x14, 0xfc, 0xa7, 0x30,
+ 0xef, 0x26, 0x63, 0x16, 0xc6, 0x67, 0xee, 0x38, 0xa2, 0x19, 0x7a, 0x1f, 0xe6, 0x52, 0x45, 0x98,
+ 0xf2, 0xfa, 0x5d, 0xa2, 0x29, 0x75, 0x4d, 0x17, 0x3f, 0x1d, 0x7c, 0x4d, 0xaf, 0x72, 0x74, 0x40,
+ 0x0c, 0x07, 0x6e, 0xde, 0x89, 0xff, 0xcb, 0x81, 0xb5, 0x1f, 0xd3, 0x93, 0x2c, 0x64, 0x54, 0x7a,
+ 0xbf, 0x71, 0x2a, 0x8d, 0xef, 0x2e, 0x2c, 0x84, 0x71, 0x40, 0x2f, 0x9e, 0x24, 0xfe, 0xf8, 0x9c,
+ 0xc6, 0xda, 0x6b, 0x14, 0x81, 0x42, 0x1d, 0x39, 0xe9, 0x06, 0x4b, 0xf2, 0x54, 0x04, 0xa2, 0x3d,
+ 0x58, 0x4f, 0xeb, 0x94, 0x4b, 0x45, 0x43, 0x1b, 0xa4, 0x56, 0xf5, 0xdc, 0xfa, 0x41, 0xe8, 0x01,
+ 0xcc, 0xa7, 0x96, 0x4c, 0x4c, 0xe6, 0xc1, 0x16, 0x94, 0x5b, 0x40, 0xc1, 0xff, 0xcd, 0x2f, 0x4a,
+ 0x07, 0xee, 0x51, 0x91, 0xc5, 0x45, 0x68, 0x84, 0x81, 0xe2, 0xab, 0x11, 0x06, 0x5c, 0x73, 0xbc,
+ 0x28, 0x4a, 0xbe, 0xa6, 0xc1, 0x3e, 0x65, 0xaf, 0x92, 0x40, 0xef, 0x50, 0x09, 0x6a, 0xe1, 0x1d,
+ 0xa4, 0xe1, 0x59, 0x18, 0xeb, 0x30, 0xa0, 0x04, 0xb5, 0xf0, 0x3e, 0xa7, 0x5e, 0x40, 0x53, 0x9d,
+ 0x32, 0x2b, 0x41, 0xb9, 0x10, 0xcf, 0xbd, 0x8b, 0x9d, 0x33, 0x7a, 0x44, 0xf9, 0xc6, 0x64, 0xfa,
+ 0x32, 0x54, 0x00, 0xf2, 0xd9, 0xe8, 0xc5, 0x28, 0xc9, 0xf2, 0xd9, 0xa4, 0x4f, 0x2c, 0x41, 0xf1,
+ 0x2e, 0xf4, 0x9e, 0xd0, 0x8c, 0x85, 0xb1, 0x64, 0x72, 0x03, 0x3a, 0x27, 0xb6, 0xdb, 0x57, 0x2d,
+ 0xee, 0xb3, 0x32, 0x3b, 0x9b, 0xa7, 0x1c, 0xbe, 0x0d, 0xc3, 0x7f, 0xee, 0xc0, 0x92, 0x4b, 0x47,
+ 0x51, 0x28, 0x1d, 0xb2, 0x9b, 0x44, 0xb4, 0x22, 0x34, 0x7e, 0x1f, 0x16, 0x3a, 0xaf, 0xc3, 0x72,
+ 0xd9, 0xe2, 0x2e, 0x8c, 0xc6, 0xde, 0x49, 0xa4, 0xee, 0x53, 0x73, 0xae, 0x6e, 0x22, 0x02, 0xbd,
+ 0x20, 0x27, 0x50, 0xc8, 0xa4, 0xb7, 0x3d, 0x4f, 0x2c, 0xa2, 0x5d, 0x1b, 0x01, 0xbf, 0x84, 0xbe,
+ 0x45, 0x44, 0x71, 0x0b, 0x11, 0xb4, 0xd2, 0x24, 0xd2, 0x9e, 0x49, 0x7c, 0xa3, 0x7b, 0xd0, 0x4e,
+ 0x85, 0x62, 0x34, 0x84, 0x62, 0x2c, 0x93, 0x12, 0x0b, 0xae, 0xec, 0xc6, 0xdf, 0x81, 0xe6, 0xb1,
+ 0x77, 0x56, 0x13, 0xec, 0x2e, 0x43, 0xf3, 0xad, 0xa7, 0xc3, 0x1f, 0xfe, 0x89, 0xbf, 0x03, 0x4b,
+ 0x7b, 0xe1, 0x29, 0xf5, 0x2f, 0xfd, 0x88, 0x3e, 0x0b, 0x23, 0x46, 0x53, 0x8b, 0x6f, 0xc7, 0xe6,
+ 0x1b, 0xff, 0x99, 0x03, 0x9d, 0x1d, 0x5f, 0x13, 0x17, 0xe7, 0x6e, 0x53, 0x7c, 0xcb, 0xd8, 0xf7,
+ 0x52, 0x8a, 0xbb, 0xed, 0x8a, 0x6f, 0xbe, 0x15, 0x01, 0x15, 0x49, 0x2b, 0x99, 0x00, 0x55, 0xe7,
+ 0xaa, 0x0d, 0xab, 0x4d, 0x00, 0xf5, 0x61, 0xf6, 0xa4, 0x70, 0x7d, 0xd6, 0x4d, 0xfc, 0x39, 0xac,
+ 0x15, 0x73, 0x99, 0xf2, 0xdc, 0xe4, 0xe7, 0x02, 0x5f, 0x71, 0xe7, 0x94, 0xd1, 0x54, 0xa5, 0x7d,
+ 0xf9, 0x16, 0xc8, 0x03, 0xad, 0xae, 0x8b, 0xfb, 0x87, 0x05, 0xc3, 0x3a, 0x37, 0xa6, 0x3a, 0x05,
+ 0xa8, 0xbd, 0x97, 0x6d, 0x41, 0xe7, 0x54, 0x88, 0x4a, 0x59, 0xf9, 0x32, 0x29, 0x89, 0xd0, 0x55,
+ 0xfd, 0x3c, 0xe2, 0xf6, 0x84, 0xc4, 0xb4, 0x2d, 0xcf, 0x12, 0x29, 0x41, 0x57, 0xc3, 0xd1, 0x9f,
+ 0xc0, 0x1d, 0x8f, 0x33, 0xb4, 0x1b, 0xfb, 0xe5, 0x04, 0x9f, 0x64, 0x4d, 0x48, 0xa0, 0xb7, 0xbd,
+ 0x4e, 0xea, 0xf8, 0x76, 0xa7, 0x0c, 0xc6, 0x7f, 0x0c, 0x37, 0x4d, 0x90, 0x9d, 0x73, 0x7b, 0xcd,
+ 0x5c, 0xd2, 0x1d, 0x68, 0x44, 0xbe, 0x52, 0xb6, 0x45, 0x52, 0x10, 0x96, 0xdb, 0x88, 0x7c, 0xfc,
+ 0x13, 0x18, 0x3c, 0xa7, 0xd5, 0xc9, 0xaf, 0x75, 0x19, 0x99, 0x36, 0xf7, 0x59, 0xc1, 0x40, 0x77,
+ 0xe3, 0xd3, 0xc4, 0xda, 0x0f, 0xa7, 0xb0, 0x1f, 0x96, 0xb6, 0x34, 0x0a, 0xda, 0x52, 0x71, 0x05,
+ 0xcd, 0x1a, 0x57, 0xf0, 0x6d, 0x71, 0x7d, 0xe5, 0xd4, 0xfa, 0x26, 0x1a, 0x57, 0x29, 0x0d, 0x03,
+ 0xc0, 0x3f, 0x11, 0x77, 0x41, 0x1d, 0x37, 0x5f, 0x8b, 0xc1, 0x0f, 0xb4, 0x70, 0x0b, 0x17, 0x2e,
+ 0x35, 0x85, 0xd5, 0x85, 0x7f, 0xd5, 0x86, 0x8e, 0x8a, 0x31, 0xeb, 0xec, 0x6a, 0x00, 0x73, 0x8c,
+ 0xc6, 0x5e, 0xcc, 0x76, 0x35, 0x7b, 0xa6, 0xcd, 0x25, 0x32, 0xce, 0x68, 0x6a, 0x92, 0x79, 0xaa,
+ 0xa5, 0x93, 0xad, 0xad, 0x72, 0xb2, 0xf5, 0x0e, 0x80, 0x9f, 0x52, 0x8f, 0xd1, 0xe3, 0xf0, 0x5c,
+ 0x27, 0xf4, 0x2c, 0x88, 0xb2, 0x80, 0x8e, 0xb1, 0x80, 0x3e, 0xcc, 0x4a, 0x5b, 0x0d, 0x44, 0x30,
+ 0x3a, 0xe7, 0xea, 0x26, 0xda, 0x85, 0xb5, 0xac, 0x26, 0xbe, 0x13, 0x71, 0x29, 0x57, 0xd6, 0xba,
+ 0xe0, 0xcf, 0xad, 0x1d, 0x82, 0x1e, 0x00, 0xbc, 0x35, 0xb7, 0x2e, 0x11, 0xb8, 0xf6, 0xb6, 0x57,
+ 0x48, 0xf9, 0x3a, 0xe6, 0x5a, 0x48, 0x3c, 0x50, 0x09, 0xe8, 0xa9, 0x37, 0x8e, 0xf2, 0x40, 0x1b,
+ 0x64, 0xa0, 0x52, 0x02, 0x73, 0x3a, 0xbf, 0xae, 0x09, 0x02, 0xfa, 0x3d, 0x45, 0x67, 0x5d, 0x84,
+ 0xe0, 0xd6, 0x0e, 0x41, 0xf7, 0xa0, 0xe5, 0x27, 0x69, 0xd6, 0x9f, 0x17, 0x43, 0x11, 0xa9, 0x1c,
+ 0xbb, 0xae, 0xe8, 0x47, 0x2f, 0xa0, 0x9f, 0x4e, 0xf0, 0xea, 0xfd, 0x05, 0x31, 0xf6, 0x26, 0x99,
+ 0xe4, 0xf6, 0xdd, 0x89, 0x43, 0xd1, 0x33, 0xd8, 0x88, 0xb4, 0x95, 0x14, 0x27, 0x5d, 0xac, 0x35,
+ 0xa2, 0x09, 0xd8, 0xdc, 0x26, 0xa4, 0xf2, 0x1d, 0x26, 0x51, 0xe8, 0x5f, 0xf6, 0x97, 0xa4, 0x4d,
+ 0xd8, 0x30, 0xa9, 0x57, 0xde, 0x19, 0xcd, 0xfa, 0xcb, 0xf2, 0x0e, 0x22, 0x5b, 0xc6, 0x57, 0xaf,
+ 0x58, 0xbe, 0xfa, 0x11, 0x2c, 0xa5, 0x45, 0x43, 0xed, 0xa3, 0xea, 0xf1, 0xc4, 0xe1, 0x6e, 0x19,
+ 0x11, 0xff, 0xa5, 0x03, 0xb3, 0xfc, 0x36, 0xc2, 0xe9, 0xaa, 0x9e, 0x56, 0xc5, 0x47, 0xa3, 0x46,
+ 0xdd, 0xa3, 0x91, 0x48, 0x08, 0x35, 0xad, 0x84, 0x10, 0x82, 0x16, 0xe5, 0x97, 0x91, 0x96, 0x4a,
+ 0x26, 0x33, 0xf9, 0x28, 0x39, 0x35, 0x2f, 0xf6, 0x37, 0x0e, 0xcc, 0x2a, 0x65, 0xb3, 0x52, 0x1b,
+ 0xe6, 0x38, 0xc8, 0x01, 0x66, 0x85, 0x86, 0xb5, 0xc2, 0x00, 0xe6, 0xc2, 0x6c, 0xcf, 0x63, 0x34,
+ 0xd3, 0x77, 0x47, 0xd3, 0xae, 0xac, 0xde, 0xaa, 0xc9, 0xc1, 0xd6, 0xa4, 0xd8, 0xf1, 0x2f, 0x66,
+ 0x45, 0x9e, 0x41, 0x25, 0x4b, 0xf3, 0x9c, 0xbe, 0x53, 0xce, 0xe9, 0x17, 0xbd, 0x78, 0xa3, 0xe2,
+ 0xc5, 0x6d, 0x07, 0xd2, 0x9c, 0xe8, 0x40, 0x5a, 0x05, 0x07, 0x72, 0x75, 0x76, 0x67, 0x00, 0x73,
+ 0x72, 0xf9, 0x5d, 0xed, 0x2c, 0x4c, 0x7b, 0x7a, 0x46, 0x4e, 0x38, 0x21, 0xd9, 0xdc, 0x0f, 0x3e,
+ 0x52, 0xe9, 0x54, 0x0b, 0x82, 0xee, 0xc2, 0x9c, 0x5a, 0x4a, 0x3f, 0x67, 0xce, 0x11, 0xb5, 0x35,
+ 0xae, 0xe9, 0xa9, 0xb8, 0x76, 0xa8, 0xba, 0xf6, 0x89, 0x4e, 0xaa, 0xf7, 0xee, 0x4e, 0xea, 0x13,
+ 0xb8, 0xa1, 0x9c, 0x82, 0x8e, 0xe8, 0x8d, 0xe7, 0x99, 0x17, 0x2b, 0x4f, 0xea, 0xd6, 0xbe, 0x78,
+ 0xa1, 0xec, 0x8b, 0xed, 0x54, 0xe7, 0x62, 0x29, 0xd5, 0xb9, 0x01, 0x9d, 0x30, 0xfb, 0x72, 0x1c,
+ 0x45, 0xc2, 0x3a, 0xe7, 0x5c, 0xd5, 0xaa, 0xc4, 0x53, 0xcb, 0xa2, 0xb7, 0x12, 0x4f, 0x09, 0x5d,
+ 0x5a, 0xb1, 0xac, 0xa2, 0x0f, 0x2d, 0xe6, 0x9d, 0x65, 0xca, 0x30, 0x5b, 0xe4, 0xd8, 0x3b, 0x73,
+ 0x05, 0xa4, 0xa2, 0x9d, 0xab, 0xf5, 0xda, 0x29, 0x34, 0x7e, 0xcd, 0xd2, 0x78, 0xed, 0x09, 0xd6,
+ 0x2d, 0x4f, 0xc0, 0x61, 0x7c, 0xcf, 0x37, 0x14, 0x8c, 0x6f, 0xf6, 0x26, 0xf4, 0xe2, 0x71, 0x14,
+ 0xe9, 0xe4, 0xf9, 0x0d, 0xf9, 0x74, 0x6d, 0x81, 0xd0, 0x2e, 0x2c, 0xfb, 0xe3, 0x8c, 0x25, 0xe7,
+ 0x3b, 0x8c, 0xa5, 0xe1, 0xc9, 0x98, 0xd1, 0xac, 0xdf, 0x17, 0x74, 0xbe, 0xa7, 0xd2, 0x93, 0x64,
+ 0x58, 0xea, 0x97, 0x29, 0xe3, 0xca, 0x30, 0xbe, 0x98, 0xda, 0x7f, 0x71, 0xe6, 0xde, 0x94, 0xba,
+ 0x67, 0x81, 0x64, 0xec, 0xee, 0x8b, 0xf2, 0x83, 0x81, 0xe0, 0x54, 0x37, 0x07, 0x43, 0x58, 0xaf,
+ 0x5d, 0xe6, 0x9d, 0x52, 0xc6, 0x2f, 0x61, 0x75, 0x2f, 0xcc, 0x54, 0x9c, 0x90, 0x5d, 0x3f, 0x2d,
+ 0x2b, 0xad, 0x55, 0xc7, 0xf5, 0x26, 0x4a, 0xd0, 0x70, 0xfc, 0x43, 0x98, 0x7f, 0xec, 0x65, 0xf4,
+ 0x1d, 0xde, 0x81, 0xb2, 0x33, 0xf3, 0x0e, 0x94, 0x9d, 0xe1, 0xf7, 0xa0, 0x27, 0xc7, 0xcb, 0xb8,
+ 0xaf, 0x14, 0xe8, 0xe2, 0xbf, 0x6b, 0x01, 0xca, 0xab, 0x2f, 0x4c, 0x39, 0xcb, 0xa4, 0x0b, 0x56,
+ 0xee, 0x1c, 0x54, 0x46, 0x68, 0xce, 0xcd, 0x01, 0x5c, 0xc4, 0xaa, 0x21, 0xbc, 0x4d, 0xdb, 0xd5,
+ 0x4d, 0x3e, 0x2e, 0xa0, 0x51, 0x78, 0x1e, 0x32, 0x15, 0xee, 0x77, 0xdd, 0x1c, 0x50, 0x29, 0x7f,
+ 0x69, 0xd7, 0x94, 0xbf, 0xf4, 0x61, 0xf6, 0xdc, 0xbb, 0xf8, 0x82, 0x5e, 0x66, 0xea, 0x21, 0x47,
+ 0x37, 0xad, 0x4b, 0xcb, 0x6c, 0xe1, 0xb2, 0xb6, 0x01, 0x9d, 0x73, 0x69, 0x2b, 0x73, 0xea, 0xa1,
+ 0x52, 0x5a, 0xc9, 0xef, 0xc1, 0x0a, 0x77, 0x39, 0x61, 0x3c, 0x16, 0x16, 0x77, 0x9c, 0xbc, 0xa1,
+ 0xb1, 0x88, 0x3d, 0xba, 0x6e, 0xb5, 0x83, 0xbb, 0xac, 0x8c, 0x79, 0x29, 0x13, 0x77, 0x08, 0xe5,
+ 0x6a, 0x2c, 0x08, 0xef, 0x3f, 0xa5, 0xcc, 0x7f, 0x25, 0x6b, 0x4c, 0x7a, 0x42, 0x24, 0x16, 0x84,
+ 0x73, 0xfe, 0x86, 0x5e, 0x2a, 0xa3, 0x95, 0xfe, 0x22, 0x07, 0xf0, 0x68, 0xc6, 0xf8, 0x56, 0x85,
+ 0xb3, 0x20, 0xa3, 0x99, 0x12, 0x18, 0x7d, 0xdf, 0xdc, 0x3c, 0xe4, 0x99, 0xff, 0x2d, 0x52, 0xdd,
+ 0x36, 0x22, 0xef, 0x20, 0xd2, 0x46, 0x14, 0xfa, 0xe0, 0x53, 0xe8, 0x59, 0xe0, 0x77, 0xd2, 0xe9,
+ 0x7f, 0x72, 0xa4, 0x52, 0x9b, 0x55, 0xae, 0xa5, 0x83, 0x9b, 0xd0, 0x0b, 0xad, 0x92, 0x15, 0xa9,
+ 0x25, 0x36, 0x88, 0xcb, 0x2c, 0xa6, 0x17, 0xac, 0x70, 0x33, 0xb4, 0x20, 0xdc, 0x3f, 0xca, 0x94,
+ 0x93, 0x4a, 0x84, 0x74, 0x5d, 0xd3, 0x56, 0x2f, 0x19, 0x9c, 0x9c, 0x7e, 0x5b, 0x99, 0x8c, 0xf5,
+ 0x92, 0xc1, 0xe1, 0xf8, 0x33, 0x58, 0x1b, 0x26, 0xe3, 0xb8, 0x42, 0xf6, 0x1a, 0xb4, 0x05, 0x5c,
+ 0x15, 0xa1, 0xc8, 0x86, 0xa9, 0x49, 0x6a, 0x58, 0x35, 0x49, 0xff, 0xe8, 0xc0, 0xca, 0x13, 0xe1,
+ 0x59, 0xe5, 0x1c, 0xbb, 0xf1, 0x68, 0x3c, 0xd9, 0x28, 0x94, 0x48, 0x1b, 0xb9, 0x48, 0x07, 0xfa,
+ 0x1c, 0xcb, 0xcf, 0x5d, 0xdd, 0x2e, 0x7b, 0xaa, 0x56, 0xd5, 0x53, 0xe9, 0x57, 0xf1, 0x76, 0xfe,
+ 0x2a, 0x7e, 0xd5, 0xb9, 0x8b, 0x19, 0x20, 0x9b, 0xd8, 0x83, 0x31, 0xe3, 0xd4, 0x5e, 0xbd, 0x49,
+ 0xe5, 0xe3, 0xa4, 0x51, 0x73, 0x9c, 0x14, 0x22, 0x81, 0x66, 0xf9, 0x79, 0xf8, 0x67, 0x0e, 0x2c,
+ 0x9a, 0xd7, 0x2d, 0x23, 0xa0, 0x42, 0x36, 0x5e, 0xdf, 0x6d, 0x96, 0xa1, 0x99, 0x3f, 0x83, 0x34,
+ 0xd5, 0x03, 0xc8, 0xcb, 0xf2, 0xd4, 0x06, 0xc0, 0xe7, 0x51, 0xb9, 0x6e, 0x19, 0x31, 0xa9, 0x16,
+ 0x87, 0xab, 0x1c, 0xb7, 0x8c, 0x96, 0x54, 0x0b, 0xff, 0xdc, 0x81, 0xa5, 0x72, 0x56, 0xe0, 0x9d,
+ 0x36, 0x6b, 0x62, 0x61, 0x84, 0x2d, 0xf6, 0x56, 0x29, 0xdc, 0x99, 0xf2, 0x60, 0xc9, 0xf2, 0x37,
+ 0x67, 0x59, 0xc8, 0xf3, 0xb7, 0x0e, 0x74, 0xb9, 0x35, 0xc9, 0xb2, 0xa1, 0xeb, 0xcb, 0x6a, 0x0a,
+ 0x7d, 0xa6, 0x74, 0xac, 0x55, 0x2a, 0x1d, 0x7b, 0x87, 0xb2, 0x35, 0x7c, 0x0b, 0xda, 0xc7, 0x9c,
+ 0x36, 0x4e, 0xb8, 0x78, 0x2c, 0x77, 0x36, 0x9b, 0x9c, 0x70, 0x51, 0x91, 0xc0, 0x60, 0x91, 0xff,
+ 0x6e, 0x8b, 0xc8, 0x4a, 0x55, 0x45, 0x34, 0xa3, 0x8c, 0x09, 0xa4, 0xde, 0x76, 0x9f, 0x14, 0x7b,
+ 0xc9, 0x5e, 0xc6, 0xa4, 0x0b, 0xe2, 0x48, 0x83, 0x8f, 0x61, 0x4e, 0x03, 0x6c, 0xe7, 0xd3, 0x9e,
+ 0xe6, 0x7c, 0xfe, 0xc5, 0x01, 0xf4, 0x5c, 0x3e, 0xd4, 0xef, 0x7b, 0x23, 0x63, 0xc4, 0x77, 0x00,
+ 0x8e, 0x53, 0x2f, 0xce, 0x42, 0x95, 0xe4, 0xe1, 0xde, 0xc1, 0x82, 0xa0, 0xcf, 0xa0, 0x2b, 0xc8,
+ 0x51, 0x11, 0x31, 0x27, 0x10, 0x93, 0xea, 0x3c, 0xc4, 0x20, 0x49, 0x52, 0xf3, 0x41, 0x83, 0x7d,
+ 0xc5, 0xae, 0xe9, 0xac, 0xf1, 0x99, 0xef, 0xdb, 0x64, 0xf7, 0xb6, 0x97, 0x4a, 0x22, 0xb0, 0xf9,
+ 0xf8, 0x1f, 0x07, 0xd6, 0x5e, 0x8c, 0x02, 0x4f, 0x98, 0xa7, 0x7c, 0x05, 0x36, 0x67, 0xec, 0xc1,
+ 0xc9, 0xeb, 0x3c, 0xae, 0x57, 0xad, 0xa9, 0xcf, 0x65, 0x75, 0x85, 0x96, 0xe5, 0x87, 0x99, 0x1f,
+ 0xc0, 0xec, 0x11, 0x65, 0x8c, 0xdf, 0xaa, 0x5b, 0x4a, 0x06, 0x75, 0x34, 0x10, 0x85, 0x24, 0x65,
+ 0xa0, 0x87, 0x0c, 0x1e, 0xc1, 0xbc, 0xdd, 0xf1, 0x4e, 0x67, 0xc6, 0xc7, 0xc5, 0x82, 0x4a, 0xae,
+ 0x50, 0x56, 0x8a, 0x49, 0x7c, 0x1b, 0x25, 0x53, 0xf9, 0x42, 0xa1, 0x64, 0x4f, 0xe0, 0xe6, 0x73,
+ 0xca, 0xec, 0xa1, 0x34, 0xf7, 0xdc, 0x1f, 0xc0, 0xac, 0x2f, 0x41, 0x4a, 0xe7, 0x16, 0x88, 0x8d,
+ 0xe9, 0xea, 0x5e, 0xfc, 0x00, 0x6e, 0x3d, 0x37, 0xa5, 0x36, 0x3c, 0x6e, 0x78, 0x7c, 0xc9, 0x67,
+ 0xd7, 0x22, 0xcf, 0xb5, 0x3b, 0x5f, 0xf8, 0x7b, 0x70, 0xbb, 0x7e, 0x48, 0x7e, 0x6a, 0x70, 0x68,
+ 0xa6, 0x74, 0x4d, 0x36, 0xf0, 0xbf, 0x39, 0xb0, 0x5e, 0x4e, 0xc8, 0x51, 0x3f, 0x49, 0x83, 0xe2,
+ 0x6b, 0xa6, 0x53, 0xad, 0x98, 0xd0, 0x66, 0xdf, 0x28, 0x98, 0x7d, 0x1f, 0x66, 0x75, 0xd5, 0x90,
+ 0xb4, 0x71, 0xdd, 0xb4, 0x1e, 0x0c, 0x8d, 0x0b, 0x7a, 0x61, 0x99, 0xff, 0x6e, 0x1c, 0x32, 0x2b,
+ 0xa5, 0x63, 0xda, 0x85, 0xbb, 0x5f, 0x67, 0xe2, 0xdd, 0x6f, 0xd6, 0xbe, 0xfb, 0xe1, 0xff, 0x75,
+ 0x64, 0x2d, 0xae, 0x24, 0x4a, 0xb3, 0x75, 0xed, 0xea, 0xb3, 0x3c, 0xc4, 0x6b, 0x4c, 0x0b, 0xf1,
+ 0xea, 0x2a, 0x9c, 0xef, 0x00, 0x9c, 0x7b, 0x17, 0x72, 0xd5, 0x4c, 0x25, 0x85, 0x2d, 0x48, 0x31,
+ 0x94, 0x6a, 0x97, 0x43, 0xa9, 0x3c, 0x0c, 0xec, 0x14, 0xc2, 0xc0, 0x7b, 0xb0, 0xa8, 0xdd, 0xa5,
+ 0x1a, 0x2a, 0x79, 0x2e, 0x41, 0xf1, 0xa7, 0xaa, 0x4e, 0xb8, 0x92, 0x0b, 0xde, 0x84, 0x5e, 0x10,
+ 0x66, 0xa3, 0xc8, 0xbb, 0xb4, 0x0c, 0xd2, 0x06, 0xe1, 0x7f, 0x76, 0xa0, 0xa3, 0x8e, 0x9d, 0xaa,
+ 0xa9, 0xd8, 0xee, 0xbb, 0x51, 0x72, 0xdf, 0x77, 0xa1, 0x1b, 0x9a, 0x0a, 0xe6, 0x62, 0x7d, 0x72,
+ 0xde, 0x81, 0x6e, 0x43, 0x3b, 0x11, 0xd1, 0x65, 0xab, 0x58, 0xc1, 0x9c, 0xe8, 0x0a, 0xe6, 0xc2,
+ 0x6d, 0xb8, 0x5d, 0x73, 0x1b, 0xbe, 0x6d, 0xd6, 0xa1, 0x5a, 0x19, 0x72, 0x00, 0xfe, 0x77, 0x55,
+ 0xd6, 0x5e, 0xdc, 0x75, 0x51, 0xaa, 0x52, 0x8a, 0xe5, 0x9c, 0x6a, 0x2c, 0x77, 0x17, 0x16, 0x78,
+ 0xe4, 0xf6, 0x85, 0xd9, 0x18, 0xf5, 0x58, 0x56, 0x00, 0x22, 0x02, 0x88, 0x03, 0x5e, 0x14, 0x37,
+ 0x42, 0x2a, 0x41, 0x4d, 0x0f, 0x67, 0xca, 0x4f, 0xce, 0xcf, 0x93, 0x58, 0xc6, 0x7d, 0x2a, 0x0a,
+ 0x2c, 0xc0, 0x78, 0x24, 0x38, 0x56, 0xba, 0xa2, 0x23, 0x41, 0x45, 0xbb, 0x86, 0xe3, 0x10, 0xfa,
+ 0x35, 0x8c, 0x5d, 0x27, 0x88, 0xfd, 0x2e, 0x74, 0x52, 0x21, 0x02, 0xe5, 0xe1, 0x6f, 0x90, 0x7a,
+ 0x09, 0xb9, 0x0a, 0x6d, 0xfb, 0x57, 0x6b, 0xd0, 0x38, 0x7a, 0x88, 0x1e, 0x42, 0xcf, 0xba, 0x06,
+ 0xa2, 0x79, 0x62, 0x5d, 0xbe, 0x06, 0x6b, 0xa4, 0xe6, 0x8a, 0x88, 0x67, 0xd0, 0x16, 0xcc, 0x0f,
+ 0x45, 0x26, 0x56, 0x39, 0x03, 0x7d, 0x0b, 0x1c, 0x2c, 0x10, 0xfb, 0xee, 0x27, 0x31, 0x65, 0xa8,
+ 0x37, 0x15, 0xf3, 0x43, 0xe8, 0x9a, 0xb4, 0x75, 0x8e, 0x86, 0x48, 0x25, 0x97, 0x8d, 0x67, 0xd0,
+ 0x7d, 0x58, 0x28, 0xd4, 0x29, 0x21, 0x1d, 0x53, 0x0f, 0xd6, 0x48, 0x4d, 0x01, 0x13, 0x9e, 0x41,
+ 0x0f, 0x60, 0xd9, 0x9c, 0x27, 0x95, 0x41, 0x88, 0x54, 0xca, 0x93, 0xf0, 0x0c, 0x7a, 0x24, 0x25,
+ 0xa3, 0x82, 0x72, 0xb4, 0x5a, 0x73, 0x7f, 0x51, 0x02, 0x2a, 0xc5, 0xed, 0x78, 0x06, 0xfd, 0x00,
+ 0xe6, 0xed, 0x88, 0xbe, 0x7e, 0xf0, 0x3a, 0xa9, 0x8b, 0xfa, 0xf1, 0x0c, 0x7a, 0x08, 0x5d, 0x43,
+ 0x10, 0x5a, 0x24, 0x85, 0xf2, 0xdb, 0x7a, 0x62, 0xb7, 0x1c, 0x2e, 0x69, 0x9b, 0xc3, 0x9c, 0xbb,
+ 0x8a, 0xa4, 0xbf, 0x27, 0x24, 0xad, 0xd0, 0x96, 0x48, 0x31, 0x26, 0x96, 0x12, 0x2f, 0xcf, 0x7f,
+ 0xdf, 0x41, 0x9f, 0xea, 0x9d, 0x54, 0x03, 0x11, 0xa9, 0x5c, 0x38, 0x06, 0xab, 0xa4, 0x1a, 0xd7,
+ 0x0b, 0xe1, 0x43, 0x1e, 0xd0, 0x94, 0x54, 0x6c, 0xb5, 0x26, 0xd6, 0xc1, 0x33, 0xe8, 0xfb, 0xb0,
+ 0x50, 0x38, 0xff, 0xd1, 0x7a, 0x6d, 0x3c, 0x50, 0x65, 0xee, 0x47, 0xe2, 0xf5, 0xa3, 0x78, 0x2c,
+ 0x97, 0x96, 0x1c, 0x90, 0x89, 0x07, 0x37, 0x9e, 0x41, 0x2f, 0x60, 0xad, 0xee, 0x78, 0x45, 0xb7,
+ 0xc9, 0x15, 0x07, 0xf5, 0xe0, 0x3d, 0x72, 0xd5, 0x99, 0x8c, 0x67, 0xd0, 0xc7, 0xb0, 0x6e, 0x1b,
+ 0x82, 0xc9, 0x7f, 0x97, 0x68, 0xab, 0xf0, 0xb3, 0x03, 0xa8, 0xfa, 0x28, 0x86, 0x06, 0x64, 0xe2,
+ 0x4b, 0x59, 0xed, 0x14, 0xd5, 0xa7, 0xaf, 0xd2, 0xba, 0xb7, 0xc8, 0xe4, 0xd7, 0x31, 0x3c, 0x83,
+ 0xfe, 0xc0, 0x4e, 0x16, 0x4d, 0x9a, 0xe3, 0x0a, 0x6f, 0x21, 0xf7, 0x6e, 0xaa, 0x0f, 0x38, 0xac,
+ 0x3b, 0xcd, 0x79, 0x90, 0x92, 0xa1, 0x3e, 0x99, 0x70, 0xce, 0x0f, 0x6e, 0x92, 0x49, 0x2e, 0x13,
+ 0xcf, 0xa0, 0x67, 0xb0, 0x6a, 0x0a, 0xe2, 0xad, 0xcb, 0xd6, 0x7a, 0x6d, 0x29, 0xfe, 0x60, 0x83,
+ 0xd4, 0x56, 0xcf, 0x0b, 0xb5, 0xaa, 0x7f, 0xcb, 0xdd, 0x20, 0xb5, 0x7f, 0x57, 0xa9, 0xb2, 0xe6,
+ 0xc2, 0x8d, 0x61, 0xfd, 0xbb, 0x27, 0xba, 0x49, 0x26, 0xfd, 0x75, 0x64, 0x30, 0x20, 0x13, 0xff,
+ 0x4a, 0x81, 0x67, 0xd0, 0x47, 0x00, 0x79, 0x05, 0x7b, 0xc5, 0x51, 0xac, 0x92, 0x6a, 0x79, 0xbb,
+ 0xf0, 0x14, 0xcf, 0x60, 0xa9, 0xf4, 0xff, 0x25, 0x74, 0x83, 0xd4, 0xff, 0x4d, 0x6c, 0xd0, 0x27,
+ 0x13, 0xfe, 0xea, 0x84, 0x67, 0xd0, 0xef, 0xc3, 0xfc, 0xce, 0x68, 0x44, 0xe3, 0x40, 0x79, 0x84,
+ 0x29, 0x9a, 0xfc, 0xbb, 0x00, 0x87, 0x89, 0x9e, 0x6b, 0x1a, 0xf2, 0x23, 0x58, 0xda, 0x09, 0x82,
+ 0x42, 0x9c, 0xba, 0x41, 0x6a, 0xe3, 0xd7, 0xea, 0xd8, 0x3f, 0xd4, 0xe9, 0x85, 0x5f, 0x6f, 0xf8,
+ 0x16, 0xc0, 0xe7, 0xd4, 0x0b, 0x8c, 0x7f, 0x2c, 0x96, 0x0d, 0x0e, 0xb4, 0x5f, 0x15, 0x4e, 0x0a,
+ 0xf2, 0x2a, 0x6a, 0x84, 0x48, 0xa5, 0xf8, 0x7c, 0xb0, 0x4a, 0xaa, 0x65, 0xd6, 0x72, 0x60, 0x5e,
+ 0x31, 0x8f, 0x10, 0xa9, 0xfc, 0x7d, 0x60, 0xb0, 0x4a, 0xaa, 0x25, 0xf5, 0xf2, 0x4c, 0xb2, 0x0a,
+ 0xe3, 0x90, 0xc1, 0xb2, 0x95, 0x78, 0x8d, 0xd4, 0xd4, 0xce, 0xe1, 0x19, 0xf4, 0x5d, 0x7d, 0xaa,
+ 0xec, 0x0c, 0xf7, 0xd0, 0x0a, 0x29, 0x57, 0x6c, 0x56, 0x05, 0xf1, 0x81, 0x3e, 0x27, 0xf8, 0x80,
+ 0x7a, 0x39, 0xec, 0x0c, 0xf7, 0xf0, 0x0c, 0xda, 0xb6, 0x5e, 0x9c, 0xcd, 0x5b, 0xc2, 0x94, 0x0d,
+ 0xfe, 0xa1, 0x28, 0x35, 0xae, 0x14, 0xb2, 0x22, 0x52, 0x29, 0xfa, 0x1c, 0x54, 0x1f, 0x58, 0x85,
+ 0xee, 0xcf, 0xdb, 0x15, 0xb9, 0x68, 0x8d, 0xd4, 0x14, 0xe8, 0x56, 0x97, 0xbd, 0x0f, 0xf3, 0x66,
+ 0x59, 0x3e, 0xac, 0x6e, 0x3d, 0xab, 0x84, 0x17, 0xcf, 0x20, 0x22, 0xfe, 0xff, 0x22, 0x21, 0xc3,
+ 0x03, 0xf7, 0x68, 0x1a, 0x63, 0x44, 0xc4, 0x26, 0xd7, 0xc7, 0x7f, 0x00, 0xcb, 0xf6, 0xc1, 0x70,
+ 0x9d, 0x21, 0xf7, 0x61, 0xc9, 0x90, 0xa4, 0x5e, 0x41, 0xa7, 0x8f, 0x30, 0x44, 0x5d, 0x6f, 0xc4,
+ 0x43, 0x6d, 0x44, 0xef, 0x32, 0xe8, 0x7d, 0x69, 0x3a, 0xca, 0xcf, 0x17, 0x91, 0xb5, 0xd7, 0xc7,
+ 0x33, 0x27, 0x1d, 0x51, 0x76, 0xf8, 0xf0, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0x02, 0x1b, 0x52,
+ 0xd9, 0xa9, 0x3a, 0x00, 0x00,
+}
+
+
+
// Code generated by protoc-gen-micro. DO NOT EDIT.
+// source: s3.proto
+
+package s3
+
+import (
+ fmt "fmt"
+ proto "github.com/golang/protobuf/proto"
+ math "math"
+)
+
+import (
+ context "context"
+ client "github.com/micro/go-micro/client"
+ server "github.com/micro/go-micro/server"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ client.Option
+var _ server.Option
+
+// Client API for S3 service
+
+type S3Service interface {
+ ListBuckets(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*ListBucketsResponse, error)
+ CreateBucket(ctx context.Context, in *Bucket, opts ...client.CallOption) (*BaseResponse, error)
+ DeleteBucket(ctx context.Context, in *Bucket, opts ...client.CallOption) (*BaseResponse, error)
+ GetBucket(ctx context.Context, in *Bucket, opts ...client.CallOption) (*GetBucketResponse, error)
+ GetObjectMeta(ctx context.Context, in *Object, opts ...client.CallOption) (*GetObjectMetaResult, error)
+ UpdateObjectMeta(ctx context.Context, in *Object, opts ...client.CallOption) (*PutObjectResponse, error)
+ ListObjects(ctx context.Context, in *ListObjectsRequest, opts ...client.CallOption) (*ListObjectsResponse, error)
+ CountObjects(ctx context.Context, in *ListObjectsRequest, opts ...client.CallOption) (*CountObjectsResponse, error)
+ PutObject(ctx context.Context, opts ...client.CallOption) (S3_PutObjectService, error)
+ UpdateObject(ctx context.Context, in *Object, opts ...client.CallOption) (*BaseResponse, error)
+ GetObject(ctx context.Context, in *GetObjectInput, opts ...client.CallOption) (S3_GetObjectService, error)
+ DeleteObject(ctx context.Context, in *DeleteObjectInput, opts ...client.CallOption) (*DeleteObjectOutput, error)
+ GetTierMap(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*GetTierMapResponse, error)
+ UpdateObjMeta(ctx context.Context, in *UpdateObjMetaRequest, opts ...client.CallOption) (*BaseResponse, error)
+ GetStorageClasses(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*GetStorageClassesResponse, error)
+ GetBackendTypeByTier(ctx context.Context, in *GetBackendTypeByTierRequest, opts ...client.CallOption) (*GetBackendTypeByTierResponse, error)
+ DeleteBucketLifecycle(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error)
+ PutBucketLifecycle(ctx context.Context, in *PutBucketLifecycleRequest, opts ...client.CallOption) (*BaseResponse, error)
+ GetBucketLifecycle(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*GetBucketLifecycleResponse, error)
+ ListBucketLifecycle(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*ListBucketsResponse, error)
+ UpdateBucket(ctx context.Context, in *Bucket, opts ...client.CallOption) (*BaseResponse, error)
+ ListBucketUploadRecords(ctx context.Context, in *ListBucketUploadRequest, opts ...client.CallOption) (*ListBucketUploadResponse, error)
+ InitMultipartUpload(ctx context.Context, in *InitMultiPartRequest, opts ...client.CallOption) (*InitMultiPartResponse, error)
+ AbortMultipartUpload(ctx context.Context, in *AbortMultipartRequest, opts ...client.CallOption) (*BaseResponse, error)
+ CompleteMultipartUpload(ctx context.Context, in *CompleteMultipartRequest, opts ...client.CallOption) (*CompleteMultipartResponse, error)
+ UploadPart(ctx context.Context, opts ...client.CallOption) (S3_UploadPartService, error)
+ ListObjectParts(ctx context.Context, in *ListObjectPartsRequest, opts ...client.CallOption) (*ListObjectPartsResponse, error)
+ AppendObject(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error)
+ PostObject(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error)
+ // For lifecycle, may need some change.
+ AddUploadRecord(ctx context.Context, in *MultipartUploadRecord, opts ...client.CallOption) (*BaseResponse, error)
+ DeleteUploadRecord(ctx context.Context, in *MultipartUploadRecord, opts ...client.CallOption) (*BaseResponse, error)
+ HeadObject(ctx context.Context, in *BaseObjRequest, opts ...client.CallOption) (*Object, error)
+ MoveObject(ctx context.Context, in *MoveObjectRequest, opts ...client.CallOption) (*MoveObjectResponse, error)
+ CopyObject(ctx context.Context, in *CopyObjectRequest, opts ...client.CallOption) (*CopyObjectResponse, error)
+ CopyObjPart(ctx context.Context, in *CopyObjPartRequest, opts ...client.CallOption) (*CopyObjPartResponse, error)
+ PutObjACL(ctx context.Context, in *PutObjACLRequest, opts ...client.CallOption) (*BaseResponse, error)
+ GetObjACL(ctx context.Context, in *BaseObjRequest, opts ...client.CallOption) (*ObjACL, error)
+ GetBucketLocation(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error)
+ GetBucketVersioning(ctx context.Context, in *BaseBucketRequest, opts ...client.CallOption) (*BucketVersioning, error)
+ PutBucketACL(ctx context.Context, in *PutBucketACLRequest, opts ...client.CallOption) (*BaseResponse, error)
+ GetBucketACL(ctx context.Context, in *BaseBucketRequest, opts ...client.CallOption) (*BucketACL, error)
+ PutBucketCORS(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error)
+ GetBucketCORS(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error)
+ DeleteBucketCORS(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error)
+ PutBucketPolicy(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error)
+ GetBucketPolicy(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error)
+ DeleteBucketPolicy(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error)
+ HeadBucket(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*Bucket, error)
+}
+
+type s3Service struct {
+ c client.Client
+ name string
+}
+
+func NewS3Service(name string, c client.Client) S3Service {
+ if c == nil {
+ c = client.NewClient()
+ }
+ if len(name) == 0 {
+ name = "s3"
+ }
+ return &s3Service{
+ c: c,
+ name: name,
+ }
+}
+
+func (c *s3Service) ListBuckets(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*ListBucketsResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.ListBuckets", in)
+ out := new(ListBucketsResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) CreateBucket(ctx context.Context, in *Bucket, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.CreateBucket", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) DeleteBucket(ctx context.Context, in *Bucket, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.DeleteBucket", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetBucket(ctx context.Context, in *Bucket, opts ...client.CallOption) (*GetBucketResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.GetBucket", in)
+ out := new(GetBucketResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetObjectMeta(ctx context.Context, in *Object, opts ...client.CallOption) (*GetObjectMetaResult, error) {
+ req := c.c.NewRequest(c.name, "S3.GetObjectMeta", in)
+ out := new(GetObjectMetaResult)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) UpdateObjectMeta(ctx context.Context, in *Object, opts ...client.CallOption) (*PutObjectResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.UpdateObjectMeta", in)
+ out := new(PutObjectResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) ListObjects(ctx context.Context, in *ListObjectsRequest, opts ...client.CallOption) (*ListObjectsResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.ListObjects", in)
+ out := new(ListObjectsResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) CountObjects(ctx context.Context, in *ListObjectsRequest, opts ...client.CallOption) (*CountObjectsResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.CountObjects", in)
+ out := new(CountObjectsResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) PutObject(ctx context.Context, opts ...client.CallOption) (S3_PutObjectService, error) {
+ req := c.c.NewRequest(c.name, "S3.PutObject", &PutDataStream{})
+ stream, err := c.c.Stream(ctx, req, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return &s3ServicePutObject{stream}, nil
+}
+
+type S3_PutObjectService interface {
+ SendMsg(interface{}) error
+ RecvMsg(interface{}) error
+ Close() error
+ Send(*PutDataStream) error
+}
+
+type s3ServicePutObject struct {
+ stream client.Stream
+}
+
+func (x *s3ServicePutObject) Close() error {
+ return x.stream.Close()
+}
+
+func (x *s3ServicePutObject) SendMsg(m interface{}) error {
+ return x.stream.Send(m)
+}
+
+func (x *s3ServicePutObject) RecvMsg(m interface{}) error {
+ return x.stream.Recv(m)
+}
+
+func (x *s3ServicePutObject) Send(m *PutDataStream) error {
+ return x.stream.Send(m)
+}
+
+func (c *s3Service) UpdateObject(ctx context.Context, in *Object, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.UpdateObject", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetObject(ctx context.Context, in *GetObjectInput, opts ...client.CallOption) (S3_GetObjectService, error) {
+ req := c.c.NewRequest(c.name, "S3.GetObject", &GetObjectInput{})
+ stream, err := c.c.Stream(ctx, req, opts...)
+ if err != nil {
+ return nil, err
+ }
+ if err := stream.Send(in); err != nil {
+ return nil, err
+ }
+ return &s3ServiceGetObject{stream}, nil
+}
+
+type S3_GetObjectService interface {
+ SendMsg(interface{}) error
+ RecvMsg(interface{}) error
+ Close() error
+ Recv() (*GetObjectResponse, error)
+}
+
+type s3ServiceGetObject struct {
+ stream client.Stream
+}
+
+func (x *s3ServiceGetObject) Close() error {
+ return x.stream.Close()
+}
+
+func (x *s3ServiceGetObject) SendMsg(m interface{}) error {
+ return x.stream.Send(m)
+}
+
+func (x *s3ServiceGetObject) RecvMsg(m interface{}) error {
+ return x.stream.Recv(m)
+}
+
+func (x *s3ServiceGetObject) Recv() (*GetObjectResponse, error) {
+ m := new(GetObjectResponse)
+ err := x.stream.Recv(m)
+ if err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+func (c *s3Service) DeleteObject(ctx context.Context, in *DeleteObjectInput, opts ...client.CallOption) (*DeleteObjectOutput, error) {
+ req := c.c.NewRequest(c.name, "S3.DeleteObject", in)
+ out := new(DeleteObjectOutput)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetTierMap(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*GetTierMapResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.GetTierMap", in)
+ out := new(GetTierMapResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) UpdateObjMeta(ctx context.Context, in *UpdateObjMetaRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.UpdateObjMeta", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetStorageClasses(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*GetStorageClassesResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.GetStorageClasses", in)
+ out := new(GetStorageClassesResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetBackendTypeByTier(ctx context.Context, in *GetBackendTypeByTierRequest, opts ...client.CallOption) (*GetBackendTypeByTierResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.GetBackendTypeByTier", in)
+ out := new(GetBackendTypeByTierResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) DeleteBucketLifecycle(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.DeleteBucketLifecycle", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) PutBucketLifecycle(ctx context.Context, in *PutBucketLifecycleRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.PutBucketLifecycle", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetBucketLifecycle(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*GetBucketLifecycleResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.GetBucketLifecycle", in)
+ out := new(GetBucketLifecycleResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) ListBucketLifecycle(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*ListBucketsResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.ListBucketLifecycle", in)
+ out := new(ListBucketsResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) UpdateBucket(ctx context.Context, in *Bucket, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.UpdateBucket", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) ListBucketUploadRecords(ctx context.Context, in *ListBucketUploadRequest, opts ...client.CallOption) (*ListBucketUploadResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.ListBucketUploadRecords", in)
+ out := new(ListBucketUploadResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) InitMultipartUpload(ctx context.Context, in *InitMultiPartRequest, opts ...client.CallOption) (*InitMultiPartResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.InitMultipartUpload", in)
+ out := new(InitMultiPartResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) AbortMultipartUpload(ctx context.Context, in *AbortMultipartRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.AbortMultipartUpload", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) CompleteMultipartUpload(ctx context.Context, in *CompleteMultipartRequest, opts ...client.CallOption) (*CompleteMultipartResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.CompleteMultipartUpload", in)
+ out := new(CompleteMultipartResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) UploadPart(ctx context.Context, opts ...client.CallOption) (S3_UploadPartService, error) {
+ req := c.c.NewRequest(c.name, "S3.UploadPart", &PutDataStream{})
+ stream, err := c.c.Stream(ctx, req, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return &s3ServiceUploadPart{stream}, nil
+}
+
+type S3_UploadPartService interface {
+ SendMsg(interface{}) error
+ RecvMsg(interface{}) error
+ Close() error
+ Send(*PutDataStream) error
+}
+
+type s3ServiceUploadPart struct {
+ stream client.Stream
+}
+
+func (x *s3ServiceUploadPart) Close() error {
+ return x.stream.Close()
+}
+
+func (x *s3ServiceUploadPart) SendMsg(m interface{}) error {
+ return x.stream.Send(m)
+}
+
+func (x *s3ServiceUploadPart) RecvMsg(m interface{}) error {
+ return x.stream.Recv(m)
+}
+
+func (x *s3ServiceUploadPart) Send(m *PutDataStream) error {
+ return x.stream.Send(m)
+}
+
+func (c *s3Service) ListObjectParts(ctx context.Context, in *ListObjectPartsRequest, opts ...client.CallOption) (*ListObjectPartsResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.ListObjectParts", in)
+ out := new(ListObjectPartsResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) AppendObject(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.AppendObject", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) PostObject(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.PostObject", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) AddUploadRecord(ctx context.Context, in *MultipartUploadRecord, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.AddUploadRecord", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) DeleteUploadRecord(ctx context.Context, in *MultipartUploadRecord, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.DeleteUploadRecord", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) HeadObject(ctx context.Context, in *BaseObjRequest, opts ...client.CallOption) (*Object, error) {
+ req := c.c.NewRequest(c.name, "S3.HeadObject", in)
+ out := new(Object)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) MoveObject(ctx context.Context, in *MoveObjectRequest, opts ...client.CallOption) (*MoveObjectResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.MoveObject", in)
+ out := new(MoveObjectResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) CopyObject(ctx context.Context, in *CopyObjectRequest, opts ...client.CallOption) (*CopyObjectResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.CopyObject", in)
+ out := new(CopyObjectResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) CopyObjPart(ctx context.Context, in *CopyObjPartRequest, opts ...client.CallOption) (*CopyObjPartResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.CopyObjPart", in)
+ out := new(CopyObjPartResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) PutObjACL(ctx context.Context, in *PutObjACLRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.PutObjACL", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetObjACL(ctx context.Context, in *BaseObjRequest, opts ...client.CallOption) (*ObjACL, error) {
+ req := c.c.NewRequest(c.name, "S3.GetObjACL", in)
+ out := new(ObjACL)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetBucketLocation(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.GetBucketLocation", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetBucketVersioning(ctx context.Context, in *BaseBucketRequest, opts ...client.CallOption) (*BucketVersioning, error) {
+ req := c.c.NewRequest(c.name, "S3.GetBucketVersioning", in)
+ out := new(BucketVersioning)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) PutBucketACL(ctx context.Context, in *PutBucketACLRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.PutBucketACL", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetBucketACL(ctx context.Context, in *BaseBucketRequest, opts ...client.CallOption) (*BucketACL, error) {
+ req := c.c.NewRequest(c.name, "S3.GetBucketACL", in)
+ out := new(BucketACL)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) PutBucketCORS(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.PutBucketCORS", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetBucketCORS(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.GetBucketCORS", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) DeleteBucketCORS(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.DeleteBucketCORS", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) PutBucketPolicy(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.PutBucketPolicy", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) GetBucketPolicy(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.GetBucketPolicy", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) DeleteBucketPolicy(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*BaseResponse, error) {
+ req := c.c.NewRequest(c.name, "S3.DeleteBucketPolicy", in)
+ out := new(BaseResponse)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *s3Service) HeadBucket(ctx context.Context, in *BaseRequest, opts ...client.CallOption) (*Bucket, error) {
+ req := c.c.NewRequest(c.name, "S3.HeadBucket", in)
+ out := new(Bucket)
+ err := c.c.Call(ctx, req, out, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+// Server API for S3 service
+
+type S3Handler interface {
+ ListBuckets(context.Context, *BaseRequest, *ListBucketsResponse) error
+ CreateBucket(context.Context, *Bucket, *BaseResponse) error
+ DeleteBucket(context.Context, *Bucket, *BaseResponse) error
+ GetBucket(context.Context, *Bucket, *GetBucketResponse) error
+ GetObjectMeta(context.Context, *Object, *GetObjectMetaResult) error
+ UpdateObjectMeta(context.Context, *Object, *PutObjectResponse) error
+ ListObjects(context.Context, *ListObjectsRequest, *ListObjectsResponse) error
+ CountObjects(context.Context, *ListObjectsRequest, *CountObjectsResponse) error
+ PutObject(context.Context, S3_PutObjectStream) error
+ UpdateObject(context.Context, *Object, *BaseResponse) error
+ GetObject(context.Context, *GetObjectInput, S3_GetObjectStream) error
+ DeleteObject(context.Context, *DeleteObjectInput, *DeleteObjectOutput) error
+ GetTierMap(context.Context, *BaseRequest, *GetTierMapResponse) error
+ UpdateObjMeta(context.Context, *UpdateObjMetaRequest, *BaseResponse) error
+ GetStorageClasses(context.Context, *BaseRequest, *GetStorageClassesResponse) error
+ GetBackendTypeByTier(context.Context, *GetBackendTypeByTierRequest, *GetBackendTypeByTierResponse) error
+ DeleteBucketLifecycle(context.Context, *BaseRequest, *BaseResponse) error
+ PutBucketLifecycle(context.Context, *PutBucketLifecycleRequest, *BaseResponse) error
+ GetBucketLifecycle(context.Context, *BaseRequest, *GetBucketLifecycleResponse) error
+ ListBucketLifecycle(context.Context, *BaseRequest, *ListBucketsResponse) error
+ UpdateBucket(context.Context, *Bucket, *BaseResponse) error
+ ListBucketUploadRecords(context.Context, *ListBucketUploadRequest, *ListBucketUploadResponse) error
+ InitMultipartUpload(context.Context, *InitMultiPartRequest, *InitMultiPartResponse) error
+ AbortMultipartUpload(context.Context, *AbortMultipartRequest, *BaseResponse) error
+ CompleteMultipartUpload(context.Context, *CompleteMultipartRequest, *CompleteMultipartResponse) error
+ UploadPart(context.Context, S3_UploadPartStream) error
+ ListObjectParts(context.Context, *ListObjectPartsRequest, *ListObjectPartsResponse) error
+ AppendObject(context.Context, *BaseRequest, *BaseResponse) error
+ PostObject(context.Context, *BaseRequest, *BaseResponse) error
+ // For lifecycle, may need some change.
+ AddUploadRecord(context.Context, *MultipartUploadRecord, *BaseResponse) error
+ DeleteUploadRecord(context.Context, *MultipartUploadRecord, *BaseResponse) error
+ HeadObject(context.Context, *BaseObjRequest, *Object) error
+ MoveObject(context.Context, *MoveObjectRequest, *MoveObjectResponse) error
+ CopyObject(context.Context, *CopyObjectRequest, *CopyObjectResponse) error
+ CopyObjPart(context.Context, *CopyObjPartRequest, *CopyObjPartResponse) error
+ PutObjACL(context.Context, *PutObjACLRequest, *BaseResponse) error
+ GetObjACL(context.Context, *BaseObjRequest, *ObjACL) error
+ GetBucketLocation(context.Context, *BaseRequest, *BaseResponse) error
+ GetBucketVersioning(context.Context, *BaseBucketRequest, *BucketVersioning) error
+ PutBucketACL(context.Context, *PutBucketACLRequest, *BaseResponse) error
+ GetBucketACL(context.Context, *BaseBucketRequest, *BucketACL) error
+ PutBucketCORS(context.Context, *BaseRequest, *BaseResponse) error
+ GetBucketCORS(context.Context, *BaseRequest, *BaseResponse) error
+ DeleteBucketCORS(context.Context, *BaseRequest, *BaseResponse) error
+ PutBucketPolicy(context.Context, *BaseRequest, *BaseResponse) error
+ GetBucketPolicy(context.Context, *BaseRequest, *BaseResponse) error
+ DeleteBucketPolicy(context.Context, *BaseRequest, *BaseResponse) error
+ HeadBucket(context.Context, *BaseRequest, *Bucket) error
+}
+
+func RegisterS3Handler(s server.Server, hdlr S3Handler, opts ...server.HandlerOption) error {
+ type s3 interface {
+ ListBuckets(ctx context.Context, in *BaseRequest, out *ListBucketsResponse) error
+ CreateBucket(ctx context.Context, in *Bucket, out *BaseResponse) error
+ DeleteBucket(ctx context.Context, in *Bucket, out *BaseResponse) error
+ GetBucket(ctx context.Context, in *Bucket, out *GetBucketResponse) error
+ GetObjectMeta(ctx context.Context, in *Object, out *GetObjectMetaResult) error
+ UpdateObjectMeta(ctx context.Context, in *Object, out *PutObjectResponse) error
+ ListObjects(ctx context.Context, in *ListObjectsRequest, out *ListObjectsResponse) error
+ CountObjects(ctx context.Context, in *ListObjectsRequest, out *CountObjectsResponse) error
+ PutObject(ctx context.Context, stream server.Stream) error
+ UpdateObject(ctx context.Context, in *Object, out *BaseResponse) error
+ GetObject(ctx context.Context, stream server.Stream) error
+ DeleteObject(ctx context.Context, in *DeleteObjectInput, out *DeleteObjectOutput) error
+ GetTierMap(ctx context.Context, in *BaseRequest, out *GetTierMapResponse) error
+ UpdateObjMeta(ctx context.Context, in *UpdateObjMetaRequest, out *BaseResponse) error
+ GetStorageClasses(ctx context.Context, in *BaseRequest, out *GetStorageClassesResponse) error
+ GetBackendTypeByTier(ctx context.Context, in *GetBackendTypeByTierRequest, out *GetBackendTypeByTierResponse) error
+ DeleteBucketLifecycle(ctx context.Context, in *BaseRequest, out *BaseResponse) error
+ PutBucketLifecycle(ctx context.Context, in *PutBucketLifecycleRequest, out *BaseResponse) error
+ GetBucketLifecycle(ctx context.Context, in *BaseRequest, out *GetBucketLifecycleResponse) error
+ ListBucketLifecycle(ctx context.Context, in *BaseRequest, out *ListBucketsResponse) error
+ UpdateBucket(ctx context.Context, in *Bucket, out *BaseResponse) error
+ ListBucketUploadRecords(ctx context.Context, in *ListBucketUploadRequest, out *ListBucketUploadResponse) error
+ InitMultipartUpload(ctx context.Context, in *InitMultiPartRequest, out *InitMultiPartResponse) error
+ AbortMultipartUpload(ctx context.Context, in *AbortMultipartRequest, out *BaseResponse) error
+ CompleteMultipartUpload(ctx context.Context, in *CompleteMultipartRequest, out *CompleteMultipartResponse) error
+ UploadPart(ctx context.Context, stream server.Stream) error
+ ListObjectParts(ctx context.Context, in *ListObjectPartsRequest, out *ListObjectPartsResponse) error
+ AppendObject(ctx context.Context, in *BaseRequest, out *BaseResponse) error
+ PostObject(ctx context.Context, in *BaseRequest, out *BaseResponse) error
+ AddUploadRecord(ctx context.Context, in *MultipartUploadRecord, out *BaseResponse) error
+ DeleteUploadRecord(ctx context.Context, in *MultipartUploadRecord, out *BaseResponse) error
+ HeadObject(ctx context.Context, in *BaseObjRequest, out *Object) error
+ MoveObject(ctx context.Context, in *MoveObjectRequest, out *MoveObjectResponse) error
+ CopyObject(ctx context.Context, in *CopyObjectRequest, out *CopyObjectResponse) error
+ CopyObjPart(ctx context.Context, in *CopyObjPartRequest, out *CopyObjPartResponse) error
+ PutObjACL(ctx context.Context, in *PutObjACLRequest, out *BaseResponse) error
+ GetObjACL(ctx context.Context, in *BaseObjRequest, out *ObjACL) error
+ GetBucketLocation(ctx context.Context, in *BaseRequest, out *BaseResponse) error
+ GetBucketVersioning(ctx context.Context, in *BaseBucketRequest, out *BucketVersioning) error
+ PutBucketACL(ctx context.Context, in *PutBucketACLRequest, out *BaseResponse) error
+ GetBucketACL(ctx context.Context, in *BaseBucketRequest, out *BucketACL) error
+ PutBucketCORS(ctx context.Context, in *BaseRequest, out *BaseResponse) error
+ GetBucketCORS(ctx context.Context, in *BaseRequest, out *BaseResponse) error
+ DeleteBucketCORS(ctx context.Context, in *BaseRequest, out *BaseResponse) error
+ PutBucketPolicy(ctx context.Context, in *BaseRequest, out *BaseResponse) error
+ GetBucketPolicy(ctx context.Context, in *BaseRequest, out *BaseResponse) error
+ DeleteBucketPolicy(ctx context.Context, in *BaseRequest, out *BaseResponse) error
+ HeadBucket(ctx context.Context, in *BaseRequest, out *Bucket) error
+ }
+ type S3 struct {
+ s3
+ }
+ h := &s3Handler{hdlr}
+ return s.Handle(s.NewHandler(&S3{h}, opts...))
+}
+
+type s3Handler struct {
+ S3Handler
+}
+
+func (h *s3Handler) ListBuckets(ctx context.Context, in *BaseRequest, out *ListBucketsResponse) error {
+ return h.S3Handler.ListBuckets(ctx, in, out)
+}
+
+func (h *s3Handler) CreateBucket(ctx context.Context, in *Bucket, out *BaseResponse) error {
+ return h.S3Handler.CreateBucket(ctx, in, out)
+}
+
+func (h *s3Handler) DeleteBucket(ctx context.Context, in *Bucket, out *BaseResponse) error {
+ return h.S3Handler.DeleteBucket(ctx, in, out)
+}
+
+func (h *s3Handler) GetBucket(ctx context.Context, in *Bucket, out *GetBucketResponse) error {
+ return h.S3Handler.GetBucket(ctx, in, out)
+}
+
+func (h *s3Handler) GetObjectMeta(ctx context.Context, in *Object, out *GetObjectMetaResult) error {
+ return h.S3Handler.GetObjectMeta(ctx, in, out)
+}
+
+func (h *s3Handler) UpdateObjectMeta(ctx context.Context, in *Object, out *PutObjectResponse) error {
+ return h.S3Handler.UpdateObjectMeta(ctx, in, out)
+}
+
+func (h *s3Handler) ListObjects(ctx context.Context, in *ListObjectsRequest, out *ListObjectsResponse) error {
+ return h.S3Handler.ListObjects(ctx, in, out)
+}
+
+func (h *s3Handler) CountObjects(ctx context.Context, in *ListObjectsRequest, out *CountObjectsResponse) error {
+ return h.S3Handler.CountObjects(ctx, in, out)
+}
+
+func (h *s3Handler) PutObject(ctx context.Context, stream server.Stream) error {
+ return h.S3Handler.PutObject(ctx, &s3PutObjectStream{stream})
+}
+
+type S3_PutObjectStream interface {
+ SendMsg(interface{}) error
+ RecvMsg(interface{}) error
+ Close() error
+ Recv() (*PutDataStream, error)
+}
+
+type s3PutObjectStream struct {
+ stream server.Stream
+}
+
+func (x *s3PutObjectStream) Close() error {
+ return x.stream.Close()
+}
+
+func (x *s3PutObjectStream) SendMsg(m interface{}) error {
+ return x.stream.Send(m)
+}
+
+func (x *s3PutObjectStream) RecvMsg(m interface{}) error {
+ return x.stream.Recv(m)
+}
+
+func (x *s3PutObjectStream) Recv() (*PutDataStream, error) {
+ m := new(PutDataStream)
+ if err := x.stream.Recv(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+func (h *s3Handler) UpdateObject(ctx context.Context, in *Object, out *BaseResponse) error {
+ return h.S3Handler.UpdateObject(ctx, in, out)
+}
+
+func (h *s3Handler) GetObject(ctx context.Context, stream server.Stream) error {
+ m := new(GetObjectInput)
+ if err := stream.Recv(m); err != nil {
+ return err
+ }
+ return h.S3Handler.GetObject(ctx, m, &s3GetObjectStream{stream})
+}
+
+type S3_GetObjectStream interface {
+ SendMsg(interface{}) error
+ RecvMsg(interface{}) error
+ Close() error
+ Send(*GetObjectResponse) error
+}
+
+type s3GetObjectStream struct {
+ stream server.Stream
+}
+
+func (x *s3GetObjectStream) Close() error {
+ return x.stream.Close()
+}
+
+func (x *s3GetObjectStream) SendMsg(m interface{}) error {
+ return x.stream.Send(m)
+}
+
+func (x *s3GetObjectStream) RecvMsg(m interface{}) error {
+ return x.stream.Recv(m)
+}
+
+func (x *s3GetObjectStream) Send(m *GetObjectResponse) error {
+ return x.stream.Send(m)
+}
+
+func (h *s3Handler) DeleteObject(ctx context.Context, in *DeleteObjectInput, out *DeleteObjectOutput) error {
+ return h.S3Handler.DeleteObject(ctx, in, out)
+}
+
+func (h *s3Handler) GetTierMap(ctx context.Context, in *BaseRequest, out *GetTierMapResponse) error {
+ return h.S3Handler.GetTierMap(ctx, in, out)
+}
+
+func (h *s3Handler) UpdateObjMeta(ctx context.Context, in *UpdateObjMetaRequest, out *BaseResponse) error {
+ return h.S3Handler.UpdateObjMeta(ctx, in, out)
+}
+
+func (h *s3Handler) GetStorageClasses(ctx context.Context, in *BaseRequest, out *GetStorageClassesResponse) error {
+ return h.S3Handler.GetStorageClasses(ctx, in, out)
+}
+
+func (h *s3Handler) GetBackendTypeByTier(ctx context.Context, in *GetBackendTypeByTierRequest, out *GetBackendTypeByTierResponse) error {
+ return h.S3Handler.GetBackendTypeByTier(ctx, in, out)
+}
+
+func (h *s3Handler) DeleteBucketLifecycle(ctx context.Context, in *BaseRequest, out *BaseResponse) error {
+ return h.S3Handler.DeleteBucketLifecycle(ctx, in, out)
+}
+
+func (h *s3Handler) PutBucketLifecycle(ctx context.Context, in *PutBucketLifecycleRequest, out *BaseResponse) error {
+ return h.S3Handler.PutBucketLifecycle(ctx, in, out)
+}
+
+func (h *s3Handler) GetBucketLifecycle(ctx context.Context, in *BaseRequest, out *GetBucketLifecycleResponse) error {
+ return h.S3Handler.GetBucketLifecycle(ctx, in, out)
+}
+
+func (h *s3Handler) ListBucketLifecycle(ctx context.Context, in *BaseRequest, out *ListBucketsResponse) error {
+ return h.S3Handler.ListBucketLifecycle(ctx, in, out)
+}
+
+func (h *s3Handler) UpdateBucket(ctx context.Context, in *Bucket, out *BaseResponse) error {
+ return h.S3Handler.UpdateBucket(ctx, in, out)
+}
+
+func (h *s3Handler) ListBucketUploadRecords(ctx context.Context, in *ListBucketUploadRequest, out *ListBucketUploadResponse) error {
+ return h.S3Handler.ListBucketUploadRecords(ctx, in, out)
+}
+
+func (h *s3Handler) InitMultipartUpload(ctx context.Context, in *InitMultiPartRequest, out *InitMultiPartResponse) error {
+ return h.S3Handler.InitMultipartUpload(ctx, in, out)
+}
+
+func (h *s3Handler) AbortMultipartUpload(ctx context.Context, in *AbortMultipartRequest, out *BaseResponse) error {
+ return h.S3Handler.AbortMultipartUpload(ctx, in, out)
+}
+
+func (h *s3Handler) CompleteMultipartUpload(ctx context.Context, in *CompleteMultipartRequest, out *CompleteMultipartResponse) error {
+ return h.S3Handler.CompleteMultipartUpload(ctx, in, out)
+}
+
+func (h *s3Handler) UploadPart(ctx context.Context, stream server.Stream) error {
+ return h.S3Handler.UploadPart(ctx, &s3UploadPartStream{stream})
+}
+
+type S3_UploadPartStream interface {
+ SendMsg(interface{}) error
+ RecvMsg(interface{}) error
+ Close() error
+ Recv() (*PutDataStream, error)
+}
+
+type s3UploadPartStream struct {
+ stream server.Stream
+}
+
+func (x *s3UploadPartStream) Close() error {
+ return x.stream.Close()
+}
+
+func (x *s3UploadPartStream) SendMsg(m interface{}) error {
+ return x.stream.Send(m)
+}
+
+func (x *s3UploadPartStream) RecvMsg(m interface{}) error {
+ return x.stream.Recv(m)
+}
+
+func (x *s3UploadPartStream) Recv() (*PutDataStream, error) {
+ m := new(PutDataStream)
+ if err := x.stream.Recv(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+func (h *s3Handler) ListObjectParts(ctx context.Context, in *ListObjectPartsRequest, out *ListObjectPartsResponse) error {
+ return h.S3Handler.ListObjectParts(ctx, in, out)
+}
+
+func (h *s3Handler) AppendObject(ctx context.Context, in *BaseRequest, out *BaseResponse) error {
+ return h.S3Handler.AppendObject(ctx, in, out)
+}
+
+func (h *s3Handler) PostObject(ctx context.Context, in *BaseRequest, out *BaseResponse) error {
+ return h.S3Handler.PostObject(ctx, in, out)
+}
+
+func (h *s3Handler) AddUploadRecord(ctx context.Context, in *MultipartUploadRecord, out *BaseResponse) error {
+ return h.S3Handler.AddUploadRecord(ctx, in, out)
+}
+
+func (h *s3Handler) DeleteUploadRecord(ctx context.Context, in *MultipartUploadRecord, out *BaseResponse) error {
+ return h.S3Handler.DeleteUploadRecord(ctx, in, out)
+}
+
+func (h *s3Handler) HeadObject(ctx context.Context, in *BaseObjRequest, out *Object) error {
+ return h.S3Handler.HeadObject(ctx, in, out)
+}
+
+func (h *s3Handler) MoveObject(ctx context.Context, in *MoveObjectRequest, out *MoveObjectResponse) error {
+ return h.S3Handler.MoveObject(ctx, in, out)
+}
+
+func (h *s3Handler) CopyObject(ctx context.Context, in *CopyObjectRequest, out *CopyObjectResponse) error {
+ return h.S3Handler.CopyObject(ctx, in, out)
+}
+
+func (h *s3Handler) CopyObjPart(ctx context.Context, in *CopyObjPartRequest, out *CopyObjPartResponse) error {
+ return h.S3Handler.CopyObjPart(ctx, in, out)
+}
+
+func (h *s3Handler) PutObjACL(ctx context.Context, in *PutObjACLRequest, out *BaseResponse) error {
+ return h.S3Handler.PutObjACL(ctx, in, out)
+}
+
+func (h *s3Handler) GetObjACL(ctx context.Context, in *BaseObjRequest, out *ObjACL) error {
+ return h.S3Handler.GetObjACL(ctx, in, out)
+}
+
+func (h *s3Handler) GetBucketLocation(ctx context.Context, in *BaseRequest, out *BaseResponse) error {
+ return h.S3Handler.GetBucketLocation(ctx, in, out)
+}
+
+func (h *s3Handler) GetBucketVersioning(ctx context.Context, in *BaseBucketRequest, out *BucketVersioning) error {
+ return h.S3Handler.GetBucketVersioning(ctx, in, out)
+}
+
+func (h *s3Handler) PutBucketACL(ctx context.Context, in *PutBucketACLRequest, out *BaseResponse) error {
+ return h.S3Handler.PutBucketACL(ctx, in, out)
+}
+
+func (h *s3Handler) GetBucketACL(ctx context.Context, in *BaseBucketRequest, out *BucketACL) error {
+ return h.S3Handler.GetBucketACL(ctx, in, out)
+}
+
+func (h *s3Handler) PutBucketCORS(ctx context.Context, in *BaseRequest, out *BaseResponse) error {
+ return h.S3Handler.PutBucketCORS(ctx, in, out)
+}
+
+func (h *s3Handler) GetBucketCORS(ctx context.Context, in *BaseRequest, out *BaseResponse) error {
+ return h.S3Handler.GetBucketCORS(ctx, in, out)
+}
+
+func (h *s3Handler) DeleteBucketCORS(ctx context.Context, in *BaseRequest, out *BaseResponse) error {
+ return h.S3Handler.DeleteBucketCORS(ctx, in, out)
+}
+
+func (h *s3Handler) PutBucketPolicy(ctx context.Context, in *BaseRequest, out *BaseResponse) error {
+ return h.S3Handler.PutBucketPolicy(ctx, in, out)
+}
+
+func (h *s3Handler) GetBucketPolicy(ctx context.Context, in *BaseRequest, out *BaseResponse) error {
+ return h.S3Handler.GetBucketPolicy(ctx, in, out)
+}
+
+func (h *s3Handler) DeleteBucketPolicy(ctx context.Context, in *BaseRequest, out *BaseResponse) error {
+ return h.S3Handler.DeleteBucketPolicy(ctx, in, out)
+}
+
+func (h *s3Handler) HeadBucket(ctx context.Context, in *BaseRequest, out *Bucket) error {
+ return h.S3Handler.HeadBucket(ctx, in, out)
+}
+
+
+
-[Release notes](https://github.com/opensds/multi-cloud/releases/tag/v0.4.0)
+## Introduction
+
+SODA Multi-cloud project provides a cloud vendor agnostic data management for hybrid cloud, intercloud or intracloud. It provides an s3-compatible interface. It can be hosted on prem or cloud native.
+
+It provides a backend manager which is S3 compatible to connect with any cloud vendors. It supports various cloud backends like MS Azure, GCP, AWS, Huawei, IBM and more. It also supports Ceph backed to enable on-prem. We have also integrated some optimizations and also YIG-Ceph backend from China Unicom YIG project.
+
+Currently it provides Object Storage and we are working to support file and block services from the cloud vendors.
+
+This is one of the SODA Core Projects and is maintained by SODA Foundation directly.
+
+## Documentation
+
+[https://docs.sodafoundation.io](https://docs.sodafoundation.io/)
+
+## Quick Start - To Use/Experience
+
+[https://docs.sodafoundation.io](https://docs.sodafoundation.io/)
+
+## Quick Start - To Develop
+
+[https://docs.sodafoundation.io](https://docs.sodafoundation.io/)
+
+## Latest Releases
+
+[https://github.com/sodafoundation/multi-cloud/releases](https://github.com/sodafoundation/multi-cloud/releases)
+
+## Support and Issues
+
+[https://github.com/sodafoundation/multi-cloud/issues](https://github.com/sodafoundation/multi-cloud/issues)
+
+## Project Community
+
+[https://sodafoundation.io/slack/](https://sodafoundation.io/slack/)
+
+## How to contribute to this project?
+
+Join [https://sodafoundation.io/slack/](https://sodafoundation.io/slack/) and share your interest in the ‘general’ channel
+
+Checkout [https://github.com/sodafoundation/multi-cloud/issues](https://github.com/sodafoundation/multi-cloud/issues) labelled with ‘good first issue’ or ‘help needed’ or ‘help wanted’ or ‘StartMyContribution’ or ‘SMC’
+
+## Project Roadmap
+
+We envision to build a SODA Distributed Data Store (SODA DDS) which can support File, Block and Object across Edge, On-prem and Cloud. We are exploring to transform or integrate SODA Multicloud to other SODA Projects to build SODA DDS.
+
+[https://docs.sodafoundation.io](https://docs.sodafoundation.io/)
+
+## Join SODA Foundation
+
+Website : [https://sodafoundation.io](https://sodafoundation.io/)
+
+Slack : [https://sodafoundation.io/slack/](https://sodafoundation.io/slack/)
+
+Twitter : @sodafoundation
+
+Mailinglist : [https://lists.opensds.io](https://lists.opensds.io/groups)
diff --git a/api/Autobuildfile b/api/Autobuildfile
new file mode 100644
index 000000000..1c0171e4c
--- /dev/null
+++ b/api/Autobuildfile
@@ -0,0 +1,6 @@
+FROM alpine:latest
+
+RUN mkdir -p /etc/opensds
+COPY api/pkg/examples/policy.json /etc/opensds/
+COPY api/api /api
+ENTRYPOINT ["/api"]
diff --git a/api/Dockerfile b/api/Dockerfile
index ab4d04f89..c4ad8ea4d 100644
--- a/api/Dockerfile
+++ b/api/Dockerfile
@@ -1,6 +1,6 @@
FROM alpine:latest
-RUN mkdir -p /etc/opensds
-COPY ./pkg/examples/policy.json /etc/opensds/
+RUN mkdir -p /etc/multicloud
+COPY ./pkg/examples/policy.json /etc/multicloud/
COPY api /api
ENTRYPOINT ["/api"]
diff --git a/api/cmd/main.go b/api/cmd/main.go
index 33715960d..26df169c2 100644
--- a/api/cmd/main.go
+++ b/api/cmd/main.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2018 Huawei Technologies Co., Ltd. All Rights Reserved.
+// Copyright 2019 The OpenSDS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -15,18 +15,19 @@
package main
import (
+ "os"
+
"github.com/emicklei/go-restful"
- "github.com/micro/go-log"
- "github.com/micro/go-web"
+ "github.com/micro/go-micro/v2/web"
"github.com/opensds/multi-cloud/api/pkg/backend"
"github.com/opensds/multi-cloud/api/pkg/dataflow"
- "github.com/opensds/multi-cloud/api/pkg/filters/context"
- "github.com/opensds/multi-cloud/api/pkg/filters/signature/signer"
-
- // _ "github.com/micro/go-plugins/client/grpc"
"github.com/opensds/multi-cloud/api/pkg/filters/auth"
+ "github.com/opensds/multi-cloud/api/pkg/filters/context"
"github.com/opensds/multi-cloud/api/pkg/filters/logging"
+ "github.com/opensds/multi-cloud/api/pkg/filters/signature/signer"
"github.com/opensds/multi-cloud/api/pkg/s3"
+ "github.com/opensds/multi-cloud/api/pkg/utils/obs"
+ log "github.com/sirupsen/logrus"
)
const (
@@ -40,32 +41,38 @@ func main() {
)
webService.Init()
+ obs.InitLogs()
wc := restful.NewContainer()
- ws := new(restful.WebService)
- ws.Path("/v1")
- ws.Doc("OpenSDS Multi-Cloud API")
- ws.Consumes(restful.MIME_JSON)
- ws.Produces(restful.MIME_JSON)
- ws.Filter(auth.FilterFactory())
- backend.RegisterRouter(ws)
- dataflow.RegisterRouter(ws)
- // add filter for authentication context
- ws.Filter(logging.FilterFactory())
- ws.Filter(context.FilterFactory())
- ws.Filter(auth.FilterFactory())
-
- s3ws := new(restful.WebService)
- s3ws.Path("/v1/s3")
- s3ws.Doc("OpenSDS Multi-Cloud API")
- s3ws.Consumes(restful.MIME_XML)
- s3ws.Produces(restful.MIME_XML)
- s3ws.Filter(logging.FilterFactory())
- s3ws.Filter(signer.FilterFactory())
- s3.RegisterRouter(s3ws)
+ flag := os.Getenv("SVC_FLAG")
+ if flag == "s3" {
+ s3ws := new(restful.WebService)
+ s3ws.Path("/")
+ s3ws.Doc("OpenSDS Multi-Cloud API")
+ s3ws.Produces(restful.MIME_XML)
+
+ s3ws.Filter(logging.FilterFactory())
+ s3ws.Filter(context.FilterFactory())
+ s3ws.Filter(signer.FilterFactory())
+ s3.RegisterRouter(s3ws)
+
+ wc.Add(s3ws)
+ } else {
+ ws := new(restful.WebService)
+ ws.Path("/v1")
+ ws.Doc("OpenSDS Multi-Cloud API")
+ ws.Consumes(restful.MIME_JSON)
+ ws.Produces(restful.MIME_JSON)
+
+ backend.RegisterRouter(ws)
+ dataflow.RegisterRouter(ws)
+ // add filter for authentication context
+ ws.Filter(logging.FilterFactory())
+ ws.Filter(context.FilterFactory())
+ ws.Filter(auth.FilterFactory())
+ wc.Add(ws)
+ }
- wc.Add(ws)
- wc.Add(s3ws)
webService.Handle("/", wc)
if err := webService.Run(); err != nil {
log.Fatal(err)
diff --git a/api/cmd/main_test.go b/api/cmd/main_test.go
new file mode 100644
index 000000000..8842639e8
--- /dev/null
+++ b/api/cmd/main_test.go
@@ -0,0 +1,31 @@
+package main
+
+import (
+ "os"
+ "strings"
+ "testing"
+)
+
+func Test_Main(t *testing.T) {
+ var (
+ args []string
+ //run bool
+ run = true
+ )
+
+ for _, arg := range os.Args {
+ switch {
+ case arg == "__DEVEL--i-heard-you-like-tests":
+ run = true
+ case strings.HasPrefix(arg, "-test"):
+ case strings.HasPrefix(arg, "__DEVEL"):
+ default:
+ args = append(args, arg)
+ }
+ }
+ os.Args = args
+
+ if run {
+ main()
+ }
+}
diff --git a/api/cmd/totalcov.html b/api/cmd/totalcov.html
new file mode 100644
index 000000000..b0d44a232
--- /dev/null
+++ b/api/cmd/totalcov.html
@@ -0,0 +1,24546 @@
+
+
+
+
+
+
+
+
+