From bb8fb0612fdbf55123296759f04f09aa1e70f681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=9E=9A=E5=A5=87?= Date: Fri, 27 Mar 2026 10:47:20 +0800 Subject: [PATCH] fix: vender update master --- go.mod | 2 +- go.sum | 4 +- vendor/modules.txt | 2 +- .../x/onecloud/pkg/apis/compute/api.go | 7 + .../yunion.io/x/onecloud/pkg/appsrv/send.go | 14 +- .../onecloud/pkg/compute/models/cloudsync.go | 2 +- .../onecloud/pkg/compute/models/containers.go | 2 +- .../pkg/compute/models/guest_queries.go | 2 +- .../x/onecloud/pkg/compute/models/guests.go | 4 +- .../pkg/compute/models/instance_snapshots.go | 12 +- .../pkg/hostman/metadata/metadatahandler.go | 3 + .../pkg/hostman/metadata/telegraf_influx.go | 258 ++++++++++++++++++ 12 files changed, 293 insertions(+), 19 deletions(-) create mode 100644 vendor/yunion.io/x/onecloud/pkg/hostman/metadata/telegraf_influx.go diff --git a/go.mod b/go.mod index a2ed92921..bf9b44f92 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( google.golang.org/protobuf v1.35.1 yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1 yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 - yunion.io/x/onecloud v0.0.0-20260319173629-70fbfe89465a + yunion.io/x/onecloud v0.0.0-20260326134600-7756be9d3800 yunion.io/x/pkg v1.10.4-0.20260224071200-5c269fa62429 ) diff --git a/go.sum b/go.sum index 9bd0af92d..520497529 100644 --- a/go.sum +++ b/go.sum @@ -741,8 +741,8 @@ yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXce yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U= yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 h1:inY5o3LDa/zgsIZuPN0HmpzKIsu/lLgsBmMttuDPGj4= yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U= -yunion.io/x/onecloud v0.0.0-20260319173629-70fbfe89465a h1:xgO6AQetVN9L5JCztzwygXTMHGzHmHubjvW46oYqJr8= -yunion.io/x/onecloud v0.0.0-20260319173629-70fbfe89465a/go.mod h1:/WPM4NY7B4Iz4YLIRQdTk3a1281u4aIl4NpzyBsWxTg= +yunion.io/x/onecloud v0.0.0-20260326134600-7756be9d3800 h1:l78Bc/Bmz0nws8b447miHNx5ZJiDOyhEa0G+g4cArK4= +yunion.io/x/onecloud v0.0.0-20260326134600-7756be9d3800/go.mod h1:/WPM4NY7B4Iz4YLIRQdTk3a1281u4aIl4NpzyBsWxTg= yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v1.10.4-0.20260224071200-5c269fa62429 h1:yskc2HFnBZXEzfYeQBFdL2gXzVfdPQ8yT4qVDyuhajM= diff --git a/vendor/modules.txt b/vendor/modules.txt index 7effddd52..c6474932c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -896,7 +896,7 @@ yunion.io/x/jsonutils ## explicit; go 1.12 yunion.io/x/log yunion.io/x/log/hooks -# yunion.io/x/onecloud v0.0.0-20260319173629-70fbfe89465a +# yunion.io/x/onecloud v0.0.0-20260326134600-7756be9d3800 ## explicit; go 1.24 yunion.io/x/onecloud/locales yunion.io/x/onecloud/pkg/apihelper diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/api.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/api.go index ce7c5cd15..3f9cd1601 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/api.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/api.go @@ -727,6 +727,13 @@ type ServerCreateInput struct { Pod *PodCreateInput `json:"pod"` } +func (c *KickstartConfig) IsEnabled() bool { + if c.Enabled != nil && !*c.Enabled { + return false + } + return true +} + // ServerUpdateKickstartStatusInput 更新虚拟机 kickstart 状态的输入 type ServerUpdateKickstartStatusInput struct { // kickstart 状态 diff --git a/vendor/yunion.io/x/onecloud/pkg/appsrv/send.go b/vendor/yunion.io/x/onecloud/pkg/appsrv/send.go index d0b0c519b..4d2ee1945 100644 --- a/vendor/yunion.io/x/onecloud/pkg/appsrv/send.go +++ b/vendor/yunion.io/x/onecloud/pkg/appsrv/send.go @@ -111,13 +111,6 @@ func SendXmlWithIndent(w http.ResponseWriter, hdr http.Header, obj interface{}, func SendStream(w http.ResponseWriter, isPartial bool, hdr http.Header, stream io.ReadCloser, sizeBytes int64) error { defer stream.Close() - if isPartial { - log.Debugf("send partial 206") - w.WriteHeader(206) - } else { - log.Debugf("send full 200") - w.WriteHeader(200) - } for k, v := range hdr { if k != "Content-Length" { log.Debugf("send %s %s", k, v) @@ -128,6 +121,13 @@ func SendStream(w http.ResponseWriter, isPartial bool, hdr http.Header, stream i log.Debugf("send content-length %d", sizeBytes) w.Header().Set("Content-Length", strconv.FormatInt(sizeBytes, 10)) } + if isPartial { + log.Debugf("send partial 206") + w.WriteHeader(206) + } else { + log.Debugf("send full 200") + w.WriteHeader(200) + } offset := 0 buf := make([]byte, 4096) for sizeBytes <= 0 || int64(offset) < sizeBytes { diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudsync.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudsync.go index 1bd6fe1df..88078cf87 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudsync.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudsync.go @@ -1335,7 +1335,7 @@ func (self *SGuest) SyncVMIsolateDevices(ctx context.Context, userCred mcclient. } func (guest *SGuest) GetContainers() ([]SContainer, error) { - q := GetContainerManager().Query().Equals("guest_id", guest.Id) + q := GetContainerManager().Query().Equals("guest_id", guest.Id).Asc("created_at") ret := []SContainer{} err := db.FetchModelObjects(GetContainerManager(), q, &ret) if err != nil { diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/containers.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/containers.go index 19cf02edd..3b37ed67a 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/containers.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/containers.go @@ -142,7 +142,7 @@ func (m *SContainerManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQue } func (m *SContainerManager) GetContainersByPod(guestId string) ([]SContainer, error) { - q := m.Query().Equals("guest_id", guestId) + q := m.Query().Equals("guest_id", guestId).Asc("created_at") ctrs := make([]SContainer, 0) if err := db.FetchModelObjects(m, q, &ctrs); err != nil { return nil, errors.Wrap(err, "db.FetchModelObjects") diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_queries.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_queries.go index ab062d9c5..f40310669 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_queries.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_queries.go @@ -928,7 +928,7 @@ func fetchGuestBackupInfo(hostIds []string) (map[string]api.BackupInfo, error) { func fetchContainers(guestIds []string) (map[string][]*api.PodContainerDesc, error) { ret := map[string][]*api.PodContainerDesc{} containers := []SContainer{} - err := GetContainerManager().Query().In("guest_id", guestIds).All(&containers) + err := GetContainerManager().Query().In("guest_id", guestIds).Asc("created_at").All(&containers) if err != nil { return nil, err } diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guests.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guests.go index 41b8ba091..4a9945c59 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/guests.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guests.go @@ -2300,7 +2300,7 @@ func (manager *SGuestManager) validateCreateData( } // validate KickstartConfig - if input.KickstartConfig != nil { + if input.KickstartConfig != nil && input.KickstartConfig.IsEnabled() { if err := validateKickstartConfig(input.KickstartConfig); err != nil { return nil, httperrors.NewInputParameterError("Invalid kickstart config: %v", err) } @@ -2807,7 +2807,7 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred kickstartConfig := &api.KickstartConfig{} if err := kickstartConfigJson.Unmarshal(kickstartConfig); err != nil { log.Errorf("unmarshal kickstart config fail: %s", err) - } else { + } else if kickstartConfig.IsEnabled() { if err := guest.SetKickstartConfig(ctx, kickstartConfig, userCred); err != nil { log.Errorf("Failed to set kickstart config for guest %s: %v", guest.Name, err) } else { diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_snapshots.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_snapshots.go index 8d27a2aba..db1c32cd0 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_snapshots.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_snapshots.go @@ -635,10 +635,16 @@ func (self *SInstanceSnapshot) PerformPurge(ctx context.Context, userCred mcclie isjp := new(SInstanceSnapshotJoint) err = InstanceSnapshotJointManager.Query(). Equals("instance_snapshot_id", self.Id).Equals("snapshot_id", snapshotId).First(isjp) - err = isjp.Delete(ctx, userCred) - if err != nil { - return nil, errors.Wrapf(err, "delete instance snapshot joint: %s", snapshotId) + if err == nil || isjp != nil { + isjp.SetModelManager(InstanceSnapshotJointManager, isjp) + err = isjp.Delete(ctx, userCred) + if err != nil { + return nil, errors.Wrapf(err, "delete instance snapshot joint: %s", snapshotId) + } + } else { + log.Errorf("failed get instance_snapshot %s join %s: %s", self.Id, snapshotId, err) } + _, err = snapshots[i].PerformPurge(ctx, userCred, query, data) if err != nil { return nil, errors.Wrapf(err, "delete snapshot: %s", snapshotId) diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/metadata/metadatahandler.go b/vendor/yunion.io/x/onecloud/pkg/hostman/metadata/metadatahandler.go index d49ee8081..667807e1b 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/metadata/metadatahandler.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/metadata/metadatahandler.go @@ -343,6 +343,9 @@ func (s *Service) monitorReverseEndpoint() *proxy.SEndpointFactory { } func (s *Service) requestManipulator(ctx context.Context, r *http.Request) (*http.Request, error) { + if err := s.rewriteTelegrafInfluxBodyIfNeeded(ctx, r); err != nil { + log.Errorf("failed rewrite telegraf body %s", err) + } path := r.URL.Path[len(s.monitorPrefix()):] log.Debugf("Path: %s => %s", r.URL.Path, path) r.URL = &url.URL{ diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/metadata/telegraf_influx.go b/vendor/yunion.io/x/onecloud/pkg/hostman/metadata/telegraf_influx.go new file mode 100644 index 000000000..7088baec9 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/metadata/telegraf_influx.go @@ -0,0 +1,258 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific + +package metadata + +import ( + "bytes" + "compress/gzip" + "context" + "io" + "net/http" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/hostman/guestman/desc" +) + +const telegrafInfluxMaxBodyBytes = 16 << 20 + +func (s *Service) rewriteTelegrafInfluxBodyIfNeeded(ctx context.Context, r *http.Request) error { + prefix := s.monitorPrefix() + if !strings.HasPrefix(r.URL.Path, prefix) { + return nil + } + sub := r.URL.Path[len(prefix):] + if r.Method != http.MethodPost && r.Method != http.MethodPut { + return nil + } + if sub != "/write" && !strings.HasPrefix(sub, "/write?") { + return nil + } + + body, err := io.ReadAll(io.LimitReader(r.Body, telegrafInfluxMaxBodyBytes+1)) + if err != nil { + return errors.Wrap(err, "read telegraf influx body") + } + if len(body) > telegrafInfluxMaxBodyBytes { + return errors.Errorf("telegraf influx body exceeds %d bytes", telegrafInfluxMaxBodyBytes) + } + _ = r.Body.Close() + + if strings.Contains(strings.ToLower(r.Header.Get("Content-Encoding")), "gzip") { + gr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return errors.Wrap(err, "gzip reader") + } + body, err = io.ReadAll(io.LimitReader(gr, telegrafInfluxMaxBodyBytes+1)) + _ = gr.Close() + if err != nil { + return errors.Wrap(err, "read gzipped telegraf body") + } + if len(body) > telegrafInfluxMaxBodyBytes { + return errors.Errorf("telegraf influx body exceeds %d bytes after gzip", telegrafInfluxMaxBodyBytes) + } + r.Header.Del("Content-Encoding") + } + + newBody, changed, err := rewriteInfluxLineProtocolTenant(body, func(vmId string) (string, bool) { + gd := s.lookupGuestDescForTelegraf(r, vmId) + if gd == nil || gd.TenantId == "" { + return "", false + } + return gd.TenantId, true + }) + if err != nil { + return err + } + if changed { + log.Debugf("metadata monitor: corrected tenant_id in telegraf influx payload from %s", r.RemoteAddr) + } + r.Body = io.NopCloser(bytes.NewReader(newBody)) + r.ContentLength = int64(len(newBody)) + r.Header.Del("Content-Length") + return nil +} + +func (s *Service) lookupGuestDescForTelegraf(r *http.Request, vmId string) *desc.SGuestDesc { + if vmId == "" { + return nil + } + gd := s.getGuestDesc(r) + if gd != nil && gd.Uuid == vmId { + return gd + } + return nil +} + +func rewriteInfluxLineProtocolTenant(body []byte, resolveTenant func(vmId string) (tenantId string, ok bool)) ([]byte, bool, error) { + raw := strings.Split(string(body), "\n") + changed := false + for i, line := range raw { + line = strings.TrimRight(line, "\r") + if line == "" || strings.HasPrefix(line, "#") { + continue + } + newLine, lineChanged, err := rewriteInfluxLineTenant(line, resolveTenant) + if err != nil { + return body, false, err + } + if lineChanged { + changed = true + raw[i] = newLine + } + } + if !changed { + return body, false, nil + } + return []byte(strings.Join(raw, "\n")), true, nil +} + +func rewriteInfluxLineTenant(line string, resolveTenant func(vmId string) (tenantId string, ok bool)) (string, bool, error) { + measTags, fields, ok := splitMeasurementTagsAndFields(line) + if !ok { + return line, false, nil + } + parts := splitOnUnescapedComma(measTags) + if len(parts) < 1 { + return line, false, nil + } + measurement := parts[0] + tagSegs := parts[1:] + vmId := "" + haveTenant := false + curTenant := "" + for _, seg := range tagSegs { + k, v := splitInfluxTagKeyValue(seg) + if k == "" { + continue + } + switch k { + case "vm_id": + vmId = v + case "tenant_id": + haveTenant = true + curTenant = v + } + } + if vmId == "" { + return line, false, nil + } + expectTenant, ok := resolveTenant(vmId) + if !ok { + return line, false, nil + } + if haveTenant && curTenant == expectTenant { + return line, false, nil + } + newSegs := make([]string, 0, len(tagSegs)+1) + for _, seg := range tagSegs { + k, _ := splitInfluxTagKeyValue(seg) + if k == "tenant_id" { + newSegs = append(newSegs, "tenant_id="+expectTenant) + } else { + newSegs = append(newSegs, seg) + } + } + if !haveTenant { + newSegs = append(newSegs, "tenant_id="+expectTenant) + } + var b strings.Builder + b.WriteString(measurement) + for _, seg := range newSegs { + b.WriteByte(',') + b.WriteString(seg) + } + b.WriteByte(' ') + b.WriteString(fields) + return b.String(), true, nil +} + +func splitMeasurementTagsAndFields(line string) (measTags string, fields string, ok bool) { + for i := 0; i < len(line); i++ { + if line[i] == ' ' && !influxByteEscaped(line, i) { + return line[:i], line[i+1:], true + } + } + return "", "", false +} + +func influxByteEscaped(line string, i int) bool { + if i == 0 { + return false + } + n := 0 + for j := i - 1; j >= 0 && line[j] == '\\'; j-- { + n++ + } + return n%2 == 1 +} + +func splitOnUnescapedComma(s string) []string { + var out []string + var b strings.Builder + escaped := false + for i := 0; i < len(s); i++ { + c := s[i] + if escaped { + b.WriteByte(c) + escaped = false + continue + } + if c == '\\' { + escaped = true + b.WriteByte('\\') + continue + } + if c == ',' { + out = append(out, b.String()) + b.Reset() + continue + } + b.WriteByte(c) + } + out = append(out, b.String()) + return out +} + +func splitInfluxTagKeyValue(seg string) (key, val string) { + for i := 0; i < len(seg); i++ { + if seg[i] == '=' && !influxByteEscaped(seg, i) { + return influxUnescapeTagKey(seg[:i]), seg[i+1:] + } + } + return "", "" +} + +func influxUnescapeTagKey(s string) string { + return influxUnescapeTag(s) +} + +func influxUnescapeTag(s string) string { + var b strings.Builder + b.Grow(len(s)) + for i := 0; i < len(s); i++ { + if s[i] == '\\' && i+1 < len(s) { + switch s[i+1] { + case '\\', ' ', ',', '=': + b.WriteByte(s[i+1]) + i++ + continue + } + } + b.WriteByte(s[i]) + } + return b.String() +}