diff --git a/pkg/api/jobrunevents/job_run_events.go b/pkg/api/jobrunevents/job_run_events.go new file mode 100644 index 0000000000..c77c766b0d --- /dev/null +++ b/pkg/api/jobrunevents/job_run_events.go @@ -0,0 +1,146 @@ +package jobrunevents + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "strings" + + "cloud.google.com/go/storage" + log "github.com/sirupsen/logrus" + + "github.com/openshift/sippy/pkg/api" + "github.com/openshift/sippy/pkg/dataloader/prowloader/gcs" + "github.com/openshift/sippy/pkg/db" +) + +// eventsJSONRegex matches paths like artifacts/*e2e*/gather-extra/artifacts/events.json +var eventsJSONRegex = regexp.MustCompile(`gather-extra/artifacts/events\.json$`) + +// KubeEvent represents a flattened Kubernetes Event for the API response +type KubeEvent struct { + FirstTimestamp string `json:"firstTimestamp"` + LastTimestamp string `json:"lastTimestamp"` + Namespace string `json:"namespace"` + Kind string `json:"kind"` + Name string `json:"name"` + Type string `json:"type"` + Reason string `json:"reason"` + Message string `json:"message"` + Count int `json:"count"` + Source string `json:"source"` +} + +// rawKubeEvent is the Kubernetes Event structure from events.json +type rawKubeEvent struct { + FirstTimestamp string `json:"firstTimestamp"` + LastTimestamp string `json:"lastTimestamp"` + EventTime string `json:"eventTime"` + InvolvedObject map[string]interface{} `json:"involvedObject"` + Type string `json:"type"` + Reason string `json:"reason"` + Message string `json:"message"` + Count int `json:"count"` + Source map[string]string `json:"source"` + Metadata map[string]interface{} `json:"metadata"` + ReportingComp string `json:"reportingComponent"` +} + +// EventListResponse is the API response for job run events +type EventListResponse struct { + Items []KubeEvent `json:"items"` + JobRunURL string `json:"jobRunURL"` +} + +// JobRunEvents fetches events.json for a given job run from the GCS path +// artifacts/*e2e*/gather-extra/artifacts/events.json +func JobRunEvents(gcsClient *storage.Client, dbc *db.DB, jobRunID int64, gcsBucket, gcsPath string, logger *log.Entry) (*EventListResponse, error) { + jobRunURL := fmt.Sprintf("https://prow.ci.openshift.org/view/gs/%s/%s", gcsBucket, gcsPath) + + jobRun, err := api.FetchJobRun(dbc, jobRunID, false, nil, logger) + if err != nil { + logger.WithError(err).Debugf("failed to fetch job run %d", jobRunID) + if gcsPath == "" { + return nil, errors.New("no GCS path given and no job run found in DB") + } + } else { + jobRunURL = jobRun.URL + gcsBucket = jobRun.GCSBucket + _, path, found := strings.Cut(jobRunURL, "/"+gcsBucket+"/") + if !found { + return nil, fmt.Errorf("job run URL %q does not contain bucket %q", jobRun.URL, gcsBucket) + } + gcsPath = path + } + + gcsJobRun := gcs.NewGCSJobRun(gcsClient.Bucket(gcsBucket), gcsPath) + matches, err := gcsJobRun.FindAllMatches([]*regexp.Regexp{eventsJSONRegex}) + if err != nil { + return &EventListResponse{JobRunURL: jobRunURL}, err + } + + if len(matches) == 0 || len(matches[0]) == 0 { + logger.Info("no events.json file found") + return &EventListResponse{Items: []KubeEvent{}, JobRunURL: jobRunURL}, nil + } + + eventsPath := matches[0][0] + logger.WithField("events_path", eventsPath).Info("found events.json") + + content, err := gcsJobRun.GetContent(context.TODO(), eventsPath) + if err != nil { + logger.WithError(err).Errorf("error getting content for file: %s", eventsPath) + return nil, err + } + + var rawEvents struct { + Items []rawKubeEvent `json:"items"` + } + if err := json.Unmarshal(content, &rawEvents); err != nil { + logger.WithError(err).Error("error unmarshaling events.json") + return nil, err + } + + events := make([]KubeEvent, 0, len(rawEvents.Items)) + for _, raw := range rawEvents.Items { + evt := flattenEvent(raw) + events = append(events, evt) + } + + return &EventListResponse{Items: events, JobRunURL: jobRunURL}, nil +} + +func flattenEvent(raw rawKubeEvent) KubeEvent { + evt := KubeEvent{ + FirstTimestamp: raw.FirstTimestamp, + LastTimestamp: raw.LastTimestamp, + Type: raw.Type, + Reason: raw.Reason, + Message: raw.Message, + Count: raw.Count, + } + if raw.Count == 0 { + evt.Count = 1 + } + if raw.InvolvedObject != nil { + if k, ok := raw.InvolvedObject["kind"].(string); ok { + evt.Kind = k + } + if n, ok := raw.InvolvedObject["name"].(string); ok { + evt.Name = n + } + } + if raw.Metadata != nil { + if ns, ok := raw.Metadata["namespace"].(string); ok { + evt.Namespace = ns + } + } + if raw.Source != nil && raw.Source["component"] != "" { + evt.Source = raw.Source["component"] + } else if raw.ReportingComp != "" { + evt.Source = raw.ReportingComp + } + return evt +} diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index d1a65a7bd4..2735ab66fe 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -44,6 +44,7 @@ import ( "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness" + "github.com/openshift/sippy/pkg/api/jobrunevents" "github.com/openshift/sippy/pkg/api/jobrunintervals" apitype "github.com/openshift/sippy/pkg/apis/api" "github.com/openshift/sippy/pkg/apis/cache" @@ -1424,6 +1425,55 @@ func (s *Server) jsonJobRunIntervals(w http.ResponseWriter, req *http.Request) { api.RespondWithJSON(http.StatusOK, w, result) } +// jsonJobRunEvents fetches Kubernetes events from events.json in the job run's GCS artifacts. +// The file is located at artifacts/*e2e*/gather-extra/artifacts/events.json +func (s *Server) jsonJobRunEvents(w http.ResponseWriter, req *http.Request) { + logger := log.WithField("func", "jsonJobRunEvents") + + if s.gcsClient == nil { + failureResponse(w, http.StatusBadRequest, "server not configured for GCS, unable to use this API") + return + } + + jobRunIDStr := s.getParamOrFail(w, req, "prow_job_run_id") + if jobRunIDStr == "" { + return + } + + jobRunID, err := strconv.ParseInt(jobRunIDStr, 10, 64) + if err != nil { + failureResponse(w, http.StatusBadRequest, "unable to parse prow_job_run_id: "+err.Error()) + return + } + logger = logger.WithField("jobRunID", jobRunID) + + jobName := param.SafeRead(req, "job_name") + repoInfo := param.SafeRead(req, "repo_info") + pullNumber := param.SafeRead(req, "pull_number") + + var gcsPath string + if len(jobName) > 0 { + if len(repoInfo) > 0 { + if repoInfo == "openshift_origin" { + gcsPath = fmt.Sprintf("pr-logs/pull/%s/%s/%s", pullNumber, jobName, jobRunIDStr) + } else { + gcsPath = fmt.Sprintf("pr-logs/pull/%s/%s/%s/%s", repoInfo, pullNumber, jobName, jobRunIDStr) + } + } else { + gcsPath = fmt.Sprintf("logs/%s/%s", jobName, jobRunIDStr) + } + } + + result, err := jobrunevents.JobRunEvents(s.gcsClient, s.db, jobRunID, s.gcsBucket, gcsPath, + logger.WithField("func", "JobRunEvents")) + if err != nil { + failureResponse(w, http.StatusBadRequest, err.Error()) + return + } + + api.RespondWithJSON(http.StatusOK, w, result) +} + func isValidProwJobRun(jobRun *models.ProwJobRun) (bool, string) { if (jobRun == nil || jobRun == &models.ProwJobRun{} || &jobRun.ProwJob == &models.ProwJob{} || jobRun.ProwJob.Name == "") { @@ -2179,6 +2229,13 @@ func (s *Server) Serve() { CacheTime: 4 * time.Hour, HandlerFunc: s.jsonJobRunIntervals, }, + { + EndpointPath: "/api/jobs/runs/events", + Description: "Returns Kubernetes events from job run artifacts (events.json)", + Capabilities: []string{LocalDBCapability}, + CacheTime: 4 * time.Hour, + HandlerFunc: s.jsonJobRunEvents, + }, { EndpointPath: "/api/jobs/analysis", Description: "Analyzes jobs from the database", diff --git a/sippy-ng/src/App.js b/sippy-ng/src/App.js index 8d1b7434c6..665c6bf72a 100644 --- a/sippy-ng/src/App.js +++ b/sippy-ng/src/App.js @@ -45,6 +45,7 @@ import ChevronRightIcon from '@mui/icons-material/ChevronRight' import CollapsibleChatDrawer from './chat/CollapsibleChatDrawer' import ComponentReadiness from './component_readiness/ComponentReadiness' import Drawer from '@mui/material/Drawer' +import EventsChart from './prow_job_runs/EventsChart' import FeatureGates from './tests/FeatureGates' import IconButton from '@mui/material/IconButton' import Install from './releases/Install' @@ -360,6 +361,19 @@ const IntervalsChartWrapper = () => { ) } +const EventsChartWrapper = () => { + const { jobrunid, jobname, repoinfo, pullnumber } = useParams() + + return ( + + ) +} + const ChatInterfaceWrapper = () => { const { id } = useParams() return @@ -756,6 +770,10 @@ function App(props) { path="/job_runs/:jobrunid/:jobname?/:repoinfo?/:pullnumber?/intervals" element={} /> + } + /> {sippyCapabilities.includes('chat') && ( <> diff --git a/sippy-ng/src/prow_job_runs/EventsChart.js b/sippy-ng/src/prow_job_runs/EventsChart.js new file mode 100644 index 0000000000..16affa945a --- /dev/null +++ b/sippy-ng/src/prow_job_runs/EventsChart.js @@ -0,0 +1,564 @@ +import { + Box, + Button, + Checkbox, + CircularProgress, + FormControl, + FormControlLabel, + InputLabel, + Link, + MenuItem, + Paper, + Select, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + TextField, + Typography, +} from '@mui/material' +import { makeStyles } from '@mui/styles' +import { Link as RouterLink, useParams } from 'react-router-dom' +import { StringParam, useQueryParam } from 'use-query-params' +import Alert from '@mui/material/Alert' +import LaunderedLink from '../components/Laundry' +import PropTypes from 'prop-types' +import React, { + Fragment, + useCallback, + useEffect, + useMemo, + useState, +} from 'react' + +const useStyles = makeStyles({ + filterRow: { + padding: '10px 0', + paddingBottom: '1rem', + }, + controls: { + marginBottom: '1rem', + }, + columnToggle: { + marginRight: '0.5rem', + marginBottom: '0.5rem', + }, +}) + +const COLUMNS = [ + { id: 'firstTimestamp', label: 'First Timestamp', visible: true }, + { id: 'lastTimestamp', label: 'Last Timestamp', visible: true }, + { id: 'namespace', label: 'Namespace', visible: true }, + { id: 'kind', label: 'Kind', visible: true }, + { id: 'name', label: 'Name', visible: true }, + { id: 'type', label: 'Type', visible: true }, + { id: 'reason', label: 'Reason', visible: true }, + { id: 'message', label: 'Message', visible: true }, + { id: 'count', label: 'Count', visible: true }, + { id: 'source', label: 'Source', visible: false }, +] + +function formatTimestamp(ts) { + if (!ts) return '' + const d = new Date(ts) + if (isNaN(d)) return ts + return d.toISOString().replace('T', ' ').replace('.000Z', ' UTC') +} + +function formatDateTimeLocalUTC(date) { + const pad = (n) => n.toString().padStart(2, '0') + return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad( + date.getUTCDate() + )}T${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}` +} + +function buildIntervalsPath(jobrunid, jobname, repoinfo, pullnumber) { + const parts = ['/job_runs', jobrunid] + if (jobname) parts.push(jobname) + if (repoinfo) parts.push(repoinfo) + if (pullnumber) parts.push(pullnumber) + return parts.join('/') + '/intervals' +} + +export default function EventsChart(props) { + const classes = useStyles() + const { jobrunid, jobname, repoinfo, pullnumber } = useParams() + const { jobRunID, jobName, repoInfo, pullNumber } = props + const effectiveJobRunID = jobRunID ?? jobrunid + const effectiveJobName = jobName ?? jobname + const effectiveRepoInfo = repoInfo ?? repoinfo + const effectivePullNumber = pullNumber ?? pullnumber + + const [fetchError, setFetchError] = useState('') + const [isLoaded, setLoaded] = useState(false) + const [allEvents, setAllEvents] = useState([]) + const [jobRunUrl, setJobRunUrl] = useState('') + const [columns, setColumns] = useState(COLUMNS) + const [sortColumn, setSortColumn] = useState('firstTimestamp') + const [sortDirection, setSortDirection] = useState('desc') + + const [timeFrom = '', setTimeFrom] = useQueryParam('timeFrom', StringParam) + const [timeTo = '', setTimeTo] = useQueryParam('timeTo', StringParam) + const [filterKind = '', setFilterKind] = useQueryParam('kind', StringParam) + const [filterNamespace = '', setFilterNamespace] = useQueryParam( + 'namespace', + StringParam + ) + const [filterName = '', setFilterName] = useQueryParam('name', StringParam) + const [filterType = '', setFilterType] = useQueryParam('type', StringParam) + const [filterReason = '', setFilterReason] = useQueryParam( + 'reason', + StringParam + ) + const [filterMessage = '', setFilterMessage] = useQueryParam( + 'message', + StringParam + ) + + const fetchData = useCallback(() => { + setFetchError('') + const url = + process.env.REACT_APP_API_URL + + '/api/jobs/runs/events?prow_job_run_id=' + + effectiveJobRunID + + (effectiveJobName + ? '&job_name=' + encodeURIComponent(effectiveJobName) + : '') + + (effectiveRepoInfo + ? '&repo_info=' + encodeURIComponent(effectiveRepoInfo) + : '') + + (effectivePullNumber + ? '&pull_number=' + encodeURIComponent(effectivePullNumber) + : '') + + fetch(url) + .then((response) => { + if (response.status !== 200) { + throw new Error('server returned ' + response.status) + } + return response.json() + }) + .then((json) => { + if (json != null) { + setJobRunUrl(json.jobRunURL || '') + setAllEvents(json.items || []) + + const events = json.items || [] + if (events.length > 0 && !timeFrom && !timeTo) { + const timestamps = events + .map((e) => new Date(e.firstTimestamp)) + .filter((d) => !isNaN(d)) + if (timestamps.length > 0) { + const min = new Date(Math.min(...timestamps)) + const max = new Date(Math.max(...timestamps)) + setTimeFrom(formatDateTimeLocalUTC(min)) + setTimeTo(formatDateTimeLocalUTC(max)) + } + } + } else { + setAllEvents([]) + } + setLoaded(true) + }) + .catch((error) => { + setFetchError( + 'Could not retrieve events for jobRunID=' + + jobRunID + + ' jobName=' + + jobName + + ', ' + + error + ) + setLoaded(true) + }) + }, [ + effectiveJobRunID, + effectiveJobName, + effectiveRepoInfo, + effectivePullNumber, + ]) + + useEffect(() => { + fetchData() + }, [fetchData]) + + const filteredEvents = useMemo(() => { + const timeFromVal = timeFrom ? new Date(timeFrom + 'Z') : null + const timeToVal = timeTo ? new Date(timeTo + 'Z') : null + const filterNameLower = filterName.toLowerCase() + const filterMessageLower = filterMessage.toLowerCase() + + return allEvents.filter((event) => { + const eventTime = new Date(event.firstTimestamp) + if (timeFromVal && eventTime < timeFromVal) return false + if (timeToVal && eventTime > timeToVal) return false + if (filterKind && event.kind !== filterKind) return false + if (filterNamespace && event.namespace !== filterNamespace) return false + if (filterName && !event.name?.toLowerCase().includes(filterNameLower)) + return false + if (filterType && event.type !== filterType) return false + if (filterReason && event.reason !== filterReason) return false + if ( + filterMessage && + !event.message?.toLowerCase().includes(filterMessageLower) + ) + return false + return true + }) + }, [ + allEvents, + timeFrom, + timeTo, + filterKind, + filterNamespace, + filterName, + filterType, + filterReason, + filterMessage, + ]) + + const sortedEvents = useMemo(() => { + return [...filteredEvents].sort((a, b) => { + let aVal = a[sortColumn] + let bVal = b[sortColumn] + + if (sortColumn.includes('Timestamp')) { + aVal = new Date(aVal || 0) + bVal = new Date(bVal || 0) + } else if (sortColumn === 'count') { + aVal = parseInt(aVal) || 0 + bVal = parseInt(bVal) || 0 + } + + if (aVal < bVal) return sortDirection === 'asc' ? -1 : 1 + if (aVal > bVal) return sortDirection === 'asc' ? 1 : -1 + return 0 + }) + }, [filteredEvents, sortColumn, sortDirection]) + + const filterOptions = useMemo(() => { + const kinds = [...new Set(allEvents.map((e) => e.kind))] + .filter(Boolean) + .sort() + const namespaces = [...new Set(allEvents.map((e) => e.namespace))] + .filter(Boolean) + .sort() + const types = [...new Set(allEvents.map((e) => e.type))] + .filter(Boolean) + .sort() + const reasons = [...new Set(allEvents.map((e) => e.reason))] + .filter(Boolean) + .sort() + return { kinds, namespaces, types, reasons } + }, [allEvents]) + + const visibleColumns = columns.filter((c) => c.visible) + + const handleSort = (colId) => { + if (sortColumn === colId) { + setSortDirection((d) => (d === 'asc' ? 'desc' : 'asc')) + } else { + setSortColumn(colId) + setSortDirection('desc') + } + } + + const toggleColumn = (colId, visible) => { + setColumns((prev) => + prev.map((c) => (c.id === colId ? { ...c, visible } : c)) + ) + } + + const clearFilters = () => { + setFilterKind('') + setFilterNamespace('') + setFilterName('') + setFilterType('') + setFilterReason('') + setFilterMessage('') + if (allEvents.length > 0) { + const timestamps = allEvents + .map((e) => new Date(e.firstTimestamp)) + .filter((d) => !isNaN(d)) + if (timestamps.length > 0) { + const min = new Date(Math.min(...timestamps)) + const max = new Date(Math.max(...timestamps)) + setTimeFrom(formatDateTimeLocalUTC(min)) + setTimeTo(formatDateTimeLocalUTC(max)) + } + } + } + + const warningCount = filteredEvents.filter((e) => e.type === 'Warning').length + + if (fetchError) { + return {fetchError} + } + + if (!isLoaded) { + return ( + + + Loading events for job run: jobRunID={jobRunID}, jobName={jobName}, + pullNumber={pullNumber}, repoInfo={repoInfo} + + + + ) + } + + return ( + + + + View Intervals Chart + + + + Loaded {allEvents.length} events from{' '} + {jobRunUrl ? ( + GCS job run + ) : ( + 'GCS job run' + )} + , filtered down to {filteredEvents.length}. + + + + + + Time Range Filter (UTC) + + + setTimeFrom(e.target.value)} + size="small" + InputLabelProps={{ shrink: true }} + /> + setTimeTo(e.target.value)} + size="small" + InputLabelProps={{ shrink: true }} + /> + + + + + + Filters + + + + Kind + setFilterKind(e.target.value)} + > + All + {filterOptions.kinds.map((v) => ( + + {v} + + ))} + + + + Namespace + setFilterNamespace(e.target.value)} + > + All + {filterOptions.namespaces.map((v) => ( + + {v} + + ))} + + + setFilterName(e.target.value)} + placeholder="Search name..." + sx={{ minWidth: 150 }} + /> + + Type + setFilterType(e.target.value)} + > + All + {filterOptions.types.map((v) => ( + + {v} + + ))} + + + + Reason + setFilterReason(e.target.value)} + > + All + {filterOptions.reasons.map((v) => ( + + {v} + + ))} + + + setFilterMessage(e.target.value)} + placeholder="Search message..." + sx={{ minWidth: 200 }} + /> + + Clear Filters + + + + + + + Toggle Columns + + + {columns.map((col) => ( + toggleColumn(col.id, e.target.checked)} + /> + } + label={col.label} + className={classes.columnToggle} + /> + ))} + + + + + + Total: {allEvents.length} events + + + Filtered: {filteredEvents.length} events + + + Warnings: {warningCount} + + + + + + + + + {visibleColumns.map((col) => ( + handleSort(col.id)} + sx={{ + cursor: 'pointer', + fontWeight: 600, + whiteSpace: 'nowrap', + }} + > + {col.label} + {sortColumn === col.id && + (sortDirection === 'asc' ? ' ▲' : ' ▼')} + + ))} + + + + {sortedEvents.length === 0 ? ( + + + {allEvents.length === 0 + ? 'No events.json found for this job run' + : 'No events match the current filters'} + + + ) : ( + sortedEvents.map((event, idx) => ( + + {visibleColumns.map((col) => { + let value = event[col.id] + if (col.id.includes('Timestamp')) { + value = formatTimestamp(value) + } + return ( + + {value ?? ''} + + ) + })} + + )) + )} + + + + + ) +} + +EventsChart.propTypes = { + jobRunID: PropTypes.string.isRequired, + jobName: PropTypes.string, + repoInfo: PropTypes.string, + pullNumber: PropTypes.string, +} diff --git a/sippy-ng/src/prow_job_runs/IntervalsChart.js b/sippy-ng/src/prow_job_runs/IntervalsChart.js index 89831a9bcd..5f727fac61 100644 --- a/sippy-ng/src/prow_job_runs/IntervalsChart.js +++ b/sippy-ng/src/prow_job_runs/IntervalsChart.js @@ -11,6 +11,7 @@ import { Button, Checkbox, CircularProgress, + Link, MenuItem, Select, TextField, @@ -18,8 +19,8 @@ import { } from '@mui/material' import { escapeRegex } from '../helpers' import { makeStyles } from '@mui/styles' +import { Link as RouterLink, useNavigate, useParams } from 'react-router-dom' import { stringify } from 'query-string' -import { useNavigate } from 'react-router-dom' import Alert from '@mui/material/Alert' import FormControlLabel from '@mui/material/FormControlLabel' import FormGroup from '@mui/material/FormGroup' @@ -214,6 +215,7 @@ const intervalColorizers = { export default function IntervalsChart(props) { const navigate = useNavigate() const classes = useStyles() + const { jobrunid, jobname, repoinfo, pullnumber } = useParams() const [fetchError, setFetchError] = React.useState('') const [isLoaded, setLoaded] = React.useState(false) @@ -570,6 +572,18 @@ export default function IntervalsChart(props) { return ( + + + View Cluster Events + + Loaded {eventIntervals.length} intervals from{' '} GCS job run, filtered
+ Loading events for job run: jobRunID={jobRunID}, jobName={jobName}, + pullNumber={pullNumber}, repoInfo={repoInfo} +
Loaded {eventIntervals.length} intervals from{' '} GCS job run, filtered