diff --git a/.github/workflows/create-sae-release.yaml b/.github/workflows/create-sae-release.yaml new file mode 100644 index 0000000..bf520bb --- /dev/null +++ b/.github/workflows/create-sae-release.yaml @@ -0,0 +1,82 @@ +name: Create SAE helm release + +on: + push: + workflow_dispatch: + inputs: + versionChoice: + type: choice + required: true + description: Version Change + options: + - patch + - minor + - major + +permissions: + contents: write + +env: + VERSION_CHOICE: ${{ github.event.inputs.versionChoice }} + CHART_NAME: sae + DOCKERHUB_USERNAME: starwit + DOCKERHUB_ORG: starwitorg + +jobs: + publish-helm: + name: Publish helm chart + runs-on: [self-hosted, linux, X64] + + steps: + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install GH CLI + uses: dev-hanz-ops/install-gh-cli-action@v0.2.1 + with: + gh-cli-version: 2.78.0 + + - name: Login to Helm registry + run: helm registry login registry-1.docker.io -u $DOCKERHUB_USERNAME -p ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Run Helm update + working-directory: helm/sae + run: helm dep update + + - name: Bump version and commit + working-directory: helm/sae + run: | + CURRENT_VERSION=$(grep version: Chart.yaml | head -1 | awk '{print $2}') + echo "RELEASE_VERSION=$(npx semver $CURRENT_VERSION -i ${{ env.VERSION_CHOICE }})" >> $GITHUB_ENV + + sed -i -r "s/^version: [0-9]+\.[0-9]+\.[0-9]+$/version: ${{ env.RELEASE_VERSION }}/" Chart.yaml + + # git add Chart.yaml + # git commit -m "Bump version to ${{ env.RELEASE_VERSION }}" + # git tag ${{ env.RELEASE_VERSION }} + # git push + # - name: Create Github release + # run: gh release create ${{ env.RELEASE_VERSION }} --generate-notes + # env: + # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # - name: Do Helm release + # working-directory: helm/sae + # run: | + # rm sae-*.tgz + # helm dependency build + # helm package . + # helm push sae-*.tgz oci://registry-1.docker.io/$DOCKERHUB_ORG + + + diff --git a/README.md b/README.md index 1a57fd5..6dfb098 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,18 @@ Please note, that regardless of deployment method - SAE works in all of them wit ## Repositories The components of the vision pipeline can be found in the following repositories: -| Component | Repository / URI | -| ---------------- | ---------------------------------------------------------- | -| Video Source | https://github.com/starwit/video-source-py | -| Object Detector | https://github.com/starwit/object-detector | -| Object Tracker | https://github.com/starwit/object-tracker | -| Geo Mapper | https://github.com/starwit/geo-mapper | -| Position Source | https://github.com/starwit/sae-position-source | -| Redis Writer | https://github.com/starwit/sae-redis-writer | -| Database Writer | https://github.com/starwit/vision-api-jms-client | -| vision-api | https://github.com/starwit/vision-api | -| vision-lib | https://github.com/starwit/vision-lib | +| Component | Repository / URI | +| -------------------- | ---------------------------------------------------------- | +| Video Source | https://github.com/starwit/sae-video-source | +| Position Source | https://github.com/starwit/sae-position-source | +| Object Detector | https://github.com/starwit/sae-object-detector | +| Object Tracker | https://github.com/starwit/sae-object-tracker | +| Geo Mapper | https://github.com/starwit/sae-geo-mapper | +| Detection Aggregator | https://github.com/starwit/detection-aggregator | +| Redis Writer | https://github.com/starwit/sae-redis-writer | +| Database Writer | https://github.com/starwit/sae-database-writer | +| vision-api | https://github.com/starwit/vision-api | +| vision-lib | https://github.com/starwit/vision-lib | ## Content Overview - [`/doc`](doc/README.md) - Documentation of the architecture and some details of the technical setup diff --git a/doc/installation/Installation.md b/doc/installation/Installation.md index 68f8c2f..a3f2f91 100644 --- a/doc/installation/Installation.md +++ b/doc/installation/Installation.md @@ -9,7 +9,7 @@ Please note, that regardless of deployment method - SAE works in all of them wit ## Installation on Kubernetes cluster 1. Set up K3s cluster \ - - Create K3s config file (adapt the value of `tls-san`) + - Create K3s config file at `/etc/rancher/k3s/config.yaml` (adapt the value of `tls-san`) ```yaml write-kubeconfig-mode: "0644" tls-san: diff --git a/docker-compose/.env.template b/docker-compose/.env.template index 443bdfc..173befb 100644 --- a/docker-compose/.env.template +++ b/docker-compose/.env.template @@ -1,12 +1,11 @@ IMAGE_REGISTRY=docker.io/starwitorg -VIDEO_SOURCE_VERSION=2.3.0 -OBJECT_DETECTOR_VERSION=3.3.0 +VIDEO_SOURCE_VERSION=2.3.1 +OBJECT_DETECTOR_VERSION=4.1.0 OBJECT_TRACKER_VERSION=3.3.0 -GEO_MAPPER_VERSION=0.9.0 -GEO_MERGER_VERSION=0.1.0 +GEO_MAPPER_VERSION=0.10.0 DATABASE_WRITER_VERSION=2.0.0 REDIS_WRITER_VERSION=2.0.2 POSITION_SOURCE_VERSION=1.1.1 -AGGREGATOR_VERSION=0.2.4 +AGGREGATOR_VERSION=0.2.7 VIDEO_PATH=/PATH/TO/VIDEO \ No newline at end of file diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 0ea5783..f442c39 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -20,9 +20,17 @@ services: depends_on: redis: condition: service_healthy + + position-source: + image: ${IMAGE_REGISTRY}/sae-position-source:${POSITION_SOURCE_VERSION} + volumes: + - ./position-source/position-source.settings.yaml:/code/settings.yaml + depends_on: + redis: + condition: service_healthy object-detector: - image: ${IMAGE_REGISTRY}/object-detector:${OBJECT_DETECTOR_VERSION} + image: ${IMAGE_REGISTRY}/sae-object-detector:${OBJECT_DETECTOR_VERSION} volumes: - ./object-detector/object-detector.settings.yaml:/code/settings.yaml depends_on: diff --git a/docker-compose/geo-mapper/geo-mapper.settings.yaml b/docker-compose/geo-mapper/geo-mapper.settings.yaml index c3b7019..eeba587 100644 --- a/docker-compose/geo-mapper/geo-mapper.settings.yaml +++ b/docker-compose/geo-mapper/geo-mapper.settings.yaml @@ -14,6 +14,9 @@ cameras: abc_distortion_b: 0.345 abc_distortion_c: -0.705 +mapping_strategy: + mode: static + log_level: INFO redis: host: redis diff --git a/docker-compose/object-detector/object-detector-was.settings.yaml b/docker-compose/object-detector/object-detector-was.settings.yaml index 95d7053..63d0f9d 100644 --- a/docker-compose/object-detector/object-detector-was.settings.yaml +++ b/docker-compose/object-detector/object-detector-was.settings.yaml @@ -1,8 +1,8 @@ model: weights_path: waste.pt device: "cpu" -inference_size: [ 640, 640 ] -#classes: [ 2 ] + inference_size: [ 640, 640 ] + #classes: [ 2 ] log_level: DEBUG redis: input_stream_prefix: videosource diff --git a/docker-compose/object-detector/object-detector.settings.yaml b/docker-compose/object-detector/object-detector.settings.yaml index c44d294..477dbc5 100644 --- a/docker-compose/object-detector/object-detector.settings.yaml +++ b/docker-compose/object-detector/object-detector.settings.yaml @@ -1,8 +1,8 @@ model: weights_path: yolov8n.pt device: "cpu" -inference_size: [ 640, 640 ] -classes: [ 2 ] + inference_size: [ 640, 640 ] + classes: [ 2 ] log_level: DEBUG redis: host: redis diff --git a/docker-compose/position-source/position-source.settings.yaml b/docker-compose/position-source/position-source.settings.yaml index 89e063d..4cc4e75 100644 --- a/docker-compose/position-source/position-source.settings.yaml +++ b/docker-compose/position-source/position-source.settings.yaml @@ -1,16 +1,16 @@ -#position_source: # static source outputs the same coordinates twice a second -# type: static -# lat: 52 -# lon: 10 -position_source: # dynamic source tries to read NMEA sentences from given serial device (i.e. USB gps receiver) - type: serial - serial_device: /dev/ttyACM0 +position_source: # static source outputs the same coordinates twice a second + type: static + lat: 52 + lon: 10 +# position_source: # dynamic source tries to read NMEA sentences from given serial device (i.e. USB gps receiver) +# type: serial +# serial_device: /dev/ttyACM0 # position_source: # command source tries to read NMEA sentences from stdout of given command # type: command # command: ["gpspipe", "-r"] # this is passed to Popen; if you want to use a bash pipeline do ["bash", "-c", "command | command2"] -log_level: DEBUG +log_level: INFO redis: host: redis port: 6379 diff --git a/docker-compose/redis-writer/redis-writer.settings.yaml b/docker-compose/redis-writer/redis-writer.settings.yaml index e80e624..04f97e0 100644 --- a/docker-compose/redis-writer/redis-writer.settings.yaml +++ b/docker-compose/redis-writer/redis-writer.settings.yaml @@ -2,12 +2,10 @@ log_level: DEBUG redis: host: redis port: 6379 - input_stream_prefix: geomapper target_redis: host: secondary-redis port: 6379 - output_stream_prefix: output buffer_length: 100 target_stream_maxlen: 100 -stream_ids: - - stream1 \ No newline at end of file +mapping_config: + - source: geomapper:stream1 \ No newline at end of file diff --git a/docker-compose/video-source-py/video-source-stream1.settings.yaml b/docker-compose/video-source-py/video-source-stream1.settings.yaml index ca56295..498538a 100644 --- a/docker-compose/video-source-py/video-source-stream1.settings.yaml +++ b/docker-compose/video-source-py/video-source-stream1.settings.yaml @@ -3,7 +3,7 @@ uri: rtsp://streaming-server:8554/video-stream reconnect_backoff_time: 1 max_fps: 10 jpeg_encode: True -log_level: DEBUG +log_level: INFO redis: host: redis port: 6379 \ No newline at end of file diff --git a/helm/sae/customvalues.template.yaml b/helm/sae/customvalues.template.yaml index 3002707..69f8d84 100644 --- a/helm/sae/customvalues.template.yaml +++ b/helm/sae/customvalues.template.yaml @@ -1,35 +1,58 @@ +positionSource: + enabled: false + settingsYaml: # Everything below this is passed as content of the settings.yaml (notice snake_case vs camelCase) + log_level: INFO + position_source: # static source outputs the same coordinates twice a second + type: static + lat: 52 + lon: 10 + # position_source: # command source tries to read NMEA sentences from stdout of given command (this is recommended) + # type: command + # command: ["gpspipe", "-r"] # this is passed to Popen; if you want to use a bash pipeline do ["bash", "-c", "command | command2"] + # position_source: # dynamic source tries to read NMEA sentences from given serial device (i.e. USB gps receiver) + # type: dynamic + # serial_device: /dev/ttyACM0 + gpsdSidecar: + enabled: false # only enable if non-static config above + image: + repository: starwitorg/sae-gpsd + tag: 2025-09-10 + hostGpsDevice: /path/to/gpsdevice # the path to a character device gpsd can use (e.g. /dev/ttyACM0) (is passed to hostPath volume mount) + entrypointSh: | + sleep 3600 + videoSource: - logLevel: DEBUG # Options: ERROR, WARNING, INFO, DEBUG - configs: + settingsYamls: # Everything below this is passed as content of the settings.yaml (notice snake_case vs camelCase) - id: stream1 # The name of the video stream. Will be used as Redis stream key and as camera_id in database output. uri: 'uri' # Where to retrieve the video from. Must be in a format that OpenCV VideoCapture understands. RTSP stream makes the most sense. max_fps: 5 # Effectively sets a lower bound on the time between to frames. 0 means off. Integer fractions of original frame rate make the most sense (i.e. 15, 10, 5 for a 30fps source). scale_width: 0 # If > 0, scale the image to the given width (preserving aspect ratio) jpeg_quality: 80 # JPEG quality 0 - 100 (100 is lossless and big); Reasonable starting points (for image height): 2160 = 80, 1080 = 90, <720 = 95 + log_level: INFO # Options: ERROR, WARNING, INFO, DEBUG objectDetector: - logLevel: DEBUG # Options: ERROR, WARNING, INFO, DEBUG customWeights: enabled: false # Whether to inject custom weights from init container (into custom_weights/*) imageTag: weights-image-tag # Which tag of the starwitorg/sae-object-detector-weights image to use - config: + settingsYaml: + log_level: INFO # Options: ERROR, WARNING, INFO, DEBUG model: weights_path: custom_weights/xyz # Which weights to load. yolov8[nsmlx].pt are shipped with the object-detector. For custom weights see above. device: cpu # Options: cpu, cuda (uses gpu; needs proper setup; see README) nms_agnostic: false # Whether to use class-agnostic non-maximum suppression (NMS) (for overlapping detections) + inference_size: [ 640, 640 ] # What resolution the image will be downscaled to before object-detection inference (should be square and a multiple of 32) + classes: [ 2 ] # Which object classes to detect (refer to coco object classes) drop_edge_detections: false # Drop detections touching the frame borders - inference_size: [ 640, 640 ] # What resolution the image will be downscaled to before object-detection inference (should be square and a multiple of 32) - classes: [ 2 ] # Which object classes to detect (refer to coco object classes) redis: stream_ids: - stream1 # On which video streams to detect objects on. Mostly all video source stream ids. objectTracker: - logLevel: INFO # Options: ERROR, WARNING, INFO, DEBUG needsGpu: False # Whether the algorithm needs gpu support (needs proper setup; see README) streamIds: # On which video streams to track objects (the same as object-detector stream ids above) - stream1 - config: # Tracker specific config (see object-tracker repo / Boxmot for details) + settingsYaml: # Tracker specific config (see object-tracker repo / Boxmot for details) + log_level: INFO # Options: ERROR, WARNING, INFO, DEBUG tracker_algorithm: OCSORT tracker_config: det_thresh: 0.2 @@ -45,7 +68,8 @@ objectTracker: geoMapper: enabled: false # If the geomapper should be deployed (by default it reads from "objecttracker:*"" and outputs into "geomapper:*") - config: + settingsYaml: + log_level: INFO # Options: ERROR, WARNING, INFO, DEBUG cameras: # Parameters have to be specified for each camera (it makes no sense to run this without correct-ish parameters) - stream_id: stream1 # This must match one of the existing camera streams passthrough: false # If the stream should be passed through without geo mapping (all other parameters are ignored if true) @@ -67,33 +91,22 @@ geoMapper: ]] remove_unmapped_detections: false # # If unmapped detections should be removed (i.e. detections filtered by mapping_area, see above) -geoMerger: - enabled: false - config: - merging_config: - max_distance_m: 2 - merging_window_ms: 1000 - target_mps: 10 - input_stream_ids: - - stream1 - - stream2 - output_stream_id: merged - redisWriter: enabled: false # If the redis-writer should be deployed (off by default) - configs: # One config per target + instances: # One config per target - name: writer1 # A unique name to identify the instance - redis: # The SAE internal Redis instance - input_stream_prefix: objecttracker # The prefix of the input stream (e.g. objecttracker or geomapper) - target_redis: # The Redis instance to write to - host: redis-host - port: 6379 - output_stream_prefix: saeoutput # The prefix of the output stream (e.g. data will be published to saeoutput:) - buffer_length: 10 # How many messages to buffer before discarding - target_stream_maxlen: 100 # How many messages to keep in the output stream (translates to XADD maxlen option) - tls: false - stream_ids: # Which video stream to attach to - - stream1 + settingsYaml: + log_level: INFO # Options: ERROR, WARNING, INFO, DEBUG + #remove_frame_data: null # If frame data should be removed (default true; only applies to SAE messages; beware of privacy and bandwidth consequences!) + target_redis: # The Valkey instance to send this to + host: host + port: 6379 + buffer_length: 10 + target_stream_maxlen: 100 + tls: false + mapping_config: # A list of source (i.e. this SAE instance) to target stream (i.e. on target_redis) mappings + - source: geomapper:stream1 # Source stream name (fully-qualified, including stream prefix) + target: geomapper:mysae1 # (optional) Target stream name; Source name is used if omitted databaseWriter: enabled: true # If the database-writer should be deployed diff --git a/helm/sae/templates/geo-merger.configmap.yaml b/helm/sae/templates/detection-aggregator.configmap.yaml similarity index 58% rename from helm/sae/templates/geo-merger.configmap.yaml rename to helm/sae/templates/detection-aggregator.configmap.yaml index c955293..770d999 100644 --- a/helm/sae/templates/geo-merger.configmap.yaml +++ b/helm/sae/templates/detection-aggregator.configmap.yaml @@ -1,13 +1,14 @@ -{{- with .Values.geoMerger }} +{{- with .Values.detectionAggregator }} {{- if .enabled }} -{{- with .config }} +{{- with .settingsYaml }} apiVersion: v1 kind: ConfigMap metadata: - name: geo-merger-config + name: detection-aggregator-config data: settings.yaml: | {{- toYaml . | nindent 4 }} +--- {{- end }} {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/sae/templates/geo-merger.statefulset.yaml b/helm/sae/templates/detection-aggregator.statefulset.yaml similarity index 80% rename from helm/sae/templates/geo-merger.statefulset.yaml rename to helm/sae/templates/detection-aggregator.statefulset.yaml index 63fcdea..c115cfe 100644 --- a/helm/sae/templates/geo-merger.statefulset.yaml +++ b/helm/sae/templates/detection-aggregator.statefulset.yaml @@ -1,15 +1,17 @@ -{{- with .Values.geoMerger }} +{{- with .Values.detectionAggregator }} {{- if .enabled }} -{{- $imageRepo := required "image repository for geo-merger is required" .image.repository }} -{{- $imageTag := required "image tag for geo-merger is required" .image.tag }} -{{- $instanceName := "geo-merger" }} +{{- $imageRepo := required "image repository for detection-aggregator is required" .image.repository }} +{{- $imageTag := required "image tag for detection-aggregator is required" .image.tag }} +{{- $rootConfig := . }} +{{- with .settingsYaml -}} +{{- $instanceName := printf "detection-aggregator" }} apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ include "sae.fullname" $ }}-{{ $instanceName }} labels: {{- include "sae.labels" $ | nindent 4 }} - app.kubernetes.io/component: geo-merger + app.kubernetes.io/component: detection-aggregator spec: selector: matchLabels: @@ -18,8 +20,8 @@ spec: template: metadata: annotations: - sae.starwit.de/stage: geo-merger - sae.starwit.de/config-checksum: {{ include (print $.Template.BasePath "/geo-merger.configmap.yaml") $ | sha256sum }} + sae.starwit.de/stage: detection-aggregator + sae.starwit.de/config-checksum: {{ include (print $.Template.BasePath "/detection-aggregator.configmap.yaml") $ | sha256sum }} labels: {{- include "sae.selectorLabels" $ | nindent 8 }} app.kubernetes.io/component: {{ $instanceName }} @@ -40,8 +42,6 @@ spec: resources: {{- toYaml $.Values.resources | nindent 12 }} env: - - name: LOG_LEVEL - value: {{ default "INFO" .logLevel | quote }} - name: REDIS__HOST value: {{ include "sae.redisServiceName" $ }} - name: REDIS__PORT @@ -72,3 +72,4 @@ spec: --- {{- end }} {{- end }} +{{- end }} diff --git a/helm/sae/templates/geo-mapper.configmap.yaml b/helm/sae/templates/geo-mapper.configmap.yaml index f89d14d..9926c1d 100644 --- a/helm/sae/templates/geo-mapper.configmap.yaml +++ b/helm/sae/templates/geo-mapper.configmap.yaml @@ -1,6 +1,6 @@ {{- with .Values.geoMapper }} {{- if .enabled }} -{{- with .config }} +{{- with .settingsYaml }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm/sae/templates/geo-mapper.statefulset.yaml b/helm/sae/templates/geo-mapper.statefulset.yaml index 08d1f34..196c50f 100644 --- a/helm/sae/templates/geo-mapper.statefulset.yaml +++ b/helm/sae/templates/geo-mapper.statefulset.yaml @@ -40,8 +40,6 @@ spec: resources: {{- toYaml $.Values.resources | nindent 12 }} env: - - name: LOG_LEVEL - value: {{ default "INFO" .logLevel | quote }} - name: REDIS__HOST value: {{ include "sae.redisServiceName" $ }} - name: REDIS__PORT diff --git a/helm/sae/templates/object-detector.configmap.yaml b/helm/sae/templates/object-detector.configmap.yaml index 63940bc..c83797f 100644 --- a/helm/sae/templates/object-detector.configmap.yaml +++ b/helm/sae/templates/object-detector.configmap.yaml @@ -1,5 +1,5 @@ {{- with .Values.objectDetector }} -{{- with .config }} +{{- with .settingsYaml }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm/sae/templates/object-detector.statefulset.yaml b/helm/sae/templates/object-detector.statefulset.yaml index c28fe94..0ccfdda 100644 --- a/helm/sae/templates/object-detector.statefulset.yaml +++ b/helm/sae/templates/object-detector.statefulset.yaml @@ -1,7 +1,8 @@ {{- with .Values.objectDetector }} {{- $imageRepo := required "image repository for object-detector is required" .image.repository }} {{- $imageTag := required "image tag for object-detector is required" .image.tag }} -{{- $isUsingGpu := hasPrefix "cuda" .config.model.device }} +{{- $isUsingNvidia := hasPrefix "cuda" .settingsYaml.model.device }} +{{- $isUsingIntel := hasPrefix "intel" .settingsYaml.model.device }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -26,7 +27,7 @@ spec: serviceAccountName: {{ include "sae.serviceAccountName" $ }} securityContext: {{- toYaml $.Values.podSecurityContext | nindent 8 }} - {{- if $isUsingGpu }} + {{- if $isUsingNvidia }} runtimeClassName: nvidia {{- end }} initContainers: @@ -46,14 +47,17 @@ spec: {{- toYaml $.Values.securityContext | nindent 12 }} image: "{{ $imageRepo }}:{{ $imageTag }}" imagePullPolicy: IfNotPresent - {{- if $isUsingGpu }} + {{- if $isUsingNvidia }} resources: limits: nvidia.com/gpu: 1 {{- end }} + {{- if $isUsingIntel }} + resources: + limits: + gpu.intel.com/i915: 1 + {{- end }} env: - - name: LOG_LEVEL - value: {{ .logLevel | quote }} - name: REDIS__HOST value: {{ include "sae.redisServiceName" $ }} - name: REDIS__PORT diff --git a/helm/sae/templates/object-tracker.configmap.yaml b/helm/sae/templates/object-tracker.configmap.yaml index 5ca09a6..98a50b8 100644 --- a/helm/sae/templates/object-tracker.configmap.yaml +++ b/helm/sae/templates/object-tracker.configmap.yaml @@ -1,7 +1,13 @@ +{{- with .Values.objectTracker }} +{{- if .enabled }} +{{- with .settingsYaml }} apiVersion: v1 kind: ConfigMap metadata: name: object-tracker-config data: settings.yaml: | - {{- toYaml .Values.objectTracker.config | nindent 4 }} \ No newline at end of file + {{- toYaml . | nindent 4 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/sae/templates/object-tracker.statefulset.yaml b/helm/sae/templates/object-tracker.statefulset.yaml index d616a18..87501a1 100644 --- a/helm/sae/templates/object-tracker.statefulset.yaml +++ b/helm/sae/templates/object-tracker.statefulset.yaml @@ -1,7 +1,8 @@ {{- with .Values.objectTracker }} +{{- if .enabled }} {{- $imageRepo := required "image repository for object-tracker is required" .image.repository }} {{- $imageTag := required "image tag for object-tracker is required" .image.tag }} -{{- $config := . }} +{{- $needsGpu := .needsGpu }} {{- range .streamIds }} {{- $streamId := . -}} {{- $instanceName := printf "object-tracker-%s" (lower .) }} @@ -29,7 +30,7 @@ spec: serviceAccountName: {{ include "sae.serviceAccountName" $ }} securityContext: {{- toYaml $.Values.podSecurityContext | nindent 8 }} - {{- if $config.needsGpu }} + {{- if $needsGpu }} runtimeClassName: nvidia {{- end }} initContainers: @@ -42,14 +43,12 @@ spec: {{- toYaml $.Values.securityContext | nindent 12 }} image: "{{ $imageRepo }}:{{ $imageTag }}" imagePullPolicy: IfNotPresent - {{- if $config.needsGpu }} + {{- if $needsGpu }} resources: limits: nvidia.com/gpu: 1 {{- end }} env: - - name: LOG_LEVEL - value: {{ $config.logLevel | quote }} - name: REDIS__HOST value: {{ include "sae.redisServiceName" $ }} - name: REDIS__PORT @@ -82,3 +81,4 @@ spec: --- {{- end }} {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/sae/templates/position-source.configmap.yaml b/helm/sae/templates/position-source.configmap.yaml new file mode 100644 index 0000000..c68d5b0 --- /dev/null +++ b/helm/sae/templates/position-source.configmap.yaml @@ -0,0 +1,24 @@ +{{- with .Values.positionSource }} +{{- if .enabled }} +{{- with .settingsYaml }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: position-source-config +data: + settings.yaml: | + {{- toYaml . | nindent 4 }} +--- +{{- end }} +{{- with .gpsdSidecar }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: position-source-gpsd-config +data: + entrypoint.sh: |- + {{- .entrypointSh | nindent 4 }} +--- +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/sae/templates/position-source.statefulset.yaml b/helm/sae/templates/position-source.statefulset.yaml new file mode 100644 index 0000000..48b4b6e --- /dev/null +++ b/helm/sae/templates/position-source.statefulset.yaml @@ -0,0 +1,105 @@ +{{- with .Values.positionSource }} +{{- if .enabled }} +{{- $imageRepo := required "image repository for position-source is required" .image.repository }} +{{- $imageTag := required "image tag for position-source is required" .image.tag }} +{{- $rootConfig := . }} +{{- with .settingsYaml -}} +{{- $instanceName := printf "position-source" }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "sae.fullname" $ }}-{{ $instanceName }} + labels: + {{- include "sae.labels" $ | nindent 4 }} + app.kubernetes.io/component: position-source +spec: + selector: + matchLabels: + {{- include "sae.selectorLabels" $ | nindent 6 }} + app.kubernetes.io/component: {{ $instanceName }} + template: + metadata: + annotations: + sae.starwit.de/stage: position-source + sae.starwit.de/config-checksum: {{ include (print $.Template.BasePath "/position-source.configmap.yaml") $ | sha256sum }} + labels: + {{- include "sae.selectorLabels" $ | nindent 8 }} + app.kubernetes.io/component: {{ $instanceName }} + spec: + serviceAccountName: {{ include "sae.serviceAccountName" $ }} + initContainers: + - name: wait-for-redis + image: valkey/valkey:7.2.6-alpine + command: ["sh", "-c", "while ! redis-cli -h {{ include "sae.redisServiceName" $ }} ping ; do echo \"$(date -Iseconds) - still waiting for redis\"; sleep 1; done"] + containers: + {{- with $rootConfig.gpsdSidecar }} + {{- if .enabled }} + - name: gpsd + securityContext: + privileged: true + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: IfNotPresent + resources: + {{- toYaml $.Values.resources | nindent 12 }} + volumeMounts: + - name: gps-device + mountPath: /dev/gps + - name: gpsd-config + mountPath: /entrypoint.sh + subPath: entrypoint.sh + {{- end }} + {{- end }} + - name: {{ $instanceName }} + securityContext: + {{- toYaml $.Values.securityContext | nindent 12 }} + image: "{{ $imageRepo }}:{{ $imageTag }}" + imagePullPolicy: IfNotPresent + resources: + {{- toYaml $.Values.resources | nindent 12 }} + env: + - name: REDIS__HOST + value: {{ include "sae.redisServiceName" $ }} + - name: REDIS__PORT + value: {{ include "sae.redisServicePort" $ }} + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + ports: + - name: prometheus + containerPort: 8000 + volumeMounts: + - name: config + mountPath: /code/settings.yaml + subPath: settings.yaml + volumes: + - name: config + configMap: + name: {{ $instanceName }}-config + {{- with $rootConfig.gpsdSidecar }} + {{- if .enabled }} + - name: gps-device + hostPath: + path: {{ .hostGpsDevice }} + type: CharDevice + - name: gpsd-config + configMap: + name: {{ $instanceName }}-gpsd-config + {{- end }} + {{- end }} + {{- with $.Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/sae/templates/redis-writer.configmap.yaml b/helm/sae/templates/redis-writer.configmap.yaml index d2691b7..a387b0c 100644 --- a/helm/sae/templates/redis-writer.configmap.yaml +++ b/helm/sae/templates/redis-writer.configmap.yaml @@ -1,6 +1,6 @@ {{- with .Values.redisWriter }} {{- if .enabled }} -{{- range .configs }} +{{- range .instances }} {{- $instanceName := printf "redis-writer-%s" (lower .name) }} apiVersion: v1 kind: ConfigMap @@ -8,7 +8,7 @@ metadata: name: {{ $instanceName }}-config data: settings.yaml: | - {{- toYaml . | nindent 4 }} + {{- toYaml .settingsYaml | nindent 4 }} --- {{- end }} {{- end }} diff --git a/helm/sae/templates/redis-writer.statefulset.yaml b/helm/sae/templates/redis-writer.statefulset.yaml index 1965922..aa1c638 100644 --- a/helm/sae/templates/redis-writer.statefulset.yaml +++ b/helm/sae/templates/redis-writer.statefulset.yaml @@ -3,7 +3,7 @@ {{- $imageRepo := required "image repository for redis-writer is required" .image.repository }} {{- $imageTag := required "image tag for redis-writer is required" .image.tag }} {{- $rootConfig := . }} -{{- range .configs -}} +{{- range .instances -}} {{- $instanceName := printf "redis-writer-%s" (lower .name) }} apiVersion: apps/v1 kind: StatefulSet @@ -42,8 +42,6 @@ spec: resources: {{- toYaml $.Values.resources | nindent 12 }} env: - - name: LOG_LEVEL - value: {{ default "INFO" .logLevel | quote }} - name: REDIS__HOST value: {{ include "sae.redisServiceName" $ }} - name: REDIS__PORT diff --git a/helm/sae/templates/video-source.configmap.yaml b/helm/sae/templates/video-source.configmap.yaml index 50defd5..aa613c8 100644 --- a/helm/sae/templates/video-source.configmap.yaml +++ b/helm/sae/templates/video-source.configmap.yaml @@ -1,5 +1,5 @@ {{- with .Values.videoSource }} -{{- range .configs -}} +{{- range .settingsYamls -}} {{- $instanceName := printf "video-source-%s" (lower .id) }} apiVersion: v1 kind: ConfigMap diff --git a/helm/sae/templates/video-source.statefulset.yaml b/helm/sae/templates/video-source.statefulset.yaml index ffdeec4..3fbbf49 100644 --- a/helm/sae/templates/video-source.statefulset.yaml +++ b/helm/sae/templates/video-source.statefulset.yaml @@ -2,7 +2,7 @@ {{- $imageRepo := required "image repository for video-source is required" .image.repository }} {{- $imageTag := required "image tag for video-source is required" .image.tag }} {{- $rootConfig := . }} -{{- range .configs -}} +{{- range .settingsYamls -}} {{- $instanceName := printf "video-source-%s" (lower .id) }} apiVersion: apps/v1 kind: StatefulSet @@ -41,8 +41,6 @@ spec: resources: {{- toYaml $.Values.resources | nindent 12 }} env: - - name: LOG_LEVEL - value: {{ default "INFO" $rootConfig.logLevel | quote }} - name: REDIS__HOST value: {{ include "sae.redisServiceName" $ }} - name: REDIS__PORT diff --git a/helm/sae/values.yaml b/helm/sae/values.yaml index 44773c1..ccb68e8 100644 --- a/helm/sae/values.yaml +++ b/helm/sae/values.yaml @@ -4,29 +4,57 @@ fullnameOverride: "" # extraImagePullSecrets: # - name: extra-pull-secret +positionSource: + image: + repository: starwitorg/sae-position-source + tag: 1.1.1 + enabled: false + settingsYaml: + log_level: INFO + position_source: # static source outputs the same coordinates twice a second + type: static + lat: 52 + lon: 10 + # position_source: # command source tries to read NMEA sentences from stdout of given command (this is recommended) + # type: command + # command: ["gpspipe", "-r"] # this is passed to Popen; if you want to use a bash pipeline do ["bash", "-c", "command | command2"] + # position_source: # dynamic source tries to read NMEA sentences from given serial device (i.e. USB gps receiver) + # type: dynamic + # serial_device: /dev/ttyACM0 + gpsdSidecar: + enabled: false # only enable if non-static config above + image: + repository: starwitorg/sae-gpsd + tag: 2025-09-10 + hostGpsDevice: /path/to/gpsdevice # the path to a character device gpsd can use (e.g. /dev/ttyACM0) (is passed to hostPath volume mount) + entrypointSh: | + sleep 3600 + videoSource: image: repository: starwitorg/sae-video-source-py - tag: 1.0.1 - configs: + tag: 2.3.1 + settingsYamls: - id: stream1 uri: 'uri' + log_level: INFO objectDetector: image: repository: starwitorg/sae-object-detector - tag: 3.0.0 + tag: 4.1.0 customWeights: enabled: false imageTag: weights-image-tag - logLevel: INFO - config: + settingsYaml: + log_level: INFO model: weights_path: yolov8n.pt device: cpu nms_agnostic: false - inference_size: [ 640, 640 ] - classes: null + inference_size: [ 640, 640 ] + classes: null + drop_edge_detections: false redis: stream_ids: - stream1 @@ -34,12 +62,13 @@ objectDetector: objectTracker: image: repository: starwitorg/sae-object-tracker - tag: 3.1.0 - logLevel: INFO - needsGpu: False + tag: 3.3.0 + enabled: true + needsGpu: false streamIds: - stream1 - config: + settingsYaml: + log_level: INFO tracker_algorithm: OCSORT tracker_config: det_thresh: 0.2 @@ -49,16 +78,17 @@ objectTracker: delta_t: 3 asso_func: 'iou' inertia: 0.2 - use_byte: False + use_byte: false Q_xy_scaling: 1 Q_s_scaling: 1 geoMapper: image: repository: starwitorg/sae-geo-mapper - tag: 0.7.0 + tag: 0.10.0 enabled: false - config: + settingsYaml: + log_level: INFO cameras: - stream_id: stream1 passthrough: false @@ -74,45 +104,44 @@ geoMapper: brown_distortion_k2: 0 brown_distortion_k3: 0 -geoMerger: +detectionAggregator: image: - repository: starwitorg/sae-geo-merger - tag: 0.1.0 + repository: starwitorg/detection-aggregator + tag: 0.2.7 enabled: false - config: - merging_config: - max_distance_m: 2 - merging_window_ms: 1000 - target_mps: 10 - input_stream_ids: - - stream1 - - stream2 - output_stream_id: merged + settingsYaml: + log_level: INFO + chunk: + buffer_size: 1 + time_in_ms: 20 + geo_coordinate: + latitude: 0 + longitude: 0 redisWriter: image: repository: starwitorg/sae-redis-writer - tag: 1.2.2 + tag: 2.0.2 enabled: false - configs: + instances: - name: output1 - redis: - input_stream_prefix: objecttracker - target_redis: - output_stream_prefix: saeoutput - host: host - port: 6379 - buffer_length: 10 - target_stream_maxlen: 100 - tls: false - stream_ids: - - stream1 + settingsYaml: + log_level: INFO + target_redis: + host: host + port: 6379 + buffer_length: 10 + target_stream_maxlen: 100 + tls: false + mapping_config: + - source: geomapper:stream1 + target: geomapper:mysae1 databaseWriter: image: repository: starwitorg/sae-database-writer tag: 2.0.0 - enabled: true + enabled: false config: redisStreamIds: - stream1 @@ -166,7 +195,7 @@ prometheus: relabel_configs: - source_labels: [__meta_kubernetes_pod_annotation_sae_starwit_de_stage] action: keep - regex: (video-source|object-detector|object-tracker|geo-mapper|geo-merger|redis-writer) + regex: (video-source|object-detector|object-tracker|geo-mapper|redis-writer) - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: pod diff --git a/tools/gpsd-container/Dockerfile b/tools/gpsd-container/Dockerfile new file mode 100644 index 0000000..0628f27 --- /dev/null +++ b/tools/gpsd-container/Dockerfile @@ -0,0 +1,7 @@ +FROM debian:trixie-slim + +ARG CACHE_BUST=1 + +RUN apt-get update && apt-get install -y gpsd gpsd-clients + +ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ] \ No newline at end of file diff --git a/tools/gpsd-container/TAG b/tools/gpsd-container/TAG new file mode 100644 index 0000000..5bd2d39 --- /dev/null +++ b/tools/gpsd-container/TAG @@ -0,0 +1 @@ +2025-09-10 \ No newline at end of file diff --git a/tools/gpsd-container/docker_build.sh b/tools/gpsd-container/docker_build.sh new file mode 100755 index 0000000..82de512 --- /dev/null +++ b/tools/gpsd-container/docker_build.sh @@ -0,0 +1,13 @@ +#!/bin/bash +read -p "Please enter a new version: " NEW_VERSION + +if [ ! -z "$NEW_VERSION" ]; then + TAG="$NEW_VERSION" +else + echo "No version given. Exiting." + exit 1 +fi + +docker build --build-arg CACHE_BUST="$(date)" -t starwitorg/sae-gpsd:${TAG} . + +echo -n ${TAG} > TAG \ No newline at end of file diff --git a/tools/gpsd-container/docker_push.sh b/tools/gpsd-container/docker_push.sh new file mode 100755 index 0000000..bf38ae4 --- /dev/null +++ b/tools/gpsd-container/docker_push.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ ! -f TAG ]; then + read -p "Please enter image tag to push: " TAG +else + TAG=$(cat TAG) +fi + +docker push starwitorg/sae-gpsd:${TAG} \ No newline at end of file diff --git a/tools/sae-introspection/poetry.lock b/tools/sae-introspection/poetry.lock index 8cfbcd2..1533cd6 100644 --- a/tools/sae-introspection/poetry.lock +++ b/tools/sae-introspection/poetry.lock @@ -1608,10 +1608,10 @@ files = [ [package.dependencies] numpy = [ + {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, + {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\""}, + {version = ">=1.23.5", markers = "python_version >= \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, - {version = ">=1.23.5", markers = "python_version == \"3.11\""}, - {version = ">=1.21.4", markers = "python_version == \"3.10\" and platform_system == \"Darwin\""}, - {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version == \"3.10\""}, ] [[package]] @@ -1638,6 +1638,93 @@ files = [ {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] +[[package]] +name = "pandas" +version = "2.3.2" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pandas-2.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:52bc29a946304c360561974c6542d1dd628ddafa69134a7131fdfd6a5d7a1a35"}, + {file = "pandas-2.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:220cc5c35ffaa764dd5bb17cf42df283b5cb7fdf49e10a7b053a06c9cb48ee2b"}, + {file = "pandas-2.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42c05e15111221384019897df20c6fe893b2f697d03c811ee67ec9e0bb5a3424"}, + {file = "pandas-2.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc03acc273c5515ab69f898df99d9d4f12c4d70dbfc24c3acc6203751d0804cf"}, + {file = "pandas-2.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d25c20a03e8870f6339bcf67281b946bd20b86f1a544ebbebb87e66a8d642cba"}, + {file = "pandas-2.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:21bb612d148bb5860b7eb2c10faacf1a810799245afd342cf297d7551513fbb6"}, + {file = "pandas-2.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:b62d586eb25cb8cb70a5746a378fc3194cb7f11ea77170d59f889f5dfe3cec7a"}, + {file = "pandas-2.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1333e9c299adcbb68ee89a9bb568fc3f20f9cbb419f1dd5225071e6cddb2a743"}, + {file = "pandas-2.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:76972bcbd7de8e91ad5f0ca884a9f2c477a2125354af624e022c49e5bd0dfff4"}, + {file = "pandas-2.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b98bdd7c456a05eef7cd21fd6b29e3ca243591fe531c62be94a2cc987efb5ac2"}, + {file = "pandas-2.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d81573b3f7db40d020983f78721e9bfc425f411e616ef019a10ebf597aedb2e"}, + {file = "pandas-2.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e190b738675a73b581736cc8ec71ae113d6c3768d0bd18bffa5b9a0927b0b6ea"}, + {file = "pandas-2.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c253828cb08f47488d60f43c5fc95114c771bbfff085da54bfc79cb4f9e3a372"}, + {file = "pandas-2.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:9467697b8083f9667b212633ad6aa4ab32436dcbaf4cd57325debb0ddef2012f"}, + {file = "pandas-2.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fbb977f802156e7a3f829e9d1d5398f6192375a3e2d1a9ee0803e35fe70a2b9"}, + {file = "pandas-2.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b9b52693123dd234b7c985c68b709b0b009f4521000d0525f2b95c22f15944b"}, + {file = "pandas-2.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bd281310d4f412733f319a5bc552f86d62cddc5f51d2e392c8787335c994175"}, + {file = "pandas-2.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96d31a6b4354e3b9b8a2c848af75d31da390657e3ac6f30c05c82068b9ed79b9"}, + {file = "pandas-2.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:df4df0b9d02bb873a106971bb85d448378ef14b86ba96f035f50bbd3688456b4"}, + {file = "pandas-2.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:213a5adf93d020b74327cb2c1b842884dbdd37f895f42dcc2f09d451d949f811"}, + {file = "pandas-2.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:8c13b81a9347eb8c7548f53fd9a4f08d4dfe996836543f805c987bafa03317ae"}, + {file = "pandas-2.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0c6ecbac99a354a051ef21c5307601093cb9e0f4b1855984a084bfec9302699e"}, + {file = "pandas-2.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c6f048aa0fd080d6a06cc7e7537c09b53be6642d330ac6f54a600c3ace857ee9"}, + {file = "pandas-2.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0064187b80a5be6f2f9c9d6bdde29372468751dfa89f4211a3c5871854cfbf7a"}, + {file = "pandas-2.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ac8c320bded4718b298281339c1a50fb00a6ba78cb2a63521c39bec95b0209b"}, + {file = "pandas-2.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:114c2fe4f4328cf98ce5716d1532f3ab79c5919f95a9cfee81d9140064a2e4d6"}, + {file = "pandas-2.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:48fa91c4dfb3b2b9bfdb5c24cd3567575f4e13f9636810462ffed8925352be5a"}, + {file = "pandas-2.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:12d039facec710f7ba305786837d0225a3444af7bbd9c15c32ca2d40d157ed8b"}, + {file = "pandas-2.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:c624b615ce97864eb588779ed4046186f967374185c047070545253a52ab2d57"}, + {file = "pandas-2.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:0cee69d583b9b128823d9514171cabb6861e09409af805b54459bd0c821a35c2"}, + {file = "pandas-2.3.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2319656ed81124982900b4c37f0e0c58c015af9a7bbc62342ba5ad07ace82ba9"}, + {file = "pandas-2.3.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b37205ad6f00d52f16b6d09f406434ba928c1a1966e2771006a9033c736d30d2"}, + {file = "pandas-2.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:837248b4fc3a9b83b9c6214699a13f069dc13510a6a6d7f9ba33145d2841a012"}, + {file = "pandas-2.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d2c3554bd31b731cd6490d94a28f3abb8dd770634a9e06eb6d2911b9827db370"}, + {file = "pandas-2.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:88080a0ff8a55eac9c84e3ff3c7665b3b5476c6fbc484775ca1910ce1c3e0b87"}, + {file = "pandas-2.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d4a558c7620340a0931828d8065688b3cc5b4c8eb674bcaf33d18ff4a6870b4a"}, + {file = "pandas-2.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45178cf09d1858a1509dc73ec261bf5b25a625a389b65be2e47b559905f0ab6a"}, + {file = "pandas-2.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77cefe00e1b210f9c76c697fedd8fdb8d3dd86563e9c8adc9fa72b90f5e9e4c2"}, + {file = "pandas-2.3.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:13bd629c653856f00c53dc495191baa59bcafbbf54860a46ecc50d3a88421a96"}, + {file = "pandas-2.3.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:36d627906fd44b5fd63c943264e11e96e923f8de77d6016dc2f667b9ad193438"}, + {file = "pandas-2.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:a9d7ec92d71a420185dec44909c32e9a362248c4ae2238234b76d5be37f208cc"}, + {file = "pandas-2.3.2.tar.gz", hash = "sha256:ab7b58f8f82706890924ccdfb5f48002b83d2b5a3845976a9fb705d36c34dcdb"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + [[package]] name = "pandocfilters" version = "1.5.1" @@ -2251,6 +2338,18 @@ files = [ [package.dependencies] numpy = "*" +[[package]] +name = "pytz" +version = "2025.2" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, + {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, +] + [[package]] name = "pywin32" version = "311" @@ -3022,6 +3121,18 @@ files = [ [package.dependencies] typing-extensions = ">=4.12.0" +[[package]] +name = "tzdata" +version = "2025.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +files = [ + {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, + {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, +] + [[package]] name = "uri-template" version = "1.3.0" @@ -3181,4 +3292,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "e5eecfe05b256bd4bbd350c8ff1589a7ad89c5df9fb41480809ca19d01d9486f" +content-hash = "acaeb1cb13f925456f2e8df2de14260db452d783b0b1b0306dc767805f4b7387" diff --git a/tools/sae-introspection/pyproject.toml b/tools/sae-introspection/pyproject.toml index 5ccfdd1..fd9b994 100644 --- a/tools/sae-introspection/pyproject.toml +++ b/tools/sae-introspection/pyproject.toml @@ -27,6 +27,7 @@ pyturbojpeg = "^1.8.2" ipyleaflet = "^0.20.0" tempora = "^5.8.1" pydeck = "^0.9.1" +pandas = "^2.3.2" [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] diff --git a/tools/sae-introspection/waste.ipynb b/tools/sae-introspection/waste.ipynb index 0e688e0..5aab0f3 100644 --- a/tools/sae-introspection/waste.ipynb +++ b/tools/sae-introspection/waste.ipynb @@ -13,292838 +13,7 @@ "execution_count": null, "id": "03636bf0", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from collections import defaultdict\n", "from typing import List, NamedTuple\n", @@ -292852,12 +21,14 @@ "import pandas as pd\n", "import pybase64\n", "import pydeck\n", + "from datetime import datetime\n", "from pydeck.types import String\n", "from visionapi.sae_pb2 import SaeMessage\n", "from visionlib.saedump import DumpMeta, Event, message_splitter\n", "\n", "\n", "class PointSample(NamedTuple):\n", + " timestamp: datetime\n", " lat: float\n", " lon: float\n", " count: int\n", @@ -292885,9 +56,9 @@ " counts[det.class_id] += 1\n", "\n", " cam_loc = sae_msg.frame.camera_location\n", - " samples.append(PointSample(cam_loc.latitude, cam_loc.longitude, sum(counts.values())))\n", + " samples.append(PointSample(datetime.fromtimestamp(sae_msg.frame.timestamp_utc_ms / 1000), cam_loc.latitude, cam_loc.longitude, sum(counts.values())))\n", " \n", - "df_samples = pd.DataFrame(samples, columns=['lat', 'lon', 'count'])\n", + "df_samples = pd.DataFrame(samples, columns=['timestamp', 'lat', 'lon', 'count'])\n", "\n", "grid_layer = pydeck.Layer(\n", " 'GridLayer',\n", @@ -292912,7 +83,19 @@ " layers=[grid_layer], \n", " initial_view_state=view_state,\n", " height=800,\n", - ").to_html(notebook_display=True, iframe_height=800)\n" + ").to_html(notebook_display=True, iframe_height=800)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d45c915", + "metadata": {}, + "outputs": [], + "source": [ + "df_samples.describe(percentiles=[0.8,0.9,0.95,0.98, 0.99])\n", + "df_samples.nlargest(50, 'count')" ] } ], diff --git a/tools/sae-object-detector-weights/entrypoint.sh b/tools/sae-object-detector-weights/entrypoint.sh index f3994f6..d0bc0c0 100755 --- a/tools/sae-object-detector-weights/entrypoint.sh +++ b/tools/sae-object-detector-weights/entrypoint.sh @@ -2,4 +2,4 @@ # This scripts purpose is to copy all the files in /weights into /weights_target -cp /weights/* /weights_target/ && echo -e "Successfully copied weights:\n$(ls /weights)" \ No newline at end of file +cp -r /weights/* /weights_target/ && echo -e "Successfully copied weights:\n$(ls /weights)" \ No newline at end of file