diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..2c6d6ff --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,37 @@ +on: [push, pull_request] +name: Supported Python Compatibility Test +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-2019] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + include: + - os: macos-latest + - os: ubuntu-latest + - os: windows-2019 + fail-fast: false + name: Python ${{ matrix.python-version }} ${{ matrix.os }} build + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' # caching pip dependencies + - name: Install requirements + run: | + pip install tox + - uses: actions/github-script@v7 + id: tox-env + with: + result-encoding: string + script: | + const frontend = "${{matrix.frontend}}" + const toxEnv = "py${{matrix.python-version}}".replace('.','') + if(frontend === ""){ + return toxEnv + } + return "py${{matrix.python-version}}".replace('.','') + "-${{matrix.frontend}}" + - name: Run tox + run: tox -e ${{ steps.tox-env.outputs.result }} diff --git a/README.md b/README.md index 602204b..88930a5 100644 --- a/README.md +++ b/README.md @@ -1,102 +1,211 @@ -scripts for automating QCTools actions -qct-parse.py | find frames that are beyond thresholds for saturation, luma, etc +# QCTools Automation Scripts + +This repository contains scripts for automating analysis of QCTools reports. + +## Overview + +### Install from source: + +* Create a new Python Virtual Environment for qct_parse + * Unix based (Mac or Linux): + `python3 -m venv name_of_env` + * Windows: + `py -m venv name_of_env` + (where 'name_of_env' is replaced with the name of your virtual environment) +* Activate virtual env + * Unix based (Mac or Linux): + `source ./name_of_env/bin/activate` + * Windows: + `name_of_env\scripts\activate` +* Install Package + * Navigate to the repo root directory `path/to/qct-parse/` + * Run the command: + `python -m pip install .` + +### Test Code + +If you intend to develop the code for your proposes or contribute to the open source project, a test directory is provided in the repo. +* Activate virtual env (see Install from source) +* Install pytest + `pip install pytest` +* Run tests + `python -m pytest` + +### Building the documentation +* Activate virtual env (see Install from source) +* Install sphinx and myst-parser + `pip install sphinx myst-parser` +* Run sphinx-build command + `sphinx-build ./docs ./dist/docs` + + +### Commands: + +- **`qct-parse -i/--input [path to QCTools report] [optional arguments]`** + Finds frames that exceed thresholds for QCTool tag(s). Full list of command line arguments below. + +--- + +# `qct-parse` + +Run a single tag against a supplied value or multiple tags using a profile. + +## Arguments -makeqctoolsreport.py | make a qctools.xml.gz report for input video file +| Argument | Description | +|-----------------------------|-------------------------------------------------------------------------------------------------------| +| `-h`, `--help` | Show this help message and exit | +| `-i`, `--input` | Path to the input `qctools.xml.gz` or `qctools.mkv` file | +| `-t`, `--tagname` | The tag name you want to test (e.g., `SATMAX`); see table of tag names below for list | +| `-o`, `--over` | Threshold overage number | +| `-u`, `--under` | Threshold under number | +| `-p`, `--profile` | Compare frame data against tag values from `config.txt`. Use `-p default` for QCTools default values | +| `-buff`, `--buffSize` | Circular buffer size. If even, defaults to the next odd number (default: 11) | +| `-te`, `--thumbExport` | Enable/disable thumbnail export (default: off) | +| `-ted`, `--thumbExportDelay`| Minimum frames between exported thumbnails (default: 9000) | +| `-tep`, `--thumbExportPath` | Path to thumbnail export. Uses input base-path if omitted | +| `-ds`, `--durationStart` | Start analysis from this time (seconds, equivalent to ffmpeg `-ss`) | +| `-de`, `--durationEnd` | End analysis after this time (seconds, equivalent to ffmpeg `-t`) | +| `-bd`, `--barsDetection` | Enable/disable bar detection (default: off) | +| `-be`, `--barsEvaluation` | Use peak values from color bars as 'profile' if bars are detected | +| `-pr`, `--print` | Print over/under frame data to console (default: off) | +| `-q`, `--quiet` | Suppress ffmpeg output in console (default: off) | -# qct-parse.py +## Tags -You can run a single tag against a supplied value from the CLI or run multiple tags against values set in the qct-parse_config.txt file. +| Tag category | Tag names | +|-----------------------------|-------------------------------------------------------------------------------------------------------| +| [YUV values](https://bavc.github.io/qctools/filter_descriptions.html#yuv) | `YMIN,YLOW,YAVG,YHIGH,YMAX`
`UMIN,ULOW,UAVG,UHIGH,UMAX`
`VMIN,VLOW,VAVG,VHIGH,VMAX` | +| [YUV values (difference)](https://bavc.github.io/qctools/filter_descriptions.html#diff) | `YDIF,UDIF,VDIF` | +| [Saturation values](https://bavc.github.io/qctools/filter_descriptions.html#saturation) | `SATMIN,SATLOW,SATAVG,SATHIGH,SATMAX` | +| [Hue values](https://bavc.github.io/qctools/filter_descriptions.html#hue) | `HUEMED,HUEAVG` | +| [Temporal outliers](https://bavc.github.io/qctools/filter_descriptions.html#tout) | `TOUT` | +| [Vertical line repetitions](https://bavc.github.io/qctools/filter_descriptions.html#vrep) | `VREP` | +| [Broadcast range](https://bavc.github.io/qctools/filter_descriptions.html#brng) | `BRNG` | +| [Mean square error fields](https://bavc.github.io/qctools/filter_descriptions.html#msef) | `mse_y,mse_u,mse_v,mse_avg` | +| [Peak signal to noise ratio fields](https://bavc.github.io/qctools/filter_descriptions.html#psnrf) | `psnr_y,psnr_u,psnr_v,psnr_avg` | -## arguments - -h, --help | show this help message and exit +## Examples - -i, --input | the path to the input qctools.xml.gz file - - -t, --tagname | the tag name you want to test, e.g. SATMAX - - -o, --over | the threshold overage number - - -u, --under | the threshold under number - - -p, --profile | compare frame data aginst tag values from config.txt file, us "-p default" for values from QCTools docs - - -buff, --buffSize | Size of the circular buffer. if user enters an even number it'll default to the next largest number to make it odd, default size 11 - - -te, --thumbExport | export thumbnails on/ off, default off - - -ted, --thumbExportDelay | minimum frames between exported thumbs, default 9000 - - -tep, --thumbExportPath | Path to thumb export. if ommitted, uses the input base-path - - -ds, --durationStart | the duration in seconds to start analysis (ffmpeg equivalent -ss) - - -de, --durationEnd | the duration in seconds to stop analysis (ffmpeg equivalent -t) - - -bd, --barsDetection | bar detection on/ off, default off - - -pr, --print | print over/under frame data to console window, default off - - -q, --quiet | print ffmpeg output to console window, default off +### Run single tag tests +```bash +qct-parse -t SATMAX -o 235 -t YMIN -u 16 -i /path/to/report.mkv.qctools.xml.gz +``` +### Run bars detection using default QCTools profile +```bash +qct-parse -bd -p default -i /path/to/report.mkv.qctools.mkv +``` -## examples +### Export thumbnails of frames beyond threshold +```bash +qct-parse -p default -te -tep /path/to/export/folder -i /path/to/report.mkv.qctools.xml.gz +``` -### single tags +### Use peak values from detected color bars as thresholds +```bash +qct-parse -bd -be -i /path/to/report.mkv.qctools.xml.gz +``` -python qct-parse.py -t SATMAX -o 235 -t YMIN -u 16 -i /path/to/report.mkv.qctools.xml.gz +## Input files -### run bars against default profile from QCTools docs +qct-parse.py will work with the following QCTools report formats: +``` +qctools.xml.gz +qctools.mkv +``` -python qct-parse.py -bd -p default -i /path/to/reportsmkv.qctools.xml.gz +If the qctools.xml.gz report is in an MKV attachment, the qctools.xml.gz report file will be extracted and saved as a separate file. -### print out thumbnails of frames beyond threshold +Both 8-bit and 10-bit values are supported. The bit depth will be detected automatically, and does not need to be specified. -python qct-parse.py -p default -te -tep C:\path\to\export\folder -i C:\path\to\the\report.mkv.qctools.xml.gz +## Config Files -## some handy applescript to grep individual tags +The provided profiles are: +* default +* highTolerance +* midTolerance +* lowTolerance -### just percentages +Each of these profiles contain the following tags with a corresponding threshold: +`YLOW, YMAX, UMIN, UMAX, VMIN, VMAX, SATMAX, TOUT, VREP` -python ./qct-parse.py -i input.mxf.qctools.xml.gz -bd -p lowTolerance | grep 'YMAX' | awk 'NR==1 {print $3}' +The profiles are stored in the config.txt files. Please note that there is a separate config.txt for 8-bit and 10-bit values. -### total number of frame failures +The process for providing user supplied profiles is in development. +Currently, if you wish to create your own profile, you will need to create your own config directory and `config.txt` file. +There is a environmental variable at the top of qct-parse.py which can be used to reset the config directory: +```bash +CONFIG_ENVIRONMENT_VARIABLE_NAME = 'QCT_PARSE_CONFIG_DIRECTORY' +``` +Simply place the full path to the user created config *directory* in place of 'QCT_PARSE_CONFIG_DIRECTORY' -python ./qct-parse.py -i input.mxf.qctools.xml.gz -bd -p lowTolerance | grep 'YMAX' | awk 'NR==1 {print $2}' +## Thumbnails -## dependencies +Thumbnails of failed frames will be exported if the `-te` flag is invoked. -Python 2.7.x. +In order to export thumbnails, the QCTools report must be in the same directory as the video file it is describing, and must have the same file name as the report (excluding the `qctools.xml.gz`). -Requires that [lxml](http://lxml.de/) is installed on your system. For more info on how it's used, see [here](http://www.ibm.com/developerworks/library/x-hiperfparse/) +If you would like to provide a path for exporting thumbnails, you can do so using the `-tep` flag. +Otherwise, thumbnails will automatically be created in the same directory as the video file and QCTools report, in a new directory. -### For Windows users: +When running qct-parse with a profile, the thumbnails will be placed in a directory named `ThumbExports`. +When run against single tags the directory will be named [TAG NAME].[THRESHOLD] -We **strongly** suggest using the pre-compiled installer found [here](https://pypi.python.org/pypi/lxml/3.3.3#downloads) +## Logging -### For Mac users: +A log file is created with the same name as the input file but with a '.log' extension. +For example: `some_video_file.mkv.qctools.xml.gz.log` -Try pip first, then try the macport. More info can be found [here](http://lxml.de/installation.html) +Log files contain every instance of values over the specified threshold. For example: +`2024-10-03 17:02:35,737 SATMAX is over 181.02 with a value of 698.0 at duration 00:00:16.4500` -# makeqctoolsreport.py +--- -python port of Morgan's [makeqctoolsreport.as](https://github.com/iamdamosuzuki/QCToolsReport) +### Legacy Commands: +Not in active development. Please file an issue if you are interested in using these. -## example +#### `makeqctoolsreport` -python makeqctoolsreport.py /path/to/input.mxf +A Python port of Morgan’s [makeqctoolsreport.as](https://github.com/iamdamosuzuki/QCToolsReport), this script generates QCTools `.xml.gz` reports from input video files. -## contributors +Example Usage: +```bash +makeqctoolsreport /path/to/input.mxf +``` -@eddycolloton +#### `overcatch` -@CoatesBrendan +A script from the original qct-parse development for running a report against multiple profiles. -@av_morgan +Example Usage: +```bash +overcatch /path/to/input.mxf +``` -## maintainers +--- -@av_morgan +## Dependencies -@eddycolloton +Ensure Python 3.x.x is installed. + +Requires FFmpeg. + +This tool uses the `lxml` python module which is automatically installed with the qct-parse package. + +For more information on `lxml` usage, check out the [lxml documentation](http://lxml.de/). + +--- + +## Contributors + +- [@eddycolloton](https://github.com/eddycolloton) +- [@CoatesBrendan](https://github.com/CoatesBrendan) +- [@av_morgan](https://github.com/av_morgan) + +## Maintainer + +- [@av_morgan](https://github.com/av_morgan) +- [@eddycolloton](https://github.com/eddycolloton) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..58ed402 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,31 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'qct-parse' +copyright = '2024, AMIA Open-Source' +author = 'AMIA Open-Source' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ['myst_parser'] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +source_suffix = { + '.rst': 'restructuredtext', + '.txt': 'markdown', + '.md': 'markdown', +} + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'alabaster' +html_static_path = ['_static'] diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..227caeb --- /dev/null +++ b/docs/development.md @@ -0,0 +1,29 @@ +# Development Information + +## Configure Development Environment + +* Create a new Python Virtual Environment for qct_parse + * Unix based (Mac or Linux): + `python3 -m venv name_of_env` + * Windows: + `py -m venv name_of_env` + (where 'name_of_env' is replaced with the name of your virtual environment) +* Activate virtual env + * Unix based (Mac or Linux): + `source ./name_of_env/bin/activate` + * Windows: + `name_of_env\scripts\activate` +* Install Package as editable package + * Navigate to the repo root directory `path/to/qct-parse/` + * Run the command: + `python -m pip install -e .` + +## Run Tests + +If you intend to develop the code for your proposes or contribute to the open source project, a test directory is provided in the repo. +* Activate virtual env (see Configure Development Environment) +* Install pytest + `pip install pytest` +* Run tests + `python -m pytest` + diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..7ebe75d --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,14 @@ +.. qct-parse documentation master file, created by + sphinx-quickstart on Tue Dec 3 20:53:22 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +qct-parse documentation +======================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + readme + development diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..954237b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 0000000..339dc8f --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1,2 @@ +.. include:: ../README.md + :parser: myst_parser.sphinx_ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0de5939 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "qct-parse" +version = "0.2.1.dev0" +dependencies =[ + 'lxml' +] + +[tool.setuptools] +packages=['qct_parse'] +include-package-data = true + +[tool.setuptools.package-data] +qct_parse = ["./qct-parse_8bit_config.txt", "./qct-parse_10bit_config.txt"] + +[project.scripts] +makeqctoolsreport = "qct_parse.makeqctoolsreport:main" +overcatch = "qct_parse.overcatch:main" +qct-parse = "qct_parse.qct_parse:main" + +[tool.tox] +envlist = ["3.9", "3.10", "3.11", "3.12", "3.13"] + +[tool.tox.env_run_base] +description = "Run test under {base_python}" +commands = [["pytest"]] +deps = ["pytest"] diff --git a/qct-parse.py b/qct-parse.py deleted file mode 100644 index 8115b0e..0000000 --- a/qct-parse.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python -#qct-parse - changes made for python3 compatibility. WIP. - -#see this link for lxml goodness: http://www.ibm.com/developerworks/xml/library/x-hiperfparse/ - -from lxml import etree #for reading XML file (you will need to install this with pip) -import argparse #for parsing input args -import configparser #grip frame data values from a config txt file -import gzip #for opening gzip file -import logging #for logging output -import collections #for circular buffer -import os #for running ffmpeg and other terminal commands -import subprocess #not currently used -import gc #not currently used -import math #used for rounding up buffer half -import sys #system stuff -import re #can't spell parse without re fam -from distutils import spawn #dependency checking - -#check that we have required software installed -def dependencies(): - depends = ['ffmpeg','ffprobe'] - for d in depends: - if spawn.find_executable(d) is None: - print("Buddy, you gotta install " + d) - sys.exit() - return - -#Creates timestamp for pkt_dts_time -def dts2ts(frame_pkt_dts_time): - seconds = float(frame_pkt_dts_time) - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - if hours < 10: - hours = "0" + str(int(hours)) - else: - hours = str(int(hours)) - if minutes < 10: - minutes = "0" + str(int(minutes)) - else: - minutes = str(int(minutes)) - secondsStr = str(round(seconds,4)) - if int(seconds) < 10: - secondsStr = "0" + secondsStr - else: - seconds = str(minutes) - while len(secondsStr) < 7: - secondsStr = secondsStr + "0" - timeStampString = hours + ":" + minutes + ":" + secondsStr - return timeStampString - -#initializes the log -def initLog(inputPath): - logPath = inputPath + '.log' - logging.basicConfig(filename=logPath,level=logging.INFO,format='%(asctime)s %(message)s') - logging.info("Started QCT-Parse") - -#finds stuff over/under threshold -def threshFinder(inFrame,args,startObj,pkt,tag,over,thumbPath,thumbDelay): - tagValue = float(inFrame[tag]) - frame_pkt_dts_time = inFrame[pkt] - if "MIN" in tag or "LOW" in tag: - under = over - if tagValue < float(under): #if the attribute is under usr set threshold - timeStampString = dts2ts(frame_pkt_dts_time) - logging.warning(tag + " is under " + str(under) + " with a value of " + str(tagValue) + " at duration " + timeStampString) - if args.te and (thumbDelay > int(args.ted)): #if thumb export is turned on and there has been enough delay between this frame and the last exported thumb, then export a new thumb - printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString) - thumbDelay = 0 - return True, thumbDelay #return true because it was over and thumbDelay - else: - return False, thumbDelay #return false because it was NOT over and thumbDelay - else: - if tagValue > float(over): #if the attribute is over usr set threshold - timeStampString = dts2ts(frame_pkt_dts_time) - logging.warning(tag + " is over " + str(over) + " with a value of " + str(tagValue) + " at duration " + timeStampString) - if args.te and (thumbDelay > int(args.ted)): #if thumb export is turned on and there has been enough delay between this frame and the last exported thumb, then export a new thumb - printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString) - thumbDelay = 0 - return True, thumbDelay #return true because it was over and thumbDelay - else: - return False, thumbDelay #return false because it was NOT over and thumbDelay - -#print thumbnail images of overs/unders -#Need to update - file naming convention has changed -def printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString): - ####init some variables using the args list - inputVid = startObj.replace(".qctools.xml.gz", "") - if os.path.isfile(inputVid): - baseName = os.path.basename(startObj) - baseName = baseName.replace(".qctools.xml.gz", "") - outputFramePath = os.path.join(thumbPath,baseName + "." + tag + "." + str(tagValue) + "." + timeStampString + ".png") - ffoutputFramePath = outputFramePath.replace(":",".") - #for windows we gotta see if that first : for the drive has been replaced by a dot and put it back - match = '' - match = re.search(r"[A-Z]\.\/",ffoutputFramePath) #matches pattern R./ which should be R:/ on windows - if match: - ffoutputFramePath = ffoutputFramePath.replace(".",":",1) #replace first instance of "." in string ffoutputFramePath - ffmpegString = "ffmpeg -ss " + timeStampString + ' -i "' + inputVid + '" -vframes 1 -s 720x486 -y "' + ffoutputFramePath + '"' #Hardcoded output frame size to 720x486 for now, need to infer from input eventually - output = subprocess.Popen(ffmpegString,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True) - out,err = output.communicate() - if args.q is False: - print(out) - print(err) - else: - print("Input video file not found. Ensure video file is in the same directory as the QCTools report.") - exit() - return - -#detect bars -def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize): - with gzip.open(startObj) as xml: - for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): #iterparse the xml doc - if elem.attrib['media_type'] == "video": #get just the video frames - frame_pkt_dts_time = elem.attrib[pkt] #get the timestamps for the current frame we're looking at - frameDict = {} #start an empty dict for the new frame - frameDict[pkt] = frame_pkt_dts_time #give the dict the timestamp, which we have now - for t in list(elem): #iterating through each attribute for each element - keySplit = t.attrib['key'].split(".") #split the names by dots - keyName = str(keySplit[-1]) #get just the last word for the key name - frameDict[keyName] = t.attrib['value'] #add each attribute to the frame dictionary - framesList.append(frameDict) - middleFrame = int(round(float(len(framesList))/2)) #i hate this calculation, but it gets us the middle index of the list as an integer - if len(framesList) == buffSize: #wait till the buffer is full to start detecting bars - ##This is where the bars detection magic actually happens - bufferRange = list(range(0, buffSize)) - if float(framesList[middleFrame]['YMAX']) > 210 and float(framesList[middleFrame]['YMIN']) < 10 and float(framesList[middleFrame]['YDIF']) < 3.0: - if durationStart == "": - durationStart = float(framesList[middleFrame][pkt]) - print("Bars start at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") - durationEnd = float(framesList[middleFrame][pkt]) - else: - print("Bars ended at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") - break - elem.clear() #we're done with that element so let's get it outta memory - return durationStart, durationEnd - -def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,frameCount=0,overallFrameFail=0): - kbeyond = {} #init a dict for each key which we'll use to track how often a given key is over - fots = "" - if args.t: - kbeyond[args.t] = 0 - else: - for k,v in profile.items(): - kbeyond[k] = 0 - with gzip.open(startObj) as xml: - for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): #iterparse the xml doc - if elem.attrib['media_type'] == "video": #get just the video frames - frameCount = frameCount + 1 - frame_pkt_dts_time = elem.attrib[pkt] #get the timestamps for the current frame we're looking at - if frame_pkt_dts_time >= str(durationStart): #only work on frames that are after the start time - if durationEnd: - if float(frame_pkt_dts_time) > durationEnd: #only work on frames that are before the end time - print("started at " + str(durationStart) + " seconds and stopped at " + str(frame_pkt_dts_time) + " seconds (" + dts2ts(frame_pkt_dts_time) + ") or " + str(frameCount) + " frames!") - break - frameDict = {} #start an empty dict for the new frame - frameDict[pkt] = frame_pkt_dts_time #make a key for the timestamp, which we have now - for t in list(elem): #iterating through each attribute for each element - keySplit = t.attrib['key'].split(".") #split the names by dots - keyName = str(keySplit[-1]) #get just the last word for the key name - if len(keyName) == 1: #if it's psnr or mse, keyName is gonna be a single char - keyName = '.'.join(keySplit[-2:]) #full attribute made by combining last 2 parts of split with a period in btw - frameDict[keyName] = t.attrib['value'] #add each attribute to the frame dictionary - framesList.append(frameDict) #add this dict to our circular buffer - if args.pr is True: #display "timestamp: Tag Value" (654.754100: YMAX 229) to the terminal window - print(framesList[-1][pkt] + ": " + args.t + " " + framesList[-1][args.t]) - #Now we can parse the frame data from the buffer! - if args.o or args.u and args.p is None: #if we're just doing a single tag - tag = args.t - if args.o: - over = float(args.o) - if args.u: - over = float(args.u) - #ACTAULLY DO THE THING ONCE FOR EACH TAG - frameOver, thumbDelay = threshFinder(framesList[-1],args,startObj,pkt,tag,over,thumbPath,thumbDelay) - if frameOver is True: - kbeyond[tag] = kbeyond[tag] + 1 #note the over in the keyover dictionary - elif args.p is not None: #if we're using a profile - for k,v in profile.items(): - tag = k - over = float(v) - #ACTUALLY DO THE THING ONCE FOR EACH TAG - frameOver, thumbDelay = threshFinder(framesList[-1],args,startObj,pkt,tag,over,thumbPath,thumbDelay) - if frameOver is True: - kbeyond[k] = kbeyond[k] + 1 #note the over in the key over dict - if not frame_pkt_dts_time in fots: #make sure that we only count each over frame once - overallFrameFail = overallFrameFail + 1 - fots = frame_pkt_dts_time #set it again so we don't dupe - thumbDelay = thumbDelay + 1 - elem.clear() #we're done with that element so let's get it outta memory - return kbeyond, frameCount, overallFrameFail - - -#This function is admittedly very ugly, but what it puts out is very pretty. Need to revamp -def printresults(kbeyond,frameCount,overallFrameFail): - if frameCount == 0: - percentOverString = "0" - else: - print("") - print("TotalFrames:\t" + str(frameCount)) - print("") - print("By Tag:") - print("") - percentOverall = float(overallFrameFail) / float(frameCount) - if percentOverall == 1: - percentOverallString = "100" - elif percentOverall == 0: - percentOverallString = "0" - elif percentOverall < 0.0001: - percentOverallString = "<0.01" - else: - percentOverallString = str(percentOverall) - percentOverallString = percentOverallString[2:4] + "." + percentOverallString[4:] - if percentOverallString[0] == "0": - percentOverallString = percentOverallString[1:] - percentOverallString = percentOverallString[:4] - else: - percentOverallString = percentOverallString[:5] - for k,v in kbeyond.items(): - percentOver = float(kbeyond[k]) / float(frameCount) - if percentOver == 1: - percentOverString = "100" - elif percentOver == 0: - percentOverString = "0" - elif percentOver < 0.0001: - percentOverString = "<0.01" - else: - percentOverString = str(percentOver) - percentOverString = percentOverString[2:4] + "." + percentOverString[4:] - if percentOverString[0] == "0": - percentOverString = percentOverString[1:] - percentOverString = percentOverString[:4] - else: - percentOverString = percentOverString[:5] - print(k + ":\t" + str(kbeyond[k]) + "\t" + percentOverString + "\t% of the total # of frames") - print("") - print("Overall:") - print("") - print("Frames With At Least One Fail:\t" + str(overallFrameFail) + "\t" + percentOverallString + "\t% of the total # of frames") - print("") - print("**************************") - print("") - return - -def main(): - ####init the stuff from the cli######## - parser = argparse.ArgumentParser(description="parses QCTools XML files for frames beyond broadcast values") - parser.add_argument('-i','--input',dest='i', help="the path to the input qctools.xml.gz file") - parser.add_argument('-t','--tagname',dest='t', help="the tag name you want to test, e.g. SATMAX") - parser.add_argument('-o','--over',dest='o', help="the threshold overage number") - parser.add_argument('-u','--under',dest='u', help="the threshold under number") - parser.add_argument('-p','--profile',dest='p',default=None,help="use values from your qct-parse-config.txt file, provide profile/ template name, e.g. 'default'") - parser.add_argument('-buff','--buffSize',dest='buff',default=11, help="Size of the circular buffer. if user enters an even number it'll default to the next largest number to make it odd (default size 11)") - parser.add_argument('-te','--thumbExport',dest='te',action='store_true',default=False, help="export thumbnail") - parser.add_argument('-ted','--thumbExportDelay',dest='ted',default=9000, help="minimum frames between exported thumbs") - parser.add_argument('-tep','--thumbExportPath',dest='tep',default='', help="Path to thumb export. if ommitted, it uses the input basename") - parser.add_argument('-ds','--durationStart',dest='ds',default=0, help="the duration in seconds to start analysis") - parser.add_argument('-de','--durationEnd',dest='de',default=99999999, help="the duration in seconds to stop analysis") - parser.add_argument('-bd','--barsDetection',dest='bd',action ='store_true',default=False, help="turns Bar Detection on and off") - parser.add_argument('-pr','--print',dest='pr',action='store_true',default=False, help="print over/under frame data to console window") - parser.add_argument('-q','--quiet',dest='q',action='store_true',default=False, help="hide ffmpeg output from console window") - args = parser.parse_args() - - - ######Initialize values from the Config Parser - profile = {} #init a dictionary where we'll store reference values from our config file - #init a list of every tag available in a QCTools Report - tagList = ["YMIN","YLOW","YAVG","YHIGH","YMAX","UMIN","ULOW","UAVG","UHIGH","UMAX","VMIN","VLOW","VAVG","VHIGH","VMAX","SATMIN","SATLOW","SATAVG","SATHIGH","SATMAX","HUEMED","HUEAVG","YDIF","UDIF","VDIF","TOUT","VREP","BRNG","mse_y","mse_u","mse_v","mse_avg","psnr_y","psnr_u","psnr_v","psnr_avg"] - if args.p is not None: - config = configparser.RawConfigParser(allow_no_value=True) - dn, fn = os.path.split(os.path.abspath(__file__)) #grip the dir where ~this script~ is located, also where config.txt should be located - config.read(os.path.join(dn,"qct-parse_config.txt")) #read in the config file - template = args.p #get the profile/ section name from CLI - for t in tagList: #loop thru every tag available and - try: #see if it's in the config section - profile[t.replace("_",".")] = config.get(template,t) #if it is, replace _ necessary for config file with . which xml attributes use, assign the value in config - except: #if no config tag exists, do nothing so we can move faster - pass - - ######Initialize some other stuff###### - startObj = args.i.replace("\\","/") - buffSize = int(args.buff) #cast the input buffer as an integer - if buffSize%2 == 0: - buffSize = buffSize + 1 - initLog(startObj) #initialize the log - overcount = 0 #init count of overs - undercount = 0 #init count of unders - count = 0 #init total frames counter - framesList = collections.deque(maxlen=buffSize) #init holding object for holding all frame data in a circular buffer. - bdFramesList = collections.deque(maxlen=buffSize) #init holding object for holding all frame data in a circular buffer. - thumbDelay = int(args.ted) #get a seconds number for the delay in the original file btw exporting tags - parentDir = os.path.dirname(startObj) - baseName = os.path.basename(startObj) - baseName = baseName.replace(".qctools.xml.gz", "") - durationStart = args.ds - durationEnd = args.de - - # we gotta find out if the qctools report has pkt_dts_time or pkt_pts_time ugh - with gzip.open(startObj) as xml: - for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): # iterparse the xml doc - if elem.attrib['media_type'] == "video": # get just the video frames - # we gotta find out if the qctools report has pkt_dts_time or pkt_pts_time ugh - match = re.search(r"pkt_.ts_time", etree.tostring(elem).decode('utf-8')) - if match: - pkt = match.group() - break - - #set the start and end duration times - if args.bd: - durationStart = "" #if bar detection is turned on then we have to calculate this - durationEnd = "" #if bar detection is turned on then we have to calculate this - elif args.ds: - durationStart = float(args.ds) #The duration at which we start analyzing the file if no bar detection is selected - elif not args.de == 99999999: - durationEnd = float(args.de) #The duration at which we stop analyzing the file if no bar detection is selected - - - #set the path for the thumbnail export - if args.tep and not args.te: - print("Buddy, you specified a thumbnail export path without specifying that you wanted to export the thumbnails. Please either add '-te' to your cli call or delete '-tep [path]'") - - if args.tep: #if user supplied thumbExportPath, use that - thumbPath = str(args.tep) - else: - if args.t: #if they supplied a single tag - if args.o: #if the supplied tag is looking for a threshold Over - thumbPath = os.path.join(parentDir, str(args.t) + "." + str(args.o)) - elif args.u: #if the supplied tag was looking for a threshold Under - thumbPath = os.path.join(parentDir, str(args.t) + "." + str(args.u)) - else: #if they're using a profile, put all thumbs in 1 dir - thumbPath = os.path.join(parentDir, "ThumbExports") - - if args.te: #make the thumb export path if it doesn't already exist - if not os.path.exists(thumbPath): - os.makedirs(thumbPath) - - - ########Iterate Through the XML for Bars detection######## - if args.bd: - print("") - print("Starting Bars Detection on " + baseName) - print("") - durationStart,durationEnd = detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,) - - - ########Iterate Through the XML for General Analysis######## - print("") - print("Starting Analysis on " + baseName) - print("") - kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) - - - print("Finished Processing File: " + baseName + ".qctools.xml.gz") - print("") - - - #do some maths for the printout - if args.o or args.u or args.p is not None: - printresults(kbeyond,frameCount,overallFrameFail) - - return - -dependencies() -main() \ No newline at end of file diff --git a/qct_parse/__init__.py b/qct_parse/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/makeqctoolsreport.py b/qct_parse/makeqctoolsreport.py similarity index 99% rename from makeqctoolsreport.py rename to qct_parse/makeqctoolsreport.py index ee9fac7..f06103e 100644 --- a/makeqctoolsreport.py +++ b/qct_parse/makeqctoolsreport.py @@ -163,6 +163,7 @@ def main(): transcode(startObj,outPath) startObj = startObj + ".temp1.nut" makeReport(startObj,outPath) - + dependencies() -main() \ No newline at end of file +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/overcatch.py b/qct_parse/overcatch.py similarity index 99% rename from overcatch.py rename to qct_parse/overcatch.py index c7530e1..fde9cc7 100644 --- a/overcatch.py +++ b/qct_parse/overcatch.py @@ -86,4 +86,5 @@ def printout(barOutDict,contentOutDict,profileDict): for cod in contentOutDict: print("Frames beyond " + profileDict[cod] + " for " + contentOutDict[cod]) -main() +if __name__ == '__main__': + main() diff --git a/qct-parse_config.txt b/qct_parse/qct-parse_10bit_config.txt similarity index 78% rename from qct-parse_config.txt rename to qct_parse/qct-parse_10bit_config.txt index e2b84f8..870ad6c 100644 --- a/qct-parse_config.txt +++ b/qct_parse/qct-parse_10bit_config.txt @@ -1,5 +1,4 @@ -#based on qctools docs -#updated to match 10bit values +# 10bit values [default] YLOW: 64 @@ -48,18 +47,6 @@ SATMAX: 181.02 TOUT: 0.009 VREP: 0.03 -#Default profile values for 8-bit depth video files -[8bitDefault] -YLOW: 16 -YHIGH: 235 -ULOW: 16 -UHIGH: 235 -VLOW: 0 -VHIGH: 255 -SATMAX: 181.02 -TOUT: 0.009 -VREP: 0.03 - [fullTagList] YMIN: YLOW: diff --git a/qct_parse/qct-parse_8bit_config.txt b/qct_parse/qct-parse_8bit_config.txt new file mode 100644 index 0000000..1451cd1 --- /dev/null +++ b/qct_parse/qct-parse_8bit_config.txt @@ -0,0 +1,87 @@ +# based on qctools docs +# 8bit values + +[default] +YLOW: 16 +YHIGH: 235 +ULOW: 16 +UHIGH: 235 +VLOW: 0 +VHIGH: 255 +SATMAX: 181.02 +TOUT: 0.009 +VREP: 0.03 + +#Higher Tolerance for Peaking +[highTolerance] +YLOW: 10 +YMAX: 250 +UMIN: 16 +UMAX: 250 +VMIN: 0 +VMAX: 255 +SATMAX: 181.02 +TOUT: 0.009 +VREP: 0.03 + +#Medium Tolerance for Peaking +[midTolerance] +YLOW: 10 +YMAX: 245 +UMIN: 16 +UMAX: 245 +VMIN: 0 +VMAX: 255 +SATMAX: 181.02 +TOUT: 0.009 +VREP: 0.03 + +#Low Tolerance for Peaking +[lowTolerance] +YLOW: 16 +YMAX: 235 +UMIN: 16 +UMAX: 235 +VMIN: 0 +VMAX: 255 +SATMAX: 181.02 +TOUT: 0.009 +VREP: 0.03 + +[fullTagList] +YMIN: +YLOW: +YAVG: +YHIGH +YMAX: +UMIN: +ULOW: +UAVG: +UHIGH: +UMAX: +VMIN: +VLOW: +VAVG: +VHIGH: +VMAX: +SATMIN: +SATLOW: +SATAVG: +SATHIGH: +SATMAX: +HUEMED: +HUEAVG: +YDIF: +UDIF: +VDIF: +TOUT: +VREP: +BRNG: +mse_y: +mse_u: +mse_v: +mse_avg: +psnr_y: +psnr_u: +psnr_v: +psnr_avg: diff --git a/qct_parse/qct_parse.py b/qct_parse/qct_parse.py new file mode 100644 index 0000000..6451316 --- /dev/null +++ b/qct_parse/qct_parse.py @@ -0,0 +1,876 @@ +#!/usr/bin/env python +#qct-parse - changes made for python3 compatibility. WIP. + +#see this link for lxml goodness: http://www.ibm.com/developerworks/xml/library/x-hiperfparse/ + +from lxml import etree # for reading XML file (you will need to install this with pip) +import argparse # for parsing input args +import configparser # grip frame data values from a config txt file +import gzip # for opening gzip file +import logging # for logging output +import collections # for circular buffer +import os # for running ffmpeg and other terminal commands +import subprocess # not currently used +import gc # not currently used +import math # used for rounding up buffer half +import sys # system stuff +import re # can't spell parse without re fam +import operator +import time +import json +import shutil # dependency checking +import csv + + +CONFIG_ENVIRONMENT_VARIABLE_NAME = 'QCT_PARSE_CONFIG_DIRECTORY' + +#check that we have required software installed +def dependencies(): + depends = ['ffmpeg','ffprobe'] + for d in depends: + if shutil.which(d) is None: + print("Buddy, you gotta install " + d) + sys.exit() + return + +#Creates timestamp for pkt_dts_time +def dts2ts(frame_pkt_dts_time): + seconds = float(frame_pkt_dts_time) + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + if hours < 10: + hours = "0" + str(int(hours)) + else: + hours = str(int(hours)) + if minutes < 10: + minutes = "0" + str(int(minutes)) + else: + minutes = str(int(minutes)) + secondsStr = str(round(seconds,4)) + if int(seconds) < 10: + secondsStr = "0" + secondsStr + else: + seconds = str(minutes) + while len(secondsStr) < 7: + secondsStr = secondsStr + "0" + timeStampString = hours + ":" + minutes + ":" + secondsStr + return timeStampString + +#initializes the log +def initLog(inputPath): + """ + Initializes a log file for the given input file. + + The log file is created with the same name as the input file but with a '.log' extension. + + Args: + inputPath (str): The file path for the input file, used to create the log file. + + Returns: + None + """ + logPath = inputPath + '.log' + logging.basicConfig(filename=logPath,level=logging.INFO,format='%(asctime)s %(message)s') + logging.info("Started QCT-Parse") + +def set_logger(input_path): + log_path = f'{input_path}.log' + logger = logging.getLogger() + if logger.hasHandlers(): + logger.handlers.clear() + logger.setLevel(logging.INFO) + + file_handler = logging.FileHandler(filename=log_path) + file_handler.setLevel(logging.INFO) + file_handler.setFormatter(logging.Formatter('%(asctime)s %(message)s')) + logger.addHandler(file_handler) + logger.info("Started QCT-Parse") + + + +# finds stuff over/under threshold +def threshFinder(inFrame,args,startObj,pkt,tag,over,thumbPath,thumbDelay,adhoc_tag): + """ + Evaluates whether a tag in a video frame exceeds or falls below a threshold value and logs the result. + + This function checks if a given frame's tag value is either below or above a threshold. + It logs a warning if the value is outside the expected range and can optionally export a thumbnail. + + Args: + inFrame (dict): Dictionary containing frame data. + args (argparse.Namespace): Parsed command-line arguments. + startObj (object): Path to the QCTools report file (.qctools.xml.gz) + pkt (str): The key used to extract timestamps from tag in qctools.xml.gz. + tag (str): Tag from frame in qctools.xml.gz, checked against thresholds. + over (float): Threshold for the tag value. + thumbPath (str): Path to save thumbnails if they are exported. + thumbDelay (int): Current delay counter for thumbnail exports. + + Returns: + tuple: + bool: indicating if threshold was met (True or False) + int: Updated `thumbDelay` value based on whether a thumbnail was exported or not. + + Behavior: + - If the tag value is below the lower threshold (for keys containing "MIN" or "LOW"), logs a warning and may export a thumbnail. + - If the tag value is above the upper threshold, logs a warning and may export a thumbnail. + - Thumbnail export occurs if enabled (`args.te`) and if the delay since the last export exceeds the user-defined threshold (`args.ted`). + """ + tagValue = float(inFrame[tag]) + frame_pkt_dts_time = inFrame[pkt] + if adhoc_tag: + if args.o: + comparision = operator.gt + elif args.u: + comparision = operator.lt + else: + if "MIN" in tag or "LOW" in tag: + comparision = operator.lt + else: + comparision = operator.gt + + if comparision(float(tagValue), float(over)): # if the attribute is over usr set threshold + timeStampString = dts2ts(frame_pkt_dts_time) + logging.warning(tag + " is over " + str(over) + " with a value of " + str(tagValue) + " at duration " + timeStampString) + if args.te and (thumbDelay > int(args.ted)): # if thumb export is turned on and there has been enough delay between this frame and the last exported thumb, then export a new thumb + printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString) + thumbDelay = 0 + return True, thumbDelay # return true because it was over and thumbDelay + else: + return False, thumbDelay # return false because it was NOT over and thumbDelay + + +def get_video_resolution(input_video): + """ + Use ffprobe to get the resolution of the input video file. + + Args: + input_video (str): Path to the input video file. + + Returns: + (width, height) (tuple): The width and height of the video. + """ + ffprobe_command = [ + 'ffprobe', '-v', 'error', '-select_streams', 'v:0', '-show_entries', + 'stream=width,height', '-of', 'json', input_video + ] + + process = subprocess.Popen(ffprobe_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = process.communicate() + + if process.returncode != 0: + print(f"Error getting resolution with ffprobe: {err.decode('utf-8')}") + return None + + # Parse the JSON output + video_info = json.loads(out) + + # Extract the width and height from the video stream info + width = video_info['streams'][0]['width'] + height = video_info['streams'][0]['height'] + + return width, height + + +# print thumbnail images of overs/unders +def printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString): + ####init some variables using the args list + inputVid = startObj.replace(".qctools.xml.gz", "") + if os.path.isfile(inputVid): + # Get the resolution using ffprobe + resolution = get_video_resolution(inputVid) + if resolution: + width, height = resolution + else: + # Fall back to hardcoded resolution if ffprobe fails + width, height = 720, 486 + + baseName = os.path.basename(startObj) + baseName = baseName.replace(".qctools.xml.gz", "") + outputFramePath = os.path.join(thumbPath,baseName + "." + tag + "." + str(tagValue) + "." + timeStampString + ".png") + ffoutputFramePath = outputFramePath.replace(":",".") + # for windows we gotta see if that first : for the drive has been replaced by a dot and put it back + + match = '' + match = re.search(r"[A-Z]\.\/",ffoutputFramePath) #matches pattern R./ which should be R:/ on windows + if match: + ffoutputFramePath = ffoutputFramePath.replace(".",":",1) # replace first instance of "." in string ffoutputFramePath + + ffmpegString = ( + f'ffmpeg -ss {timeStampString} -i "{inputVid}" -vframes 1 ' + f'-s {width}x{height} -y -update 1 "{ffoutputFramePath}"' + ) + output = subprocess.Popen(ffmpegString,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True) + out,err = output.communicate() + # Decode byte strings to handle newlines properly + out = out.decode('utf-8') + err = err.decode('utf-8') + + if args.q is False: + print(out) + print(err) + else: + print("Input video file not found. Ensure video file is in the same directory as the QCTools report.") + exit() + return + + +# detect bars +def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,bit_depth_10): + """ + Detects color bars in a video by analyzing frames within a buffered window and logging the start and end times of the bars. + + This function iterates through the frames in a QCTools report, parses each frame, + and analyzes specific tags (YMAX, YMIN, YDIF) to detect the presence of color bars. + The detection checks a frame each time the buffer reaches the specified size (`buffSize`) and ends when the frame tags no longer match the expected bar values. + + Args: + args (argparse.Namespace): Parsed command-line arguments. + startObj (str): Path to the QCTools report file (.qctools.xml.gz) + pkt (str): Key used to identify the packet timestamp (pkt_*ts_time) in the XML frames. + durationStart (str): The timestamp when the bars start, initially an empty string. + durationEnd (str): The timestamp when the bars end, initially an empty string. + framesList (list): List of dictionaries storing the parsed frame data. + buffSize (int): The size of the frame buffer to hold frames for analysis. + + Returns: + tuple: + float: The timestamp (`durationStart`) when the bars were first detected. + float: The timestamp (`durationEnd`) when the bars were last detected. + + Behavior: + - Parses the input XML file frame by frame. + - Each frame's timestamp (`pkt_*ts_time`) and key-value pairs are stored in a dictionary (`frameDict`). + - Once the buffer reaches the specified size (`buffSize`), it checks the middle frame's attributes: + - Color bars are detected if `YMAX > 210`, `YMIN < 10`, and `YDIF < 3.0`. + - Logs the start and end times of the bars and stops detection once the bars end. + - Clears the memory of parsed elements to avoid excessive memory usage during parsing. + + Example log outputs: + - "Bars start at [timestamp] ([formatted timestamp])" + - "Bars ended at [timestamp] ([formatted timestamp])" + """ + if bit_depth_10: + YMAX_thresh = 800 + YMIN_thresh = 10 + YDIF_thresh = 10 + else: + YMAX_thresh = 210 + YMIN_thresh = 10 + YDIF_thresh = 3.0 + + with gzip.open(startObj) as xml: + for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): #iterparse the xml doc + if elem.attrib['media_type'] == "video": #get just the video frames + frame_pkt_dts_time = elem.attrib[pkt] #get the timestamps for the current frame we're looking at + frameDict = {} #start an empty dict for the new frame + frameDict[pkt] = frame_pkt_dts_time #give the dict the timestamp, which we have now + for t in list(elem): #iterating through each attribute for each element + keySplit = t.attrib['key'].split(".") #split the names by dots + keyName = str(keySplit[-1]) #get just the last word for the key name + frameDict[keyName] = t.attrib['value'] #add each attribute to the frame dictionary + framesList.append(frameDict) + middleFrame = int(round(float(len(framesList))/2)) # i hate this calculation, but it gets us the middle index of the list as an integer + if len(framesList) == buffSize: # wait till the buffer is full to start detecting bars + ## This is where the bars detection magic actually happens + # Check conditions + if (float(framesList[middleFrame]['YMAX']) > YMAX_thresh and + float(framesList[middleFrame]['YMIN']) < YMIN_thresh and + float(framesList[middleFrame]['YDIF']) < YDIF_thresh): + if durationStart == "": + durationStart = float(framesList[middleFrame][pkt]) + print("Bars start at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") + durationEnd = float(framesList[middleFrame][pkt]) + else: + if durationStart != "" and durationEnd != "" and durationEnd - durationStart > 2: + print("Bars ended at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") + break + elem.clear() # we're done with that element so let's get it outta memory + return durationStart, durationEnd + + +def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag=False,frameCount=0,overallFrameFail=0): + """ + Analyzes video frames from the QCTools report to detect threshold exceedances for specified tags or profiles and logs frame failures. + + This function iteratively parses video frames from a QCTools report (`.qctools.xml.gz`) and checks whether the frame attributes exceed user-defined thresholds + (either single tags or profiles). Threshold exceedances are logged, and frames can be flagged for further analysis. Optionally, thumbnails of failing frames can be generated. + + Args: + args (argparse.Namespace): Parsed command-line arguments, including tag thresholds and options for profile, thumbnail export, etc. + profile (dict): A dictionary of key-value pairs of tag names and their corresponding threshold values. + startObj (str): Path to the QCTools report file (.qctools.xml.gz) + pkt (str): Key used to identify the pkt_*ts_time in the XML frames. + durationStart (float): The starting time for analyzing frames (in seconds). + durationEnd (float): The ending time for analyzing frames (in seconds). Can be `None` to process until the end. + thumbPath (str): Path to save the thumbnail images of frames exceeding thresholds. + thumbDelay (int): Delay counter between consecutive thumbnail generations to prevent spamming. + framesList (list): A circular buffer to hold dictionaries of parsed frame attributes. + frameCount (int, optional): The total number of frames analyzed (defaults to 0). + overallFrameFail (int, optional): A count of how many frames failed threshold checks across all tags (defaults to 0). + + Returns: + tuple: + - kbeyond (dict): A dictionary where each tag is associated with a count of how many times its threshold was exceeded. + - frameCount (int): The total number of frames analyzed. + - overallFrameFail (int): The total number of frames that exceeded thresholds across all tags. + + Behavior: + - Iteratively parses the input XML file and analyzes frames after `durationStart` and before `durationEnd`. + - Frames are stored in a circular buffer (`framesList`), and attributes (tags) are extracted into dictionaries. + - For each frame, checks whether specified tags exceed user-defined thresholds (from `args.o`, `args.u`, or `profile`). + - Logs threshold exceedances and updates the count of failed frames. + - Optionally, generates thumbnails for frames that exceed thresholds, ensuring a delay between consecutive thumbnails. + + Example usage: + - Analyzing frames using a single tag threshold: `analyzeIt(args, {}, startObj, pkt, durationStart, durationEnd, thumbPath, thumbDelay, framesList)` + - Analyzing frames using a profile: `analyzeIt(args, profile, startObj, pkt, durationStart, durationEnd, thumbPath, thumbDelay, framesList)` + """ + kbeyond = {} # init a dict for each key which we'll use to track how often a given key is over + fots = "" + for k,v in profile.items(): + kbeyond[k] = 0 + with gzip.open(startObj) as xml: + for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): #iterparse the xml doc + if elem.attrib['media_type'] == "video": #get just the video frames + frameCount = frameCount + 1 + frame_pkt_dts_time = elem.attrib[pkt] #get the timestamps for the current frame we're looking at + if frame_pkt_dts_time >= str(durationStart): #only work on frames that are after the start time + if durationEnd: + if float(frame_pkt_dts_time) > durationEnd: #only work on frames that are before the end time + print("started at " + str(durationStart) + " seconds and stopped at " + str(frame_pkt_dts_time) + " seconds (" + dts2ts(frame_pkt_dts_time) + ") or " + str(frameCount) + " frames!") + break + frameDict = {} #start an empty dict for the new frame + frameDict[pkt] = frame_pkt_dts_time #make a key for the timestamp, which we have now + for t in list(elem): #iterating through each attribute for each element + keySplit = t.attrib['key'].split(".") #split the names by dots + keyName = str(keySplit[-1]) #get just the last word for the key name + if len(keyName) == 1: #if it's psnr or mse, keyName is gonna be a single char + keyName = '.'.join(keySplit[-2:]) #full attribute made by combining last 2 parts of split with a period in btw + frameDict[keyName] = t.attrib['value'] #add each attribute to the frame dictionary + framesList.append(frameDict) #add this dict to our circular buffer + if args.pr is True: #display "timestamp: Tag Value" (654.754100: YMAX 229) to the terminal window + print(framesList[-1][pkt] + ": " + args.t + " " + framesList[-1][args.t]) + # Now we can parse the frame data from the buffer! + for k,v in profile.items(): + tag = k + over = float(v) + # ACTUALLY DO THE THING ONCE FOR EACH TAG + frameOver, thumbDelay = threshFinder(framesList[-1],args,startObj,pkt,tag,over,thumbPath,thumbDelay,adhoc_tag) + if frameOver is True: + kbeyond[k] = kbeyond[k] + 1 # note the over in the key over dict + if not frame_pkt_dts_time in fots: # make sure that we only count each over frame once + overallFrameFail = overallFrameFail + 1 + fots = frame_pkt_dts_time # set it again so we don't dupe + thumbDelay = thumbDelay + 1 + elem.clear() #we're done with that element so let's get it outta memory + return kbeyond, frameCount, overallFrameFail + + +def detectBitdepth(startObj,pkt,framesList,buffSize): + bit_depth_10 = False + with gzip.open(startObj) as xml: + for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): # iterparse the xml doc + if elem.attrib['media_type'] == "video": # get just the video frames + frame_pkt_dts_time = elem.attrib[pkt] # get the timestamps for the current frame we're looking at + frameDict = {} # start an empty dict for the new frame + frameDict[pkt] = frame_pkt_dts_time # give the dict the timestamp, which we have now + for t in list(elem): # iterating through each attribute for each element + keySplit = t.attrib['key'].split(".") # split the names by dots + keyName = str(keySplit[-1]) # get just the last word for the key name + frameDict[keyName] = t.attrib['value'] # add each attribute to the frame dictionary + framesList.append(frameDict) + middleFrame = int(round(float(len(framesList))/2)) # i hate this calculation, but it gets us the middle index of the list as an integer + if len(framesList) == buffSize: # wait till the buffer is full to start detecting bars + ## This is where the bars detection magic actually happens + bufferRange = list(range(0, buffSize)) + if float(framesList[middleFrame]['YMAX']) > 250: + bit_depth_10 = True + break + elem.clear() # we're done with that element so let's get it outta memory + + return bit_depth_10 + + +def evalBars(startObj,pkt,durationStart,durationEnd,framesList,buffSize): + # Define the keys for which you want to calculate the average + keys_to_check = ['YMAX', 'YMIN', 'UMIN', 'UMAX', 'VMIN', 'VMAX', 'SATMAX', 'SATMIN'] + # Initialize a dictionary to store the highest values for each key + maxBarsDict = {} + # adds the list keys_to_check as keys to a dictionary + for key_being_checked in keys_to_check: + # assign 'dummy' threshold to be overwritten + if "MAX" in key_being_checked: + maxBarsDict[key_being_checked] = 0 + elif "MIN" in key_being_checked: + maxBarsDict[key_being_checked] = 1023 + + with gzip.open(startObj) as xml: + for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): # iterparse the xml doc + if elem.attrib['media_type'] == "video": # get just the video frames + frame_pkt_dts_time = elem.attrib[pkt] # get the timestamps for the current frame we're looking at + if frame_pkt_dts_time >= str(durationStart): # only work on frames that are after the start time # only work on frames that are after the start time + if float(frame_pkt_dts_time) > durationEnd: # only work on frames that are before the end time + break + frameDict = {} # start an empty dict for the new frame + frameDict[pkt] = frame_pkt_dts_time # give the dict the timestamp, which we have now + for t in list(elem): # iterating through each attribute for each element + keySplit = t.attrib['key'].split(".") # split the names by dots + keyName = str(keySplit[-1]) # get just the last word for the key name + frameDict[keyName] = t.attrib['value'] # add each attribute to the frame dictionary + framesList.append(frameDict) + if len(framesList) == buffSize: # wait till the buffer is full to start detecting bars + ## This is where the bars detection magic actually happens + for colorbar_key in keys_to_check: + if colorbar_key in frameDict: + if "MAX" in colorbar_key: + # Convert the value to float and compare it with the current highest value + value = float(frameDict[colorbar_key]) + if value > maxBarsDict[colorbar_key]: + maxBarsDict[colorbar_key] = value + elif "MIN" in colorbar_key: + # Convert the value to float and compare it with the current highest value + value = float(frameDict[colorbar_key]) + if value < maxBarsDict[colorbar_key]: + maxBarsDict[colorbar_key] = value + # Convert highest values to integer + maxBarsDict = {colorbar_key: int(value) for colorbar_key, value in maxBarsDict.items()} + + return maxBarsDict + + +def extract_report_mkv(startObj): + + report_file_output = startObj.replace(".qctools.mkv", ".qctools.xml.gz") + if os.path.isfile(report_file_output): + while True: + user_input = input(f"The file {os.path.basename(report_file_output)} already exists. \nExtract xml.gz from {os.path.basename(startObj)} and overwrite existing file? \n(y/n):\n") + if user_input.lower() in ["yes", "y"]: + os.remove(report_file_output) + # Run ffmpeg command to extract xml.gz report + full_command = [ + 'ffmpeg', + '-hide_banner', + '-loglevel', 'panic', + '-dump_attachment:t:0', report_file_output, + '-i', startObj + ] + print(f'Extracting qctools.xml.gz report from {os.path.basename(startObj)}\n') + print(f'Running command: {" ".join(full_command)}\n') + subprocess.run(full_command) + break + elif user_input.lower() in ["no", "n"]: + print('Processing existing qctools report, not extracting file\n') + break + else: + print("Invalid input. Please enter yes/no.\n") + else: + # Run ffmpeg command to extract xml.gz report + full_command = [ + 'ffmpeg', + '-hide_banner', + '-loglevel', 'panic', + '-dump_attachment:t:0', report_file_output, + '-i', startObj + ] + print(f'Extracting qctools.xml.gz report from {os.path.basename(startObj)}\n') + print(f'Running command: {" ".join(full_command)}\n') + subprocess.run(full_command) + + if os.path.isfile(report_file_output): + startObj = report_file_output + else: + print(f'Unable to extract XML from QCTools mkv report file\n') + startObj = None + exit() + + return startObj + + +def print_peak_colorbars(maxBarsDict): + # ASCI formatting + BOLD = "\033[1m" + RESET = "\033[0m" + + print("\nReporting frames outside of these thresholds:") + + # Create two lists for even and odd indices + tags = list(maxBarsDict.keys()) + values = list(maxBarsDict.values()) + + # Print even-indexed tags and values on the first line + for i in range(0, len(tags), 2): + print(f"{BOLD}{tags[i]:<6}{RESET} {values[i]:<5}", end=" ") + print() # Move to the next line + + # Print odd-indexed tags and values on the second line + for i in range(1, len(tags), 2): + print(f"{BOLD}{tags[i]:<6}{RESET} {values[i]:<5}", end=" ") + print() # Move to the next line + + +# Print results from analyzeIt +def printresults(kbeyond, frameCount, overallFrameFail): + """ + Prints the analysis results of frame data, including counts of frames exceeding thresholds + for various tags and the percentage of total frames that are affected. + + Args: + kbeyond (dict): A dictionary where keys are tag names and values are the counts of frames + that exceed the threshold for each tag. + frameCount (int): The total number of frames analyzed. + overallFrameFail (int): The number of frames where at least one tag exceeds its threshold. + + Prints: + - The total number of frames analyzed. + - A breakdown of frame counts for each tag in `kbeyond` and the corresponding percentage + of the total frames that exceeded the tag's threshold. + - The overall count and percentage of frames that failed at least one threshold. + + Notes: + - If `frameCount` is zero, it prints "TotalFrames: 0" and returns early. + - Percentages are formatted as whole numbers (e.g., "100"), two decimal places + (e.g., "12.34"), or "<0.01" for values less than 0.01%. + """ + # Define ANSI escape codes for color and formatting + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + RESET = "\033[0m" + + RED = "\033[91m" + YELLOW = "\033[93m" + GREEN = "\033[92m" + + def format_percentage(value): + percent = value * 100 + if percent == 100: + return "100" + elif percent == 0: + return "0" + elif percent < 0.01: + return "<0.01" + else: + return f"{percent:.2f}" + + def color_percentage(value): + percent = value * 100 + if percent > 10: + return RED + elif percent > 1: + return YELLOW + else: + return GREEN + + if frameCount == 0: + print(f"{UNDERLINE}TotalFrames:{RESET}\t0") + return + + print(f"\n{UNDERLINE}TotalFrames{RESET}:\t{frameCount}\n") + print(f"{UNDERLINE}By Tag{RESET}:\n") + + for tag, count in kbeyond.items(): + percent = count / frameCount + percent_over_string = format_percentage(percent) + color = color_percentage(percent) + print(f"{BOLD}{tag}{RESET}:\t{count}\t{color}{percent_over_string}{RESET}\t% of the total # of frames\n") + + print(f"{BOLD}Overall:{RESET}\n") + overall_percent = overallFrameFail / frameCount + percent_overall_string = format_percentage(overall_percent) + color = color_percentage(overall_percent) + print(f"Frames With At Least One Fail:\t{overallFrameFail}\t{color}{percent_overall_string}{RESET}\t% of the total # of frames\n") + print(f"{BOLD}**************************{RESET}\n") + + +def print_results_to_csv(kbeyond, frameCount, overallFrameFail, startObj, output_file): + """ + Writes the analysis results of frame data to a CSV file. + + Args: + kbeyond (dict): A dictionary where keys are tag names and values are the counts of frames + that exceed the threshold for each tag. + frameCount (int): The total number of frames analyzed. + overallFrameFail (int): The number of frames where at least one tag exceeds its threshold. + output_file (str): The name of the CSV file to save the results. + + Outputs: + A CSV file with the following structure: + - TotalFrames: [frameCount] + - By Tag: [Tag, Count, Percentage] + - Overall: Frames With At Least One Fail, Count, Percentage + + Notes: + - Percentages are formatted as whole numbers (e.g., "100"), two decimal places + (e.g., "12.34"), or "<0.01" for values less than 0.01%. + """ + + def format_percentage(value): + percent = value * 100 + if percent == 100: + return "100" + elif percent == 0: + return "0" + elif percent < 0.01: + return "<0.01" + else: + return f"{percent:.2f}" + + # Write results to CSV + with open(output_file, mode='w', newline='', encoding='utf-8') as file: + writer = csv.writer(file) + + # Title row + writer.writerow(["qct-parse summary report for file:", startObj]) + + # Total Frames + writer.writerow(["TotalFrames", frameCount]) + + # By Tag + writer.writerow(["By Tag"]) + writer.writerow(["Tag", "Count", "Percentage of Total Frames"]) + for tag, count in kbeyond.items(): + percent = count / frameCount + writer.writerow([tag, count, format_percentage(percent)]) + + # Overall + writer.writerow([]) + writer.writerow(["Overall"]) + overall_percent = overallFrameFail / frameCount + writer.writerow(["Frames With At Least One Fail", overallFrameFail, format_percentage(overall_percent)]) + + print(f"Results successfully written to {output_file}") + +def get_arg_parser(): + parser = argparse.ArgumentParser(description="parses QCTools XML files for frames beyond broadcast values") + parser.add_argument('-i','--input',dest='i', action='append', help="the path to the input qctools.xml.gz file") + parser.add_argument('-t','--tagname',dest='t', help="the tag name you want to test, e.g. SATMAX") + parser.add_argument('-o','--over',dest='o', help="the threshold overage number") + parser.add_argument('-u','--under',dest='u', help="the threshold under number") + parser.add_argument('-p','--profile', dest='p', nargs='*', default=None, help="use values from your qct-parse-config.txt file, provide profile/ template name, e.g. 'default'") + parser.add_argument('-buff','--buffSize',dest='buff',default=11, help="Size of the circular buffer. if user enters an even number it'll default to the next largest number to make it odd (default size 11)") + parser.add_argument('-te','--thumbExport',dest='te',action='store_true',default=False, help="export thumbnail") + parser.add_argument('-ted','--thumbExportDelay',dest='ted',default=9000, help="minimum frames between exported thumbs") + parser.add_argument('-tep','--thumbExportPath',dest='tep',default='', help="Path to thumb export. if omitted, it uses the input basename") + parser.add_argument('-ds','--durationStart',dest='ds',default=0, help="the duration in seconds to start analysis") + parser.add_argument('-de','--durationEnd',dest='de',default=99999999, help="the duration in seconds to stop analysis") + parser.add_argument('-bd','--barsDetection',dest='bd',action ='store_true',default=False, help="turns Bar Detection on and off") + parser.add_argument('-be','--barsEvaluation',dest='be',action ='store_true',default=False, help="turns Color Bar Evaluation on and off") + parser.add_argument('-pr','--print',dest='pr',action='store_true',default=False, help="print over/under frame data to console window") + parser.add_argument('-csv', '--csvreport',dest='csv',action ='store_true',default=False, help="print summary results to a csv sidecar file") + parser.add_argument('-q','--quiet',dest='q',action='store_true',default=False, help="hide ffmpeg output from console window") + return parser + + +def parse_single_qc_tools_report(input_file, args): + startObj = input_file.replace("\\","/") + extension = os.path.splitext(startObj)[1] + # If qctools report is in an MKV attachment, extract .qctools.xml.gz report + if extension.lower().endswith('mkv'): + startObj = extract_report_mkv(startObj) + buffSize = int(args.buff) # cast the input buffer as an integer + if buffSize%2 == 0: + buffSize = buffSize + 1 + set_logger(startObj) + overcount = 0 # init count of overs + undercount = 0 # init count of unders + count = 0 # init total frames counter + framesList = collections.deque(maxlen=buffSize) # init framesList + thumbDelay = int(args.ted) # get a seconds number for the delay in the original file btw exporting tags + parentDir = os.path.dirname(startObj) + baseName = os.path.basename(startObj) + baseName = baseName.replace(".qctools.xml.gz", "") + durationStart = args.ds + durationEnd = args.de + # set the path for the csv report + if args.csv: + result_csv_file = os.path.join(parentDir, str(baseName) + ".qct_summary_report.csv") + else: + result_csv_file = None + + # we gotta find out if the qctools report has pkt_dts_time or pkt_pts_time ugh + with gzip.open(startObj) as xml: + for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): # iterparse the xml doc + if elem.attrib['media_type'] == "video": # get just the video frames + # we gotta find out if the qctools report has pkt_dts_time or pkt_pts_time ugh + match = re.search(r"pkt_.ts_time", etree.tostring(elem).decode('utf-8')) + if match: + pkt = match.group() + break + + ###### Initialize values from the Config Parser + # Determine if video values are 10 bit depth + bit_depth_10 = detectBitdepth(startObj,pkt,framesList,buffSize) + # init a dictionary where we'll store reference values from our config file + profile = {} + # init a list of every tag available in a QCTools Report + tagList = ["YMIN","YLOW","YAVG","YHIGH","YMAX","UMIN","ULOW","UAVG","UHIGH","UMAX","VMIN","VLOW","VAVG","VHIGH","VMAX","SATMIN","SATLOW","SATAVG","SATHIGH","SATMAX","HUEMED","HUEAVG","YDIF","UDIF","VDIF","TOUT","VREP","BRNG","mse_y","mse_u","mse_v","mse_avg","psnr_y","psnr_u","psnr_v","psnr_avg"] + + # set the start and end duration times + if args.bd: + durationStart = "" # if bar detection is turned on then we have to calculate this + durationEnd = "" # if bar detection is turned on then we have to calculate this + elif args.ds: + durationStart = float(args.ds) # The duration at which we start analyzing the file if no bar detection is selected + elif not args.de == 99999999: + durationEnd = float(args.de) # The duration at which we stop analyzing the file if no bar detection is selected + + + # set the path for the thumbnail export + if args.tep and not args.te: + print("Buddy, you specified a thumbnail export path without specifying that you wanted to export the thumbnails. Please either add '-te' to your cli call or delete '-tep [path]'") + exit() + + if args.tep: # if user supplied thumbExportPath, use that + thumbPath = str(args.tep) + else: + if args.t: # if they supplied a single tag + if args.o: # if the supplied tag is looking for a threshold Over + thumbPath = os.path.join(parentDir, str(args.t) + "." + str(args.o)) + elif args.u: # if the supplied tag was looking for a threshold Under + thumbPath = os.path.join(parentDir, str(args.t) + "." + str(args.u)) + else: # if they're using a profile, put all thumbs in 1 dir + thumbPath = os.path.join(parentDir, "ThumbExports") + + if args.te: # make the thumb export path if it doesn't already exist + if not os.path.exists(thumbPath): + os.makedirs(thumbPath) + + + ######## Iterate Through the XML for Bars detection ######## + if args.bd: + print(f"\nStarting Bars Detection on {baseName}\n") + durationStart, durationEnd = detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,bit_depth_10) + if args.be and durationStart != "" and durationEnd != "": + maxBarsDict = evalBars(startObj,pkt,durationStart,durationEnd,framesList,buffSize) + if maxBarsDict is None: + print("\nSomething went wrong - cannot run colorbars evaluation") + else: + print("\nNow comparing peak values of color bars to the rest of the video.") + print_peak_colorbars(maxBarsDict) + # Reset start and stop time to eval the whole video (color bars won't be flagged because we already have their max values) + durationStart = 0 + durationEnd = 99999999 + profile = maxBarsDict + kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag=False) + if args.csv: + print_results_to_csv(kbeyond, frameCount, overallFrameFail, startObj, result_csv_file) + printresults(kbeyond,frameCount,overallFrameFail) + else: + printresults(kbeyond,frameCount,overallFrameFail) + else: + durationStart = "" + durationEnd = "" + + if args.p is not None: + # create list of profiles + list_of_templates = args.p + # setup configparser + config = configparser.RawConfigParser(allow_no_value=True) + dn, fn = os.path.split(os.path.abspath(__file__)) # grip the dir where ~this script~ is located, also where config.txt should be located + # assign config based on bit depth of tag values + if CONFIG_ENVIRONMENT_VARIABLE_NAME in os.environ: + print(f"Using config files in ${CONFIG_ENVIRONMENT_VARIABLE_NAME}") + dn = os.environ[CONFIG_ENVIRONMENT_VARIABLE_NAME] + + if bit_depth_10: + config.read(os.path.join(dn,"qct-parse_10bit_config.txt")) # read in the config file + else: + config.read(os.path.join(dn,"qct-parse_8bit_config.txt")) # read in the config file + for template in list_of_templates: + # Check if the template is a valid section in the config + if not config.has_section(template): + print(f"Profile '{template}' does not match any section in the config.") + continue # Skip to the next template if section doesn't exist + for t in tagList: # loop thru every tag available and + try: # see if it's in the config section + profile[t.replace("_",".")] = config.get(template,t) # if it is, replace _ necessary for config file with . which xml attributes use, assign the value in config + except: # if no config tag exists, do nothing so we can move faster + pass + + ######## Iterate Through the XML for General Analysis ######## + print(f"\nStarting Analysis on {baseName} using assigned profile {template}\n") + kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag=False) + if args.csv: + print_results_to_csv(kbeyond, frameCount, overallFrameFail, startObj, result_csv_file) + printresults(kbeyond,frameCount,overallFrameFail) + else: + printresults(kbeyond,frameCount,overallFrameFail) + + if args.t and args.o or args.u: + profile = {} + tag = args.t + if args.o: + over = float(args.o) + if args.u: + over = float(args.u) + profile[tag] = over + print(f"\nStarting Analysis on {baseName} using user specified tag {tag} w/ threshold {over}\n") + kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag = True) + if args.csv: + print_results_to_csv(kbeyond, frameCount, overallFrameFail, startObj, result_csv_file) + printresults(kbeyond,frameCount,overallFrameFail) + else: + printresults(kbeyond,frameCount,overallFrameFail) + + print(f"\nFinished Processing File: {baseName}.qctools.xml.gz\n") + +def parse_qc_tools_report(args): + ##### Initialize variables and buffers ###### + for input_file in args.i: + parse_single_qc_tools_report(input_file, args) + +def main(): + """ + Main function that parses QCTools XML files, applies analysis, and optionally exports thumbnails. + + This function handles command-line arguments to process a QCTools report, extract frame data from the XML, + apply threshold analysis for broadcast values, optionally detect color bars, and export analysis results to + the console or thumbnails. + + Command-line Arguments: + -i, --input (str): Path to the input QCTools XML.gz file. + -t, --tagname (str): Tag name to analyze, e.g., SATMAX. + -o, --over (float): Overage threshold for the tag specified. + -u, --under (float): Under threshold for the tag specified. + -p, --profile (str): Profile or template name from the qct-parse_config.txt file, e.g., 'default'. + -buff, --buffSize (int): Circular buffer size. Defaults to 11, ensures odd number. + -te, --thumbExport: Export thumbnails if flag is set. + -ted, --thumbExportDelay (int): Minimum number of frames between exported thumbnails. + -tep, --thumbExportPath (str): Path to export thumbnails, defaults to input basename if not provided. + -ds, --durationStart (float): Start time in seconds for analysis. + -de, --durationEnd (float): End time in seconds for analysis. + -bd, --barsDetection: Flag to enable color bars detection. + -pr, --print: Flag to print frame data to the console. + -q, --quiet: Hide ffmpeg output if flag is set. + + Workflow: + 1. Parse command-line arguments. + 2. Optionally load reference threshold values from a profile in `qct-parse_config.txt`. + 3. Initialize buffers, frame counters, and paths for thumbnail export. + 4. Check for `pkt_dts_time` or `pkt_pts_time` in the QCTools XML file. + 5. Set the analysis duration start and end times. + 6. Perform bars detection if enabled, otherwise proceed with general analysis. + 7. Call the `analyzeIt` function to perform frame-by-frame analysis and calculate exceedances. + 8. Print results using `printresults` if applicable. + 9. Handle errors or invalid input (e.g., missing thumbnail export flag but specifying a path). + + Example usage: + python qct-parse.py -i sample.qctools.xml.gz -t SATMAX -o 5.0 -u -5.0 -te + + Returns: + None: The function processes the XML file, performs analysis, and optionally exports thumbnails and prints results to the console. + """ + #### init the stuff from the cli ######## + parser = get_arg_parser() + args = parser.parse_args() + ## Validate required arguments + if not args.i: + parser.error("the following arguments are required: -i/--input [path to QCTools report]") + if args.o and args.u: + parser.error("Both the -o and -u options were used. Cannot set threshold for both over and under, only one at a time.") + parse_qc_tools_report(args) + + +if __name__ == '__main__': + dependencies() + main() diff --git a/tests/test_qct_parse.py b/tests/test_qct_parse.py new file mode 100644 index 0000000..483d2c1 --- /dev/null +++ b/tests/test_qct_parse.py @@ -0,0 +1,4 @@ +from qct_parse import qct_parse + +def test_dts2ts(): + assert qct_parse.dts2ts("0.0330000") == '00:00:00.0330'