From b010f805e26bb1a0d4b3f192fce8277214d51d5b Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Fri, 27 Sep 2024 08:26:32 -0600 Subject: [PATCH 01/42] changing framesList froma circular buffer to a regular list to get improve bars detection results. --- qct-parse.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 48c0ae5..36b83ee 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -245,19 +245,20 @@ def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize): keyName = str(keySplit[-1]) # get just the last word for the key name frameDict[keyName] = t.attrib['value'] # add each attribute to the frame dictionary framesList.append(frameDict) - middleFrame = int(round(float(len(framesList))/2)) # i hate this calculation, but it gets us the middle index of the list as an integer - if len(framesList) == buffSize: # wait till the buffer is full to start detecting bars - ## This is where the bars detection magic actually happens - bufferRange = list(range(0, buffSize)) - if float(framesList[middleFrame]['YMAX']) > 210 and float(framesList[middleFrame]['YMIN']) < 10 and float(framesList[middleFrame]['YDIF']) < 3.0: - if durationStart == "": - durationStart = float(framesList[middleFrame][pkt]) - print("Bars start at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") - durationEnd = float(framesList[middleFrame][pkt]) - else: - print("Bars ended at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") - break elem.clear() # we're done with that element so let's get it outta memory + frame_count = 0 + for frameDict in framesList: + frame_count += 1 + if frame_count % 25 == 0: + if float(frameDict['YMAX']) > 800 and float(frameDict['YMIN']) < 10 and float(frameDict['YDIF']) < 7: + if durationStart == "": + durationStart = float(frameDict[pkt]) + print("Bars start at " + str(frameDict[pkt]) + " (" + dts2ts(frameDict[pkt]) + ")") + durationEnd = float(frameDict[pkt]) + else: + if durationStart != "" and durationEnd != "" and durationEnd - durationStart > 2: + print("Bars ended at " + str(frameDict[pkt]) + " (" + dts2ts(frameDict[pkt]) + ")\n") + break return durationStart, durationEnd @@ -497,8 +498,7 @@ def main(): overcount = 0 # init count of overs undercount = 0 # init count of unders count = 0 # init total frames counter - framesList = collections.deque(maxlen=buffSize) # init holding object for holding all frame data in a circular buffer. - bdFramesList = collections.deque(maxlen=buffSize) # init holding object for holding all frame data in a circular buffer. + framesList = [] # init framesList thumbDelay = int(args.ted) # get a seconds number for the delay in the original file btw exporting tags parentDir = os.path.dirname(startObj) baseName = os.path.basename(startObj) From 2dae67ec339d3bc40d2acccc87153fde81224abe Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Mon, 30 Sep 2024 14:30:00 -0600 Subject: [PATCH 02/42] fixed typo, extra comma in calling detectBars function --- qct-parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qct-parse.py b/qct-parse.py index 48c0ae5..bb498e5 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -573,7 +573,7 @@ def main(): print("") print("Starting Bars Detection on " + baseName) print("") - durationStart,durationEnd = detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,) + durationStart,durationEnd = detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize) ######## Iterate Through the XML for General Analysis ######## From d4df78d02541866a127c7ce94396a3113c0fb588 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Mon, 30 Sep 2024 15:37:57 -0600 Subject: [PATCH 03/42] put circular buffer back in to framesList, found good 10bit thresholds --- qct-parse.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 36b83ee..3bd17e3 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -245,20 +245,20 @@ def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize): keyName = str(keySplit[-1]) # get just the last word for the key name frameDict[keyName] = t.attrib['value'] # add each attribute to the frame dictionary framesList.append(frameDict) + middleFrame = int(round(float(len(framesList))/2)) # i hate this calculation, but it gets us the middle index of the list as an integer + if len(framesList) == buffSize: # wait till the buffer is full to start detecting bars + ## This is where the bars detection magic actually happens + # bufferRange = list(range(0, buffSize)) + if float(framesList[middleFrame]['YMAX']) > 800 and float(framesList[middleFrame]['YMIN']) < 10 and float(framesList[middleFrame]['YDIF']) < 10: + if durationStart == "": + durationStart = float(framesList[middleFrame][pkt]) + print("Bars start at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") + durationEnd = float(framesList[middleFrame][pkt]) + else: + if durationStart != "" and durationEnd != "" and durationEnd - durationStart > 2: + print("Bars ended at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") + break elem.clear() # we're done with that element so let's get it outta memory - frame_count = 0 - for frameDict in framesList: - frame_count += 1 - if frame_count % 25 == 0: - if float(frameDict['YMAX']) > 800 and float(frameDict['YMIN']) < 10 and float(frameDict['YDIF']) < 7: - if durationStart == "": - durationStart = float(frameDict[pkt]) - print("Bars start at " + str(frameDict[pkt]) + " (" + dts2ts(frameDict[pkt]) + ")") - durationEnd = float(frameDict[pkt]) - else: - if durationStart != "" and durationEnd != "" and durationEnd - durationStart > 2: - print("Bars ended at " + str(frameDict[pkt]) + " (" + dts2ts(frameDict[pkt]) + ")\n") - break return durationStart, durationEnd @@ -498,7 +498,7 @@ def main(): overcount = 0 # init count of overs undercount = 0 # init count of unders count = 0 # init total frames counter - framesList = [] # init framesList + framesList = collections.deque(maxlen=buffSize) # init framesList thumbDelay = int(args.ted) # get a seconds number for the delay in the original file btw exporting tags parentDir = os.path.dirname(startObj) baseName = os.path.basename(startObj) From d8429f81d0021ae8f3a1010193a8d5b1ce679ffe Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Mon, 30 Sep 2024 16:21:01 -0600 Subject: [PATCH 04/42] removed durationStart and durationStop from barsDetection output. Can add them back in when we do evalBars --- qct-parse.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 3bd17e3..e349a8a 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -248,7 +248,6 @@ def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize): middleFrame = int(round(float(len(framesList))/2)) # i hate this calculation, but it gets us the middle index of the list as an integer if len(framesList) == buffSize: # wait till the buffer is full to start detecting bars ## This is where the bars detection magic actually happens - # bufferRange = list(range(0, buffSize)) if float(framesList[middleFrame]['YMAX']) > 800 and float(framesList[middleFrame]['YMIN']) < 10 and float(framesList[middleFrame]['YDIF']) < 10: if durationStart == "": durationStart = float(framesList[middleFrame][pkt]) @@ -259,7 +258,6 @@ def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize): print("Bars ended at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") break elem.clear() # we're done with that element so let's get it outta memory - return durationStart, durationEnd def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,frameCount=0,overallFrameFail=0): @@ -488,7 +486,6 @@ def main(): parser.add_argument('-q','--quiet',dest='q',action='store_true',default=False, help="hide ffmpeg output from console window") args = parser.parse_args() - ##### Initialize variables and buffers ###### startObj = args.i.replace("\\","/") buffSize = int(args.buff) # cast the input buffer as an integer @@ -573,7 +570,7 @@ def main(): print("") print("Starting Bars Detection on " + baseName) print("") - durationStart,durationEnd = detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,) + detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize) ######## Iterate Through the XML for General Analysis ######## @@ -590,7 +587,6 @@ def main(): # do some maths for the printout if args.o or args.u or args.p is not None: printresults(kbeyond,frameCount,overallFrameFail) - return dependencies() From e1cc571e5ce4c321a116be140c024f224a218dd2 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Mon, 30 Sep 2024 16:45:29 -0600 Subject: [PATCH 05/42] added conditional for different bars detection thresholds for 8bit vs 10bit values --- qct-parse.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index e349a8a..9531ca4 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -200,7 +200,7 @@ def printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString): # detect bars -def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize): +def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,bit_depth_10): """ Detects color bars in a video by analyzing frames within a buffered window and logging the start and end times of the bars. @@ -234,6 +234,15 @@ def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize): - "Bars start at [timestamp] ([formatted timestamp])" - "Bars ended at [timestamp] ([formatted timestamp])" """ + if bit_depth_10: + YMAX_thresh = 800 + YMIN_thresh = 10 + YDIF_thresh = 10 + else: + YMAX_thresh = 210 + YMIN_thresh = 10 + YDIF_thresh = 3.0 + with gzip.open(startObj) as xml: for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): # iterparse the xml doc if elem.attrib['media_type'] == "video": # get just the video frames @@ -248,11 +257,14 @@ def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize): middleFrame = int(round(float(len(framesList))/2)) # i hate this calculation, but it gets us the middle index of the list as an integer if len(framesList) == buffSize: # wait till the buffer is full to start detecting bars ## This is where the bars detection magic actually happens - if float(framesList[middleFrame]['YMAX']) > 800 and float(framesList[middleFrame]['YMIN']) < 10 and float(framesList[middleFrame]['YDIF']) < 10: - if durationStart == "": - durationStart = float(framesList[middleFrame][pkt]) - print("Bars start at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") - durationEnd = float(framesList[middleFrame][pkt]) + # Check conditions + if (float(framesList[middleFrame]['YMAX']) > YMAX_thresh and + float(framesList[middleFrame]['YMIN']) < YMIN_thresh and + float(framesList[middleFrame]['YDIF']) < YDIF_thresh): + if durationStart == "": + durationStart = float(framesList[middleFrame][pkt]) + print("Bars start at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") + durationEnd = float(framesList[middleFrame][pkt]) else: if durationStart != "" and durationEnd != "" and durationEnd - durationStart > 2: print("Bars ended at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") @@ -514,12 +526,13 @@ def main(): break ###### Initialize values from the Config Parser - profile = {} # init a dictionary where we'll store reference values from our config file + # Determine if video values are 10 bit depth + bit_depth_10 = detectBitdepth(startObj,pkt,framesList,buffSize) + # init a dictionary where we'll store reference values from our config file + profile = {} # init a list of every tag available in a QCTools Report tagList = ["YMIN","YLOW","YAVG","YHIGH","YMAX","UMIN","ULOW","UAVG","UHIGH","UMAX","VMIN","VLOW","VAVG","VHIGH","VMAX","SATMIN","SATLOW","SATAVG","SATHIGH","SATMAX","HUEMED","HUEAVG","YDIF","UDIF","VDIF","TOUT","VREP","BRNG","mse_y","mse_u","mse_v","mse_avg","psnr_y","psnr_u","psnr_v","psnr_avg"] if args.p is not None: - # Determine if video values are 10 bit depth - bit_depth_10 = detectBitdepth(startObj,pkt,framesList,buffSize) # setup configparser config = configparser.RawConfigParser(allow_no_value=True) dn, fn = os.path.split(os.path.abspath(__file__)) # grip the dir where ~this script~ is located, also where config.txt should be located @@ -570,7 +583,7 @@ def main(): print("") print("Starting Bars Detection on " + baseName) print("") - detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize) + detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,bit_depth_10) ######## Iterate Through the XML for General Analysis ######## From c472ada67cc88ce1693f88dacaae5544b8412b09 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Mon, 30 Sep 2024 17:47:55 -0600 Subject: [PATCH 06/42] adding color bars evaluation funciton, after color bars detection. --- qct-parse.py | 104 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 21 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 9531ca4..8d727df 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -270,6 +270,7 @@ def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,b print("Bars ended at " + str(framesList[middleFrame][pkt]) + " (" + dts2ts(framesList[middleFrame][pkt]) + ")") break elem.clear() # we're done with that element so let's get it outta memory + return durationStart, durationEnd def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,frameCount=0,overallFrameFail=0): @@ -348,7 +349,7 @@ def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thum frameOver, thumbDelay = threshFinder(framesList[-1],args,startObj,pkt,tag,over,thumbPath,thumbDelay) if frameOver is True: kbeyond[tag] = kbeyond[tag] + 1 # note the over in the keyover dictionary - elif args.p is not None: # if we're using a profile + elif args.p or args.be is not None: # if we're using a profile or color bars evaluations for k,v in profile.items(): tag = k over = float(v) @@ -388,6 +389,54 @@ def detectBitdepth(startObj,pkt,framesList,buffSize): return bit_depth_10 +def evalBars(startObj,pkt,durationStart,durationEnd,framesList,buffSize): + # Define the keys for which you want to calculate the average + keys_to_check = ['YMAX', 'YMIN', 'UMIN', 'UMAX', 'VMIN', 'VMAX', 'SATMAX', 'SATMIN'] + # Initialize a dictionary to store the highest values for each key + maxBarsDict = {} + # adds the list keys_to_check as keys to a dictionary + for key_being_checked in keys_to_check: + # assign 'dummy' threshold to be overwritten + if "MAX" in key_being_checked: + maxBarsDict[key_being_checked] = 0 + elif "MIN" in key_being_checked: + maxBarsDict[key_being_checked] = 1023 + + with gzip.open(startObj) as xml: + for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): # iterparse the xml doc + if elem.attrib['media_type'] == "video": # get just the video frames + frame_pkt_dts_time = elem.attrib[pkt] # get the timestamps for the current frame we're looking at + if frame_pkt_dts_time >= str(durationStart): # only work on frames that are after the start time # only work on frames that are after the start time + if float(frame_pkt_dts_time) > durationEnd: # only work on frames that are before the end time + break + frameDict = {} # start an empty dict for the new frame + frameDict[pkt] = frame_pkt_dts_time # give the dict the timestamp, which we have now + for t in list(elem): # iterating through each attribute for each element + keySplit = t.attrib['key'].split(".") # split the names by dots + keyName = str(keySplit[-1]) # get just the last word for the key name + frameDict[keyName] = t.attrib['value'] # add each attribute to the frame dictionary + framesList.append(frameDict) + middleFrame = int(round(float(len(framesList))/2)) # i hate this calculation, but it gets us the middle index of the list as an integer + if len(framesList) == buffSize: # wait till the buffer is full to start detecting bars + ## This is where the bars detection magic actually happens + for colorbar_key in keys_to_check: + if colorbar_key in frameDict: + if "MAX" in colorbar_key: + # Convert the value to float and compare it with the current highest value + value = float(frameDict[colorbar_key]) + if value > maxBarsDict[colorbar_key]: + maxBarsDict[colorbar_key] = value + elif "MIN" in colorbar_key: + # Convert the value to float and compare it with the current highest value + value = float(frameDict[colorbar_key]) + if value < maxBarsDict[colorbar_key]: + maxBarsDict[colorbar_key] = value + # Convert highest values to integer + maxBarsDict = {colorbar_key: int(value) for colorbar_key, value in maxBarsDict.items()} + + return maxBarsDict + + # This function is admittedly very ugly, but what it puts out is very pretty. Need to revamp def printresults(kbeyond, frameCount, overallFrameFail): """ @@ -494,6 +543,7 @@ def main(): parser.add_argument('-ds','--durationStart',dest='ds',default=0, help="the duration in seconds to start analysis") parser.add_argument('-de','--durationEnd',dest='de',default=99999999, help="the duration in seconds to stop analysis") parser.add_argument('-bd','--barsDetection',dest='bd',action ='store_true',default=False, help="turns Bar Detection on and off") + parser.add_argument('-be','--barsEvaluation',dest='be',action ='store_true',default=False, help="turns Color Bar Evaluation on and off") parser.add_argument('-pr','--print',dest='pr',action='store_true',default=False, help="print over/under frame data to console window") parser.add_argument('-q','--quiet',dest='q',action='store_true',default=False, help="hide ffmpeg output from console window") args = parser.parse_args() @@ -532,21 +582,6 @@ def main(): profile = {} # init a list of every tag available in a QCTools Report tagList = ["YMIN","YLOW","YAVG","YHIGH","YMAX","UMIN","ULOW","UAVG","UHIGH","UMAX","VMIN","VLOW","VAVG","VHIGH","VMAX","SATMIN","SATLOW","SATAVG","SATHIGH","SATMAX","HUEMED","HUEAVG","YDIF","UDIF","VDIF","TOUT","VREP","BRNG","mse_y","mse_u","mse_v","mse_avg","psnr_y","psnr_u","psnr_v","psnr_avg"] - if args.p is not None: - # setup configparser - config = configparser.RawConfigParser(allow_no_value=True) - dn, fn = os.path.split(os.path.abspath(__file__)) # grip the dir where ~this script~ is located, also where config.txt should be located - # assign config based on bit depth of tag values - if bit_depth_10: - config.read(os.path.join(dn,"qct-parse_10bit_config.txt")) # read in the config file - else: - config.read(os.path.join(dn,"qct-parse_8bit_config.txt")) # read in the config file - template = args.p # get the profile/ section name from CLI - for t in tagList: # loop thru every tag available and - try: # see if it's in the config section - profile[t.replace("_",".")] = config.get(template,t) # if it is, replace _ necessary for config file with . which xml attributes use, assign the value in config - except: # if no config tag exists, do nothing so we can move faster - pass # set the start and end duration times if args.bd: @@ -583,20 +618,47 @@ def main(): print("") print("Starting Bars Detection on " + baseName) print("") - detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,bit_depth_10) + durationStart, durationEnd = detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,bit_depth_10) + if args.be and durationStart != "" and durationEnd != "": + maxBarsDict = evalBars(startObj,pkt,durationStart,durationEnd,framesList,buffSize) + if maxBarsDict is None: + print("\nSomehting went wrong - cannot run colorbars evaluation") + else: + print("\nColor bars found, comparing color bars thresholds with the rest of the video") + durationStart = 0 + durationEnd = 99999999 + profile = maxBarsDict + kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) + printresults(kbeyond,frameCount,overallFrameFail) + else: + durationStart = "" + durationEnd = "" + + if args.p is not None: + # setup configparser + config = configparser.RawConfigParser(allow_no_value=True) + dn, fn = os.path.split(os.path.abspath(__file__)) # grip the dir where ~this script~ is located, also where config.txt should be located + # assign config based on bit depth of tag values + if bit_depth_10: + config.read(os.path.join(dn,"qct-parse_10bit_config.txt")) # read in the config file + else: + config.read(os.path.join(dn,"qct-parse_8bit_config.txt")) # read in the config file + template = args.p # get the profile/ section name from CLI + for t in tagList: # loop thru every tag available and + try: # see if it's in the config section + profile[t.replace("_",".")] = config.get(template,t) # if it is, replace _ necessary for config file with . which xml attributes use, assign the value in config + except: # if no config tag exists, do nothing so we can move faster + pass - ######## Iterate Through the XML for General Analysis ######## print("") print("Starting Analysis on " + baseName) print("") kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) - print(f"\nFinished Processing File: " + baseName + ".qctools.xml.gz") print("") - - + # do some maths for the printout if args.o or args.u or args.p is not None: printresults(kbeyond,frameCount,overallFrameFail) From 39709c2f6779f51c278bbee9315bffbfd4a6b625 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Tue, 1 Oct 2024 10:12:59 -0600 Subject: [PATCH 07/42] added -be to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index cb49d1e..2010a0d 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ You can run a single tag against a supplied value from the CLI or run multiple t -de, --durationEnd | the duration in seconds to stop analysis (ffmpeg equivalent -t) -bd, --barsDetection | bar detection on/ off, default off + + -be, --barsEvaluation | if bars are found, use peak values from color bars as 'profile' -pr, --print | print over/under frame data to console window, default off From 3f1a6d8fb737d08209111077eba2a98783218fb1 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Tue, 1 Oct 2024 14:03:04 -0600 Subject: [PATCH 08/42] removed print('') statements with \n --- qct-parse.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 8d727df..1d84231 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -615,9 +615,7 @@ def main(): ######## Iterate Through the XML for Bars detection ######## if args.bd: - print("") - print("Starting Bars Detection on " + baseName) - print("") + print(f"\nStarting Bars Detection on {baseName}\n") durationStart, durationEnd = detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,bit_depth_10) if args.be and durationStart != "" and durationEnd != "": maxBarsDict = evalBars(startObj,pkt,durationStart,durationEnd,framesList,buffSize) @@ -651,13 +649,10 @@ def main(): pass ######## Iterate Through the XML for General Analysis ######## - print("") - print("Starting Analysis on " + baseName) - print("") + print(f"\nStarting Analysis on {baseName}\n") kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) - print(f"\nFinished Processing File: " + baseName + ".qctools.xml.gz") - print("") + print(f"\nFinished Processing File: {baseName}.qctools.xml.gz\n") # do some maths for the printout if args.o or args.u or args.p is not None: From 0673ca9f7db3a0a4fb8f0e52e43b894fe9b8770c Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Tue, 1 Oct 2024 17:07:08 -0600 Subject: [PATCH 09/42] adding asci colors to print results, and args conditional statement that contextualize output of print results --- qct-parse.py | 136 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 82 insertions(+), 54 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 1d84231..415224b 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -16,7 +16,7 @@ import sys # system stuff import re # can't spell parse without re fam import time -from distutils import spawn # dependency checking +import shutil # dependency checking # check that we have required software installed @@ -32,7 +32,7 @@ def dependencies(): """ depends = ['ffmpeg','ffprobe'] for d in depends: - if spawn.find_executable(d) is None: + if shutil.which(d) is None: print("Buddy, you gotta install " + d) sys.exit() return @@ -416,7 +416,6 @@ def evalBars(startObj,pkt,durationStart,durationEnd,framesList,buffSize): keyName = str(keySplit[-1]) # get just the last word for the key name frameDict[keyName] = t.attrib['value'] # add each attribute to the frame dictionary framesList.append(frameDict) - middleFrame = int(round(float(len(framesList))/2)) # i hate this calculation, but it gets us the middle index of the list as an integer if len(framesList) == buffSize: # wait till the buffer is full to start detecting bars ## This is where the bars detection magic actually happens for colorbar_key in keys_to_check: @@ -437,55 +436,77 @@ def evalBars(startObj,pkt,durationStart,durationEnd,framesList,buffSize): return maxBarsDict -# This function is admittedly very ugly, but what it puts out is very pretty. Need to revamp +# Print results from analyzeIt def printresults(kbeyond, frameCount, overallFrameFail): - """ - Prints the analysis results of frame data, including counts of frames exceeding thresholds - for various tags and the percentage of total frames that are affected. + """ + Prints the analysis results of frame data, including counts of frames exceeding thresholds + for various tags and the percentage of total frames that are affected. + + Args: + kbeyond (dict): A dictionary where keys are tag names and values are the counts of frames + that exceed the threshold for each tag. + frameCount (int): The total number of frames analyzed. + overallFrameFail (int): The number of frames where at least one tag exceeds its threshold. + + Prints: + - The total number of frames analyzed. + - A breakdown of frame counts for each tag in `kbeyond` and the corresponding percentage + of the total frames that exceeded the tag's threshold. + - The overall count and percentage of frames that failed at least one threshold. + + Notes: + - If `frameCount` is zero, it prints "TotalFrames: 0" and returns early. + - Percentages are formatted as whole numbers (e.g., "100"), two decimal places + (e.g., "12.34"), or "<0.01" for values less than 0.01%. + """ + # Define ANSI escape codes for color and formatting + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + RESET = "\033[0m" + + RED = "\033[91m" + YELLOW = "\033[93m" + GREEN = "\033[92m" + + def format_percentage(value): + percent = value * 100 + if percent == 100: + return "100" + elif percent == 0: + return "0" + elif percent < 0.01: + return "<0.01" + else: + return f"{percent:.2f}" + + def color_percentage(value): + percent = value * 100 + if percent > 10: + return RED + elif percent > 1: + return YELLOW + else: + return GREEN - Args: - kbeyond (dict): A dictionary where keys are tag names and values are the counts of frames - that exceed the threshold for each tag. - frameCount (int): The total number of frames analyzed. - overallFrameFail (int): The number of frames where at least one tag exceeds its threshold. - - Prints: - - The total number of frames analyzed. - - A breakdown of frame counts for each tag in `kbeyond` and the corresponding percentage - of the total frames that exceeded the tag's threshold. - - The overall count and percentage of frames that failed at least one threshold. - - Notes: - - If `frameCount` is zero, it prints "TotalFrames: 0" and returns early. - - Percentages are formatted as whole numbers (e.g., "100"), two decimal places - (e.g., "12.34"), or "<0.01" for values less than 0.01%. - """ - def format_percentage(value): - percent = value * 100 - if percent == 100: - return "100" - elif percent == 0: - return "0" - elif percent < 0.01: - return "<0.01" - else: - return f"{percent:.2f}" - - if frameCount == 0: - print("TotalFrames:\t0") - return - - print(f"\nTotalFrames:\t{frameCount}\n") - print("By Tag:\n") - - for tag, count in kbeyond.items(): - percent_over_string = format_percentage(count / frameCount) - print(f"{tag}:\t{count}\t{percent_over_string}\t% of the total # of frames\n") - - print("Overall:\n") - percent_overall_string = format_percentage(overallFrameFail / frameCount) - print(f"Frames With At Least One Fail:\t{overallFrameFail}\t{percent_overall_string}\t% of the total # of frames\n") - print("**************************\n") + if frameCount == 0: + print(f"{UNDERLINE}TotalFrames:{RESET}\t0") + return + + print(f"\n{UNDERLINE}TotalFrames{RESET}:\t{frameCount}\n") + print(f"{UNDERLINE}By Tag{RESET}:\n") + + for tag, count in kbeyond.items(): + percent = count / frameCount + percent_over_string = format_percentage(percent) + color = color_percentage(percent) + print(f"{BOLD}{tag}{RESET}:\t{count}\t{color}{percent_over_string}{RESET}\t% of the total # of frames\n") + + print(f"{BOLD}Overall:{RESET}\n") + overall_percent = overallFrameFail / frameCount + percent_overall_string = format_percentage(overall_percent) + color = color_percentage(overall_percent) + print(f"Frames With At Least One Fail:\t{overallFrameFail}\t{color}{percent_overall_string}{RESET}\t% of the total # of frames\n") + print(f"{BOLD}**************************{RESET}\n") def main(): @@ -620,9 +641,12 @@ def main(): if args.be and durationStart != "" and durationEnd != "": maxBarsDict = evalBars(startObj,pkt,durationStart,durationEnd,framesList,buffSize) if maxBarsDict is None: - print("\nSomehting went wrong - cannot run colorbars evaluation") + print("\nSomething went wrong - cannot run colorbars evaluation") else: - print("\nColor bars found, comparing color bars thresholds with the rest of the video") + print("\nNow compairing peak values of color bars to the rest of the video.") + print("\nReporting frames outside of these thresholds:") + for tag, value in maxBarsDict.items(): + print(tag, value) durationStart = 0 durationEnd = 99999999 profile = maxBarsDict @@ -649,8 +673,12 @@ def main(): pass ######## Iterate Through the XML for General Analysis ######## - print(f"\nStarting Analysis on {baseName}\n") - kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) + if args.p: + print(f"\nStarting Analysis on {baseName} using assigned profile\n") + kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) + elif args.t and args.o or args.u: + print(f"\nStarting Analysis on {baseName} using user specified tag threshold\n") + kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) print(f"\nFinished Processing File: {baseName}.qctools.xml.gz\n") From 054a876106f17278c1fb4c6d5c4eee87026dd800 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Wed, 2 Oct 2024 13:00:17 -0600 Subject: [PATCH 10/42] made print_peak_colorbars() function --- qct-parse.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 415224b..8c67249 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -434,6 +434,28 @@ def evalBars(startObj,pkt,durationStart,durationEnd,framesList,buffSize): maxBarsDict = {colorbar_key: int(value) for colorbar_key, value in maxBarsDict.items()} return maxBarsDict + + +def print_peak_colorbars(maxBarsDict): + # ASCI formatting + BOLD = "\033[1m" + RESET = "\033[0m" + + print("\nReporting frames outside of these thresholds:") + + # Create two lists for even and odd indices + tags = list(maxBarsDict.keys()) + values = list(maxBarsDict.values()) + + # Print even-indexed tags and values on the first line + for i in range(0, len(tags), 2): + print(f"{BOLD}{tags[i]:<6}{RESET} {values[i]:<5}", end=" ") + print() # Move to the next line + + # Print odd-indexed tags and values on the second line + for i in range(1, len(tags), 2): + print(f"{BOLD}{tags[i]:<6}{RESET} {values[i]:<5}", end=" ") + print() # Move to the next line # Print results from analyzeIt @@ -644,9 +666,8 @@ def main(): print("\nSomething went wrong - cannot run colorbars evaluation") else: print("\nNow compairing peak values of color bars to the rest of the video.") - print("\nReporting frames outside of these thresholds:") - for tag, value in maxBarsDict.items(): - print(tag, value) + print_peak_colorbars(maxBarsDict) + # Reset start and stop time to eval the whole video (color bars won't be flagged because we already have their max values) durationStart = 0 durationEnd = 99999999 profile = maxBarsDict From 88d821c31ddc3c3fe3fbd7ddc62c5b8bbd8cb269 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Wed, 2 Oct 2024 13:13:18 -0600 Subject: [PATCH 11/42] small spell check changes --- qct-parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 8c67249..a046ba4 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -345,7 +345,7 @@ def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thum over = float(args.o) if args.u: over = float(args.u) - # ACTAULLY DO THE THING ONCE FOR EACH TAG + # ACTUALLY DO THE THING ONCE FOR EACH TAG frameOver, thumbDelay = threshFinder(framesList[-1],args,startObj,pkt,tag,over,thumbPath,thumbDelay) if frameOver is True: kbeyond[tag] = kbeyond[tag] + 1 # note the over in the keyover dictionary @@ -665,7 +665,7 @@ def main(): if maxBarsDict is None: print("\nSomething went wrong - cannot run colorbars evaluation") else: - print("\nNow compairing peak values of color bars to the rest of the video.") + print("\nNow comparing peak values of color bars to the rest of the video.") print_peak_colorbars(maxBarsDict) # Reset start and stop time to eval the whole video (color bars won't be flagged because we already have their max values) durationStart = 0 From 1e78130ffaed9de59999b20583200ca1ddfc2373 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Thu, 3 Oct 2024 14:50:33 -0600 Subject: [PATCH 12/42] New README --- README.md | 152 +++++++++++++++++++++++++++++------------------------- 1 file changed, 82 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 2010a0d..0a47a4a 100644 --- a/README.md +++ b/README.md @@ -1,102 +1,114 @@ -scripts for automating QCTools actions -qct-parse.py | find frames that are beyond thresholds for saturation, luma, etc +# QCTools Automation Scripts -makeqctoolsreport.py | make a qctools.xml.gz report for input video file +This repository contains scripts for automating common QCTools actions, such as parsing frame data for threshold violations and generating reports. -# qct-parse.py +## Overview -You can run a single tag against a supplied value from the CLI or run multiple tags against values set in the qct-parse_config.txt file. +### Scripts: -## arguments - -h, --help | show this help message and exit - - -i, --input | the path to the input qctools.xml.gz file - - -t, --tagname | the tag name you want to test, e.g. SATMAX - - -o, --over | the threshold overage number - - -u, --under | the threshold under number - - -p, --profile | compare frame data aginst tag values from config.txt file, us "-p default" for values from QCTools docs +- **`qct-parse.py`** + Finds frames that exceed thresholds for saturation, luma, and more. - -buff, --buffSize | Size of the circular buffer. if user enters an even number it'll default to the next largest number to make it odd, default size 11 - - -te, --thumbExport | export thumbnails on/ off, default off - - -ted, --thumbExportDelay | minimum frames between exported thumbs, default 9000 - - -tep, --thumbExportPath | Path to thumb export. if ommitted, uses the input base-path - - -ds, --durationStart | the duration in seconds to start analysis (ffmpeg equivalent -ss) - - -de, --durationEnd | the duration in seconds to stop analysis (ffmpeg equivalent -t) - - -bd, --barsDetection | bar detection on/ off, default off - - -be, --barsEvaluation | if bars are found, use peak values from color bars as 'profile' - - -pr, --print | print over/under frame data to console window, default off - - -q, --quiet | print ffmpeg output to console window, default off - - -## examples - -### single tags - +- **`makeqctoolsreport.py`** + Generates a QCTools `.xml.gz` report for a given input video file. + +--- + +# `qct-parse.py` + +Easily run a single tag against a supplied value or multiple tags using a configuration file (`qct-parse_config.txt`). + +## Arguments + +| Argument | Description | +|-----------------------------|-------------------------------------------------------------------------------------------------------| +| `-h`, `--help` | Show this help message and exit | +| `-i`, `--input` | Path to the input `qctools.xml.gz` file | +| `-t`, `--tagname` | The tag name you want to test (e.g., `SATMAX`) | +| `-o`, `--over` | Threshold overage number | +| `-u`, `--under` | Threshold under number | +| `-p`, `--profile` | Compare frame data against tag values from `config.txt`. Use `-p default` for QCTools default values | +| `-buff`, `--buffSize` | Circular buffer size. If even, defaults to the next odd number (default: 11) | +| `-te`, `--thumbExport` | Enable/disable thumbnail export (default: off) | +| `-ted`, `--thumbExportDelay`| Minimum frames between exported thumbnails (default: 9000) | +| `-tep`, `--thumbExportPath` | Path to thumbnail export. Uses input base-path if omitted | +| `-ds`, `--durationStart` | Start analysis from this time (seconds, equivalent to ffmpeg `-ss`) | +| `-de`, `--durationEnd` | End analysis after this time (seconds, equivalent to ffmpeg `-t`) | +| `-bd`, `--barsDetection` | Enable/disable bar detection (default: off) | +| `-be`, `--barsEvaluation` | Use peak values from color bars as 'profile' if bars are detected | +| `-pr`, `--print` | Print over/under frame data to console (default: off) | +| `-q`, `--quiet` | Suppress ffmpeg output in console (default: off) | + +## Examples + +### Run single tag tests +```bash python qct-parse.py -t SATMAX -o 235 -t YMIN -u 16 -i /path/to/report.mkv.qctools.xml.gz +``` -### run bars against default profile from QCTools docs - -python qct-parse.py -bd -p default -i /path/to/reportsmkv.qctools.xml.gz - -### print out thumbnails of frames beyond threshold +### Run bars detection using default QCTools profile +```bash +python qct-parse.py -bd -p default -i /path/to/report.mkv.qctools.xml.gz +``` -python qct-parse.py -p default -te -tep C:\path\to\export\folder -i C:\path\to\the\report.mkv.qctools.xml.gz +### Export thumbnails of frames beyond threshold +```bash +python qct-parse.py -p default -te -tep /path/to/export/folder -i /path/to/report.mkv.qctools.xml.gz +``` -## some handy applescript to grep individual tags - -### just percentages +## Handy Applescript for Tag Extraction +### Extract percentage values for `YMAX` +```bash python ./qct-parse.py -i input.mxf.qctools.xml.gz -bd -p lowTolerance | grep 'YMAX' | awk 'NR==1 {print $3}' +``` -### total number of frame failures - +### Get the total number of frame failures +```bash python ./qct-parse.py -i input.mxf.qctools.xml.gz -bd -p lowTolerance | grep 'YMAX' | awk 'NR==1 {print $2}' +``` -## dependencies +--- -Python 2.7.x. +# `makeqctoolsreport.py` -Requires that [lxml](http://lxml.de/) is installed on your system. For more info on how it's used, see [here](http://www.ibm.com/developerworks/library/x-hiperfparse/) +A Python port of Morgan’s [makeqctoolsreport.as](https://github.com/iamdamosuzuki/QCToolsReport), this script generates QCTools `.xml.gz` reports from input video files. -### For Windows users: +## Example Usage +```bash +python makeqctoolsreport.py /path/to/input.mxf +``` -We **strongly** suggest using the pre-compiled installer found [here](https://pypi.python.org/pypi/lxml/3.3.3#downloads) +--- -### For Mac users: +## Dependencies -Try pip first, then try the macport. More info can be found [here](http://lxml.de/installation.html) +Ensure Python 3.x.x is installed. -# makeqctoolsreport.py +Additionally, install the `lxml` library: +```bash +pip install lxml +``` -python port of Morgan's [makeqctoolsreport.as](https://github.com/iamdamosuzuki/QCToolsReport) +For more information on `lxml` usage, check out the [lxml documentation](http://lxml.de/). +### Windows Users -## example +We **strongly recommend** using the pre-compiled installer available [here](https://pypi.python.org/pypi/lxml/3.3.3#downloads). -python makeqctoolsreport.py /path/to/input.mxf +### Mac Users -## contributors +Try installing `lxml` with `pip` first. If you encounter issues, consider using MacPorts. Further instructions can be found [here](http://lxml.de/installation.html). -@eddycolloton +--- -@CoatesBrendan +## Contributors -@av_morgan +- [@eddycolloton](https://github.com/eddycolloton) +- [@CoatesBrendan](https://github.com/CoatesBrendan) +- [@av_morgan](https://github.com/av_morgan) -## maintainer +## Maintainer -@av_morgan +- [@av_morgan](https://github.com/av_morgan) From cf933cf620ecdc96192c270e6df99c4b523cd8ef Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Thu, 3 Oct 2024 14:57:00 -0600 Subject: [PATCH 13/42] Removing applescript grep code from README --- README.md | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 0a47a4a..fd5b650 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ This repository contains scripts for automating common QCTools actions, such as # `qct-parse.py` -Easily run a single tag against a supplied value or multiple tags using a configuration file (`qct-parse_config.txt`). +Run a single tag against a supplied value or multiple tags using a config file (`qct-parse_config.txt`). ## Arguments @@ -57,18 +57,6 @@ python qct-parse.py -bd -p default -i /path/to/report.mkv.qctools.xml.gz python qct-parse.py -p default -te -tep /path/to/export/folder -i /path/to/report.mkv.qctools.xml.gz ``` -## Handy Applescript for Tag Extraction - -### Extract percentage values for `YMAX` -```bash -python ./qct-parse.py -i input.mxf.qctools.xml.gz -bd -p lowTolerance | grep 'YMAX' | awk 'NR==1 {print $3}' -``` - -### Get the total number of frame failures -```bash -python ./qct-parse.py -i input.mxf.qctools.xml.gz -bd -p lowTolerance | grep 'YMAX' | awk 'NR==1 {print $2}' -``` - --- # `makeqctoolsreport.py` @@ -85,6 +73,7 @@ python makeqctoolsreport.py /path/to/input.mxf ## Dependencies Ensure Python 3.x.x is installed. +Requires FFmpeg. Additionally, install the `lxml` library: ```bash @@ -93,14 +82,6 @@ pip install lxml For more information on `lxml` usage, check out the [lxml documentation](http://lxml.de/). -### Windows Users - -We **strongly recommend** using the pre-compiled installer available [here](https://pypi.python.org/pypi/lxml/3.3.3#downloads). - -### Mac Users - -Try installing `lxml` with `pip` first. If you encounter issues, consider using MacPorts. Further instructions can be found [here](http://lxml.de/installation.html). - --- ## Contributors From d1cc944e53d864d939bb38952b049c7b0e7a6e83 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Thu, 3 Oct 2024 17:15:51 -0600 Subject: [PATCH 14/42] adding support for extracting report from mkvs --- qct-parse.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/qct-parse.py b/qct-parse.py index a046ba4..77a696a 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -436,6 +436,53 @@ def evalBars(startObj,pkt,durationStart,durationEnd,framesList,buffSize): return maxBarsDict +def extract_report_mkv(startObj): + + report_file_output = startObj.replace(".qctools.mkv", ".qctools.xml.gz") + if os.path.isfile(report_file_output): + while True: + user_input = input(f"The file {os.path.basename(report_file_output)} already exists. \nExtract xml.gz from {os.path.basename(startObj)} and overwrite existing file? \n(y/n):\n") + if user_input.lower() in ["yes", "y"]: + os.remove(report_file_output) + # Run ffmpeg command to extract xml.gz report + full_command = [ + 'ffmpeg', + '-hide_banner', + '-loglevel', 'panic', + '-dump_attachment:t:0', report_file_output, + '-i', startObj + ] + print(f'Extracting qctools.xml.gz report from {os.path.basename(startObj)}\n') + print(f'Running command: {" ".join(full_command)}\n') + subprocess.run(full_command) + break + elif user_input.lower() in ["no", "n"]: + print('Processing existing qctools report, not extracting file\n') + break + else: + print("Invalid input. Please enter yes/no.\n") + else: + # Run ffmpeg command to extract xml.gz report + full_command = [ + 'ffmpeg', + '-hide_banner', + '-loglevel', 'panic', + '-dump_attachment:t:0', report_file_output, + '-i', startObj + ] + print(f'Extracting qctools.xml.gz report from {os.path.basename(startObj)}\n') + print(f'Running command: {" ".join(full_command)}\n') + subprocess.run(full_command) + + if os.path.isfile(report_file_output): + startObj = report_file_output + else: + print(f'Unable to extract XML from QCTools mkv report file\n') + startObj = None + + return startObj + + def print_peak_colorbars(maxBarsDict): # ASCI formatting BOLD = "\033[1m" @@ -593,6 +640,10 @@ def main(): ##### Initialize variables and buffers ###### startObj = args.i.replace("\\","/") + extension = os.path.splitext(startObj)[1] + # If qctools report is in an MKV attachment, extract .qctools.xml.gz report + if extension.lower().endswith('mkv'): + startObj = extract_report_mkv(startObj) buffSize = int(args.buff) # cast the input buffer as an integer if buffSize%2 == 0: buffSize = buffSize + 1 From ff56d34ed6d64e9c46fa61dd2a4abc3bea454cd4 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Fri, 4 Oct 2024 10:58:21 -0600 Subject: [PATCH 15/42] added input files and logging sections to README --- README.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fd5b650..792bb10 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Run a single tag against a supplied value or multiple tags using a config file ( | Argument | Description | |-----------------------------|-------------------------------------------------------------------------------------------------------| | `-h`, `--help` | Show this help message and exit | -| `-i`, `--input` | Path to the input `qctools.xml.gz` file | +| `-i`, `--input` | Path to the input `qctools.xml.gz` or `qctools.mkv` file | | `-t`, `--tagname` | The tag name you want to test (e.g., `SATMAX`) | | `-o`, `--over` | Threshold overage number | | `-u`, `--under` | Threshold under number | @@ -49,7 +49,7 @@ python qct-parse.py -t SATMAX -o 235 -t YMIN -u 16 -i /path/to/report.mkv.qctool ### Run bars detection using default QCTools profile ```bash -python qct-parse.py -bd -p default -i /path/to/report.mkv.qctools.xml.gz +python qct-parse.py -bd -p default -i /path/to/report.mkv.qctools.mkv ``` ### Export thumbnails of frames beyond threshold @@ -57,6 +57,26 @@ python qct-parse.py -bd -p default -i /path/to/report.mkv.qctools.xml.gz python qct-parse.py -p default -te -tep /path/to/export/folder -i /path/to/report.mkv.qctools.xml.gz ``` +## Input files + +qct-parse.py will work with the following QCTools report formats: +``` +qctools.xml.gz +qctools.mkv +``` + +If the qctools.xml.gz report is in an MKV attachment, the qctools.xml.gz report file will be extracted and saved as a separate file. + +In order to export thumbnails, the QCTools report must be in the same directory as the video file it is describing, and must have the same file name as the report (excluding the `qctools.xml.gz`). + +## Logging + +A log file is created with the same name as the input file but with a '.log' extension. +For example: `some_video_file.mkv.qctools.xml.gz.log` + +Log files contain every instance of values over the specified threshold. For example: +`2024-10-03 17:02:35,737 SATMAX is over 181.02 with a value of 698.0 at duration 00:00:16.4500` + --- # `makeqctoolsreport.py` From 3b0f653401290107d72b42418035beb43f4e394d Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Fri, 4 Oct 2024 10:59:30 -0600 Subject: [PATCH 16/42] small change to the logging docstring --- qct-parse.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qct-parse.py b/qct-parse.py index a046ba4..a80431b 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -77,7 +77,6 @@ def initLog(inputPath): Initializes a log file for the given input file. The log file is created with the same name as the input file but with a '.log' extension. - Log entries include timestamps and messages, and the logging level is set to INFO. Args: inputPath (str): The file path for the input file, used to create the log file. From 478ed76d7c475bb4d57288d661ec3d50d5923dc2 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Fri, 4 Oct 2024 11:07:03 -0600 Subject: [PATCH 17/42] noted support for both 8 and 10 bit in README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 792bb10..ee9bf6b 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,10 @@ qctools.mkv If the qctools.xml.gz report is in an MKV attachment, the qctools.xml.gz report file will be extracted and saved as a separate file. +Both 10-bit and 8-bit values are supported. The bit depth will be detected automatically, and does not need to be specified. + +If you wish to edit the profiles stored in the config.txt files, please note that there is a separate config.txt for 8-bit and 10-bit values. + In order to export thumbnails, the QCTools report must be in the same directory as the video file it is describing, and must have the same file name as the report (excluding the `qctools.xml.gz`). ## Logging From 88d30507f151837544ee42ec4cf7f66b63d14b20 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Fri, 4 Oct 2024 11:09:01 -0600 Subject: [PATCH 18/42] small adjustments to white space in readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ee9bf6b..4377483 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ qctools.mkv If the qctools.xml.gz report is in an MKV attachment, the qctools.xml.gz report file will be extracted and saved as a separate file. -Both 10-bit and 8-bit values are supported. The bit depth will be detected automatically, and does not need to be specified. +Both 8-bit and 10-bit values are supported. The bit depth will be detected automatically, and does not need to be specified. If you wish to edit the profiles stored in the config.txt files, please note that there is a separate config.txt for 8-bit and 10-bit values. @@ -97,6 +97,7 @@ python makeqctoolsreport.py /path/to/input.mxf ## Dependencies Ensure Python 3.x.x is installed. + Requires FFmpeg. Additionally, install the `lxml` library: From 907c63dceb867237734089801b892a6ea1088490 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Fri, 4 Oct 2024 11:15:58 -0600 Subject: [PATCH 19/42] added exit() if report extraction from MKV fails --- qct-parse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qct-parse.py b/qct-parse.py index da8b067..d022ad0 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -363,6 +363,7 @@ def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thum elem.clear() # we're done with that element so let's get it outta memory return kbeyond, frameCount, overallFrameFail + def detectBitdepth(startObj,pkt,framesList,buffSize): bit_depth_10 = False with gzip.open(startObj) as xml: @@ -478,6 +479,7 @@ def extract_report_mkv(startObj): else: print(f'Unable to extract XML from QCTools mkv report file\n') startObj = None + exit() return startObj From 41786511b56d0dc037af128519405b137601c9de Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Fri, 4 Oct 2024 11:18:44 -0600 Subject: [PATCH 20/42] added exit() if thumbnail path is specified but thumbnail export is not --- qct-parse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qct-parse.py b/qct-parse.py index d022ad0..2c02214 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -691,6 +691,7 @@ def main(): # set the path for the thumbnail export if args.tep and not args.te: print("Buddy, you specified a thumbnail export path without specifying that you wanted to export the thumbnails. Please either add '-te' to your cli call or delete '-tep [path]'") + exit() if args.tep: # if user supplied thumbExportPath, use that thumbPath = str(args.tep) From 15aff56ebc2d0be0799f77cf5fa3fd6e288f575d Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Fri, 4 Oct 2024 11:31:27 -0600 Subject: [PATCH 21/42] adding ffprobe detection of resolution for thumbnail size --- qct-parse.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 2c02214..4f08c05 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -16,6 +16,7 @@ import sys # system stuff import re # can't spell parse without re fam import time +import json import shutil # dependency checking @@ -142,8 +143,39 @@ def threshFinder(inFrame,args,startObj,pkt,tag,over,thumbPath,thumbDelay): return False, thumbDelay # return false because it was NOT over and thumbDelay +def get_video_resolution(input_video): + """ + Use ffprobe to get the resolution of the input video file. + + Args: + input_video (str): Path to the input video file. + + Returns: + (width, height) (tuple): The width and height of the video. + """ + ffprobe_command = [ + 'ffprobe', '-v', 'error', '-select_streams', 'v:0', '-show_entries', + 'stream=width,height', '-of', 'json', input_video + ] + + process = subprocess.Popen(ffprobe_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = process.communicate() + + if process.returncode != 0: + print(f"Error getting resolution with ffprobe: {err.decode('utf-8')}") + return None + + # Parse the JSON output + video_info = json.loads(out) + + # Extract the width and height from the video stream info + width = video_info['streams'][0]['width'] + height = video_info['streams'][0]['height'] + + return width, height + + # print thumbnail images of overs/unders -# Need to update - file naming convention has changed def printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString): """ Generates a thumbnail image from the video based on a timestamp and attribute value. @@ -174,21 +206,35 @@ def printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString): #### init some variables using the args list inputVid = startObj.replace(".qctools.xml.gz", "") if os.path.isfile(inputVid): + # Get the resolution using ffprobe + resolution = get_video_resolution(inputVid) + if resolution: + width, height = resolution + else: + # Fall back to hardcoded resolution if ffprobe fails + width, height = 720, 486 + baseName = os.path.basename(startObj) baseName = baseName.replace(".qctools.xml.gz", "") outputFramePath = os.path.join(thumbPath,baseName + "." + tag + "." + str(tagValue) + "." + timeStampString + ".png") ffoutputFramePath = outputFramePath.replace(":",".") # for windows we gotta see if that first : for the drive has been replaced by a dot and put it back + match = '' match = re.search(r"[A-Z]\.\/",ffoutputFramePath) # matches pattern R./ which should be R:/ on windows if match: ffoutputFramePath = ffoutputFramePath.replace(".",":",1) # replace first instance of "." in string ffoutputFramePath - ffmpegString = "ffmpeg -ss " + timeStampString + ' -i "' + inputVid + '" -vframes 1 -s 720x486 -y "' + ffoutputFramePath + '"' # Hardcoded output frame size to 720x486 for now, need to infer from input eventually + + ffmpegString = ( + f'ffmpeg -ss {timeStampString} -i "{inputVid}" -vframes 1 ' + f'-s {width}x{height} -y "{ffoutputFramePath}"' + ) output = subprocess.Popen(ffmpegString,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True) out,err = output.communicate() # Decode byte strings to handle newlines properly out = out.decode('utf-8') err = err.decode('utf-8') + if args.q is False: print(out) print(err) From 9756a305dd187de04c4edf24ab8143247a141df6 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Fri, 4 Oct 2024 12:59:59 -0600 Subject: [PATCH 22/42] added -update 1 to ffmpeg command to supress ffmpeg warning expecting an image sequence w -vframes --- qct-parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qct-parse.py b/qct-parse.py index 4f08c05..5eabffb 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -227,7 +227,7 @@ def printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString): ffmpegString = ( f'ffmpeg -ss {timeStampString} -i "{inputVid}" -vframes 1 ' - f'-s {width}x{height} -y "{ffoutputFramePath}"' + f'-s {width}x{height} -y -update 1 "{ffoutputFramePath}"' ) output = subprocess.Popen(ffmpegString,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True) out,err = output.communicate() From db5b21765523be27d770222dae1f400c22dcf640 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Mon, 7 Oct 2024 16:46:25 -0600 Subject: [PATCH 23/42] allowing for multiple profiles to be provided to -p arg --- qct-parse.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 5eabffb..236c515 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -672,7 +672,7 @@ def main(): parser.add_argument('-t','--tagname',dest='t', help="the tag name you want to test, e.g. SATMAX") parser.add_argument('-o','--over',dest='o', help="the threshold overage number") parser.add_argument('-u','--under',dest='u', help="the threshold under number") - parser.add_argument('-p','--profile',dest='p',default=None,help="use values from your qct-parse-config.txt file, provide profile/ template name, e.g. 'default'") + parser.add_argument('-p','--profile', dest='p', nargs='*', default=None, help="use values from your qct-parse-config.txt file, provide profile/ template name, e.g. 'default'") parser.add_argument('-buff','--buffSize',dest='buff',default=11, help="Size of the circular buffer. if user enters an even number it'll default to the next largest number to make it odd (default size 11)") parser.add_argument('-te','--thumbExport',dest='te',action='store_true',default=False, help="export thumbnail") parser.add_argument('-ted','--thumbExportDelay',dest='ted',default=9000, help="minimum frames between exported thumbs") @@ -685,6 +685,10 @@ def main(): parser.add_argument('-q','--quiet',dest='q',action='store_true',default=False, help="hide ffmpeg output from console window") args = parser.parse_args() + ## Validate required arguments + if not args.i: + parser.error("the following arguments are required: -i [path to QCTools report]") + ##### Initialize variables and buffers ###### startObj = args.i.replace("\\","/") extension = os.path.splitext(startObj)[1] @@ -777,6 +781,8 @@ def main(): durationEnd = "" if args.p is not None: + # create list of profiles + list_of_templates = args.p # setup configparser config = configparser.RawConfigParser(allow_no_value=True) dn, fn = os.path.split(os.path.abspath(__file__)) # grip the dir where ~this script~ is located, also where config.txt should be located @@ -785,18 +791,17 @@ def main(): config.read(os.path.join(dn,"qct-parse_10bit_config.txt")) # read in the config file else: config.read(os.path.join(dn,"qct-parse_8bit_config.txt")) # read in the config file - template = args.p # get the profile/ section name from CLI - for t in tagList: # loop thru every tag available and - try: # see if it's in the config section - profile[t.replace("_",".")] = config.get(template,t) # if it is, replace _ necessary for config file with . which xml attributes use, assign the value in config - except: # if no config tag exists, do nothing so we can move faster - pass + for template in list_of_templates: + for t in tagList: # loop thru every tag available and + try: # see if it's in the config section + profile[t.replace("_",".")] = config.get(template,t) # if it is, replace _ necessary for config file with . which xml attributes use, assign the value in config + except: # if no config tag exists, do nothing so we can move faster + pass + ######## Iterate Through the XML for General Analysis ######## + print(f"\nStarting Analysis on {baseName} using assigned profile {template}\n") + kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) - ######## Iterate Through the XML for General Analysis ######## - if args.p: - print(f"\nStarting Analysis on {baseName} using assigned profile\n") - kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) - elif args.t and args.o or args.u: + if args.t and args.o or args.u: print(f"\nStarting Analysis on {baseName} using user specified tag threshold\n") kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) From b68697ef7003a14b89374fbf687c9a2f1f746286 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Tue, 8 Oct 2024 11:06:48 -0600 Subject: [PATCH 24/42] small updates to README --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4377483..6a4cc51 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # QCTools Automation Scripts -This repository contains scripts for automating common QCTools actions, such as parsing frame data for threshold violations and generating reports. +This repository contains scripts for automating analysis of QCTools reports. ## Overview ### Scripts: - **`qct-parse.py`** - Finds frames that exceed thresholds for saturation, luma, and more. + Finds frames that exceed thresholds for QCTool tag(s). Detect color bars with the `-bd` option. - **`makeqctoolsreport.py`** Generates a QCTools `.xml.gz` report for a given input video file. @@ -17,7 +17,7 @@ This repository contains scripts for automating common QCTools actions, such as # `qct-parse.py` -Run a single tag against a supplied value or multiple tags using a config file (`qct-parse_config.txt`). +Run a single tag against a supplied value or multiple tags using a config file (`qct-parse_[#]bit_config.txt`). ## Arguments @@ -57,6 +57,11 @@ python qct-parse.py -bd -p default -i /path/to/report.mkv.qctools.mkv python qct-parse.py -p default -te -tep /path/to/export/folder -i /path/to/report.mkv.qctools.xml.gz ``` +### Use peak values from detected color bars as thresholds +```bash +python qct-parse.py -bd -be -i /path/to/report.mkv.qctools.xml.gz +``` + ## Input files qct-parse.py will work with the following QCTools report formats: From 2e56e778d26293c4767cb9d59e6a91f28c08c2a9 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Sat, 12 Oct 2024 09:42:34 -0600 Subject: [PATCH 25/42] confirms specified profile matches one of the config sections, and throws parser error if both -p and -t args are used together. Would need to modify logic of analyzeIt and threshFinder to support both -p and -t together. --- qct-parse.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index 236c515..d5615e6 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -120,7 +120,7 @@ def threshFinder(inFrame,args,startObj,pkt,tag,over,thumbPath,thumbDelay): """ tagValue = float(inFrame[tag]) frame_pkt_dts_time = inFrame[pkt] - if "MIN" in tag or "LOW" in tag: + if "MIN" in tag or "LOW" in tag: under = over if tagValue < float(under): # if the attribute is under usr set threshold timeStampString = dts2ts(frame_pkt_dts_time) @@ -688,6 +688,9 @@ def main(): ## Validate required arguments if not args.i: parser.error("the following arguments are required: -i [path to QCTools report]") + + if args.p and args.t: + parser.error("Running both profile and individual tag thresholds is not currently supported. They must be run individually.") ##### Initialize variables and buffers ###### startObj = args.i.replace("\\","/") @@ -792,24 +795,28 @@ def main(): else: config.read(os.path.join(dn,"qct-parse_8bit_config.txt")) # read in the config file for template in list_of_templates: + # Check if the template is a valid section in the config + if not config.has_section(template): + print(f"Profile '{template}' does not match any section in the config.") + continue # Skip to the next template if section doesn't exist for t in tagList: # loop thru every tag available and try: # see if it's in the config section profile[t.replace("_",".")] = config.get(template,t) # if it is, replace _ necessary for config file with . which xml attributes use, assign the value in config except: # if no config tag exists, do nothing so we can move faster pass + ######## Iterate Through the XML for General Analysis ######## print(f"\nStarting Analysis on {baseName} using assigned profile {template}\n") kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) + printresults(kbeyond,frameCount,overallFrameFail) if args.t and args.o or args.u: print(f"\nStarting Analysis on {baseName} using user specified tag threshold\n") kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) + printresults(kbeyond,frameCount,overallFrameFail) print(f"\nFinished Processing File: {baseName}.qctools.xml.gz\n") - - # do some maths for the printout - if args.o or args.u or args.p is not None: - printresults(kbeyond,frameCount,overallFrameFail) + return dependencies() From b7099aa655da9f1aa725746caf5b9d5948fee247 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Sat, 12 Oct 2024 10:20:16 -0600 Subject: [PATCH 26/42] modifying analyzeIt, threshFinder, and how -o and -u args are parsed to allow for both -p and -t flags to be run together --- qct-parse.py | 94 +++++++++++++++++++++++----------------------------- 1 file changed, 42 insertions(+), 52 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index d5615e6..abeeff5 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -15,6 +15,7 @@ import math # used for rounding up buffer half import sys # system stuff import re # can't spell parse without re fam +import operator import time import json import shutil # dependency checking @@ -91,7 +92,7 @@ def initLog(inputPath): # finds stuff over/under threshold -def threshFinder(inFrame,args,startObj,pkt,tag,over,thumbPath,thumbDelay): +def threshFinder(inFrame,args,startObj,pkt,tag,over,thumbPath,thumbDelay,adhoc_tag): """ Evaluates whether a tag in a video frame exceeds or falls below a threshold value and logs the result. @@ -120,27 +121,26 @@ def threshFinder(inFrame,args,startObj,pkt,tag,over,thumbPath,thumbDelay): """ tagValue = float(inFrame[tag]) frame_pkt_dts_time = inFrame[pkt] - if "MIN" in tag or "LOW" in tag: - under = over - if tagValue < float(under): # if the attribute is under usr set threshold - timeStampString = dts2ts(frame_pkt_dts_time) - logging.warning(tag + " is under " + str(under) + " with a value of " + str(tagValue) + " at duration " + timeStampString) - if args.te and (thumbDelay > int(args.ted)): # if thumb export is turned on and there has been enough delay between this frame and the last exported thumb, then export a new thumb - printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString) - thumbDelay = 0 - return True, thumbDelay # return true because it was over and thumbDelay - else: - return False, thumbDelay # return false because it was NOT over and thumbDelay + if adhoc_tag: + if args.o: + comparision = operator.gt + elif args.u: + comparision = operator.lt else: - if tagValue > float(over): # if the attribute is over usr set threshold - timeStampString = dts2ts(frame_pkt_dts_time) - logging.warning(tag + " is over " + str(over) + " with a value of " + str(tagValue) + " at duration " + timeStampString) - if args.te and (thumbDelay > int(args.ted)): # if thumb export is turned on and there has been enough delay between this frame and the last exported thumb, then export a new thumb - printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString) - thumbDelay = 0 - return True, thumbDelay # return true because it was over and thumbDelay + if "MIN" in tag or "LOW" in tag: + comparision = operator.lt else: - return False, thumbDelay # return false because it was NOT over and thumbDelay + comparision = operator.gt + + if comparision(float(tagValue), float(over)): # if the attribute is over usr set threshold + timeStampString = dts2ts(frame_pkt_dts_time) + logging.warning(tag + " is over " + str(over) + " with a value of " + str(tagValue) + " at duration " + timeStampString) + if args.te and (thumbDelay > int(args.ted)): # if thumb export is turned on and there has been enough delay between this frame and the last exported thumb, then export a new thumb + printThumb(args,tag,startObj,thumbPath,tagValue,timeStampString) + thumbDelay = 0 + return True, thumbDelay # return true because it was over and thumbDelay + else: + return False, thumbDelay # return false because it was NOT over and thumbDelay def get_video_resolution(input_video): @@ -318,7 +318,7 @@ def detectBars(args,startObj,pkt,durationStart,durationEnd,framesList,buffSize,b return durationStart, durationEnd -def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,frameCount=0,overallFrameFail=0): +def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag=False,frameCount=0,overallFrameFail=0): """ Analyzes video frames from the QCTools report to detect threshold exceedances for specified tags or profiles and logs frame failures. @@ -357,11 +357,8 @@ def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thum """ kbeyond = {} # init a dict for each key which we'll use to track how often a given key is over fots = "" - if args.t: - kbeyond[args.t] = 0 - else: - for k,v in profile.items(): - kbeyond[k] = 0 + for k,v in profile.items(): + kbeyond[k] = 0 with gzip.open(startObj) as xml: for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): # iterparse the xml doc if elem.attrib['media_type'] == "video": # get just the video frames @@ -384,27 +381,16 @@ def analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thum if args.pr is True: # display "timestamp: Tag Value" (654.754100: YMAX 229) to the terminal window print(framesList[-1][pkt] + ": " + args.t + " " + framesList[-1][args.t]) # Now we can parse the frame data from the buffer! - if args.o or args.u and args.p is None: # if we're just doing a single tag - tag = args.t - if args.o: - over = float(args.o) - if args.u: - over = float(args.u) + for k,v in profile.items(): + tag = k + over = float(v) # ACTUALLY DO THE THING ONCE FOR EACH TAG - frameOver, thumbDelay = threshFinder(framesList[-1],args,startObj,pkt,tag,over,thumbPath,thumbDelay) + frameOver, thumbDelay = threshFinder(framesList[-1],args,startObj,pkt,tag,over,thumbPath,thumbDelay,adhoc_tag) if frameOver is True: - kbeyond[tag] = kbeyond[tag] + 1 # note the over in the keyover dictionary - elif args.p or args.be is not None: # if we're using a profile or color bars evaluations - for k,v in profile.items(): - tag = k - over = float(v) - # ACTUALLY DO THE THING ONCE FOR EACH TAG - frameOver, thumbDelay = threshFinder(framesList[-1],args,startObj,pkt,tag,over,thumbPath,thumbDelay) - if frameOver is True: - kbeyond[k] = kbeyond[k] + 1 # note the over in the key over dict - if not frame_pkt_dts_time in fots: # make sure that we only count each over frame once - overallFrameFail = overallFrameFail + 1 - fots = frame_pkt_dts_time # set it again so we don't dupe + kbeyond[k] = kbeyond[k] + 1 # note the over in the key over dict + if not frame_pkt_dts_time in fots: # make sure that we only count each over frame once + overallFrameFail = overallFrameFail + 1 + fots = frame_pkt_dts_time # set it again so we don't dupe thumbDelay = thumbDelay + 1 elem.clear() # we're done with that element so let's get it outta memory return kbeyond, frameCount, overallFrameFail @@ -688,9 +674,6 @@ def main(): ## Validate required arguments if not args.i: parser.error("the following arguments are required: -i [path to QCTools report]") - - if args.p and args.t: - parser.error("Running both profile and individual tag thresholds is not currently supported. They must be run individually.") ##### Initialize variables and buffers ###### startObj = args.i.replace("\\","/") @@ -777,7 +760,7 @@ def main(): durationStart = 0 durationEnd = 99999999 profile = maxBarsDict - kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) + kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag=False) printresults(kbeyond,frameCount,overallFrameFail) else: durationStart = "" @@ -807,12 +790,19 @@ def main(): ######## Iterate Through the XML for General Analysis ######## print(f"\nStarting Analysis on {baseName} using assigned profile {template}\n") - kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) + kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag=False) printresults(kbeyond,frameCount,overallFrameFail) if args.t and args.o or args.u: - print(f"\nStarting Analysis on {baseName} using user specified tag threshold\n") - kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList) + profile = {} + tag = args.t + if args.o: + over = float(args.o) + if args.u: + over = float(args.u) + profile[tag] = over + print(f"\nStarting Analysis on {baseName} using user specified tag {tag} w/ threshold {over}\n") + kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag = True) printresults(kbeyond,frameCount,overallFrameFail) print(f"\nFinished Processing File: {baseName}.qctools.xml.gz\n") From 8dd95510b16f3ba03df0f41b0ad7b964a89380fa Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Sun, 13 Oct 2024 16:47:30 -0600 Subject: [PATCH 27/42] small changes to parser help output --- qct-parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qct-parse.py b/qct-parse.py index abeeff5..4d46a33 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -662,7 +662,7 @@ def main(): parser.add_argument('-buff','--buffSize',dest='buff',default=11, help="Size of the circular buffer. if user enters an even number it'll default to the next largest number to make it odd (default size 11)") parser.add_argument('-te','--thumbExport',dest='te',action='store_true',default=False, help="export thumbnail") parser.add_argument('-ted','--thumbExportDelay',dest='ted',default=9000, help="minimum frames between exported thumbs") - parser.add_argument('-tep','--thumbExportPath',dest='tep',default='', help="Path to thumb export. if ommitted, it uses the input basename") + parser.add_argument('-tep','--thumbExportPath',dest='tep',default='', help="Path to thumb export. if omitted, it uses the input basename") parser.add_argument('-ds','--durationStart',dest='ds',default=0, help="the duration in seconds to start analysis") parser.add_argument('-de','--durationEnd',dest='de',default=99999999, help="the duration in seconds to stop analysis") parser.add_argument('-bd','--barsDetection',dest='bd',action ='store_true',default=False, help="turns Bar Detection on and off") @@ -673,7 +673,7 @@ def main(): ## Validate required arguments if not args.i: - parser.error("the following arguments are required: -i [path to QCTools report]") + parser.error("the following arguments are required: -i/--input [path to QCTools report]") ##### Initialize variables and buffers ###### startObj = args.i.replace("\\","/") From 660195a1310358b3f63f823b50856f7c1c664199 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Sun, 13 Oct 2024 17:01:14 -0600 Subject: [PATCH 28/42] throw error if over and under options are used together --- qct-parse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qct-parse.py b/qct-parse.py index 4d46a33..daba1af 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -674,6 +674,8 @@ def main(): ## Validate required arguments if not args.i: parser.error("the following arguments are required: -i/--input [path to QCTools report]") + if args.o and args.u: + parser.error("Both the -o and -u options were used. Cannot set threshold for both over and under, only one at a time.") ##### Initialize variables and buffers ###### startObj = args.i.replace("\\","/") From 4bb57e58fef7d6c6161745fadbd37d93eec93533 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Tue, 3 Dec 2024 10:25:07 -0700 Subject: [PATCH 29/42] small change to comments spacing --- qct-parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qct-parse.py b/qct-parse.py index daba1af..5c15e63 100644 --- a/qct-parse.py +++ b/qct-parse.py @@ -18,7 +18,7 @@ import operator import time import json -import shutil # dependency checking +import shutil # dependency checking # check that we have required software installed From 59f59a5e62124566338f6f6d3dbc115666c8d1c7 Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Tue, 3 Dec 2024 11:53:04 -0600 Subject: [PATCH 30/42] qct-parse is now packagable --- README.md | 21 ++++++++++++++++--- pyproject.toml | 15 +++++++++++++ qct_parse/__init__.py | 0 .../makeqctoolsreport.py | 5 +++-- overcatch.py => qct_parse/overcatch.py | 3 ++- qct-parse.py => qct_parse/qct_parse.py | 5 +++-- 6 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 pyproject.toml create mode 100644 qct_parse/__init__.py rename makeqctoolsreport.py => qct_parse/makeqctoolsreport.py (99%) rename overcatch.py => qct_parse/overcatch.py (99%) rename qct-parse.py => qct_parse/qct_parse.py (99%) diff --git a/README.md b/README.md index 6a4cc51..fb5de68 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,32 @@ This repository contains scripts for automating analysis of QCTools reports. ## Overview +### Install from source: + +* Create a new Python Virtual Environment for qct_parse + * Unix based (Mac or Linux): + `python3 -m venv venv` + * Windows: + `py -m venv venv` +* Activate virtual env + * Unix based: + `source ./venv/bin/activate` + * Windows: + `venv\scripts\activate` +* Install Package + `python -m pip install .` + ### Scripts: -- **`qct-parse.py`** +- **`qct-parse`** Finds frames that exceed thresholds for QCTool tag(s). Detect color bars with the `-bd` option. -- **`makeqctoolsreport.py`** +- **`makeqctoolsreport`** Generates a QCTools `.xml.gz` report for a given input video file. --- -# `qct-parse.py` +# `qct-parse` Run a single tag against a supplied value or multiple tags using a config file (`qct-parse_[#]bit_config.txt`). diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1f7a928 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "qct-parse" +version = "0.2.1.dev0" +dependencies =[ + 'lxml' +] + +[tool.setuptools] +packages=['qct_parse'] + +[project.scripts] +makeqctoolsreport = "qct_parse.makeqctoolsreport:main" +overcatch = "qct_parse.overcatch:main" +qct-parse = "qct_parse.qct_parse:main" + diff --git a/qct_parse/__init__.py b/qct_parse/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/makeqctoolsreport.py b/qct_parse/makeqctoolsreport.py similarity index 99% rename from makeqctoolsreport.py rename to qct_parse/makeqctoolsreport.py index ee9fac7..f06103e 100644 --- a/makeqctoolsreport.py +++ b/qct_parse/makeqctoolsreport.py @@ -163,6 +163,7 @@ def main(): transcode(startObj,outPath) startObj = startObj + ".temp1.nut" makeReport(startObj,outPath) - + dependencies() -main() \ No newline at end of file +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/overcatch.py b/qct_parse/overcatch.py similarity index 99% rename from overcatch.py rename to qct_parse/overcatch.py index c7530e1..fde9cc7 100644 --- a/overcatch.py +++ b/qct_parse/overcatch.py @@ -86,4 +86,5 @@ def printout(barOutDict,contentOutDict,profileDict): for cod in contentOutDict: print("Frames beyond " + profileDict[cod] + " for " + contentOutDict[cod]) -main() +if __name__ == '__main__': + main() diff --git a/qct-parse.py b/qct_parse/qct_parse.py similarity index 99% rename from qct-parse.py rename to qct_parse/qct_parse.py index daba1af..99226bc 100644 --- a/qct-parse.py +++ b/qct_parse/qct_parse.py @@ -811,5 +811,6 @@ def main(): return -dependencies() -main() \ No newline at end of file +if __name__ == '__main__': + dependencies() + main() \ No newline at end of file From 198c2dfd1ab71c442916fe07b734d64852068108 Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Tue, 3 Dec 2024 12:51:27 -0600 Subject: [PATCH 31/42] config file location is set with an environment variable --- qct_parse/qct_parse.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/qct_parse/qct_parse.py b/qct_parse/qct_parse.py index a88d5d8..0a90b4f 100644 --- a/qct_parse/qct_parse.py +++ b/qct_parse/qct_parse.py @@ -21,6 +21,9 @@ import shutil # dependency checking +CONFIG_ENVIRONMENT_VARIABLE_NAME = 'QCT_PARSE_CONFIG_DIRECTORY' + + # check that we have required software installed def dependencies(): """ @@ -775,6 +778,13 @@ def main(): config = configparser.RawConfigParser(allow_no_value=True) dn, fn = os.path.split(os.path.abspath(__file__)) # grip the dir where ~this script~ is located, also where config.txt should be located # assign config based on bit depth of tag values + config_file_path = os.environ.get(CONFIG_ENVIRONMENT_VARIABLE_NAME) + if not config_file_path: + raise AttributeError( + f'{CONFIG_ENVIRONMENT_VARIABLE_NAME} is not set. ' + f'This is needed to locate config files.' + ) + if bit_depth_10: config.read(os.path.join(dn,"qct-parse_10bit_config.txt")) # read in the config file else: From 2b4d44a3a8e7b8a048538d3b5beb1434b8613a86 Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Tue, 3 Dec 2024 13:02:40 -0600 Subject: [PATCH 32/42] GitHub actions tests codes across all supported versions of python --- .github/workflows/tests.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..2c6d6ff --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,37 @@ +on: [push, pull_request] +name: Supported Python Compatibility Test +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-2019] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + include: + - os: macos-latest + - os: ubuntu-latest + - os: windows-2019 + fail-fast: false + name: Python ${{ matrix.python-version }} ${{ matrix.os }} build + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' # caching pip dependencies + - name: Install requirements + run: | + pip install tox + - uses: actions/github-script@v7 + id: tox-env + with: + result-encoding: string + script: | + const frontend = "${{matrix.frontend}}" + const toxEnv = "py${{matrix.python-version}}".replace('.','') + if(frontend === ""){ + return toxEnv + } + return "py${{matrix.python-version}}".replace('.','') + "-${{matrix.frontend}}" + - name: Run tox + run: tox -e ${{ steps.tox-env.outputs.result }} From 0c4fddac8b106b8c5ffe880b7c89d3a3b0f7b7d6 Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Tue, 3 Dec 2024 13:09:29 -0600 Subject: [PATCH 33/42] support tox tool for testing --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1f7a928..939b5e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,3 +13,5 @@ makeqctoolsreport = "qct_parse.makeqctoolsreport:main" overcatch = "qct_parse.overcatch:main" qct-parse = "qct_parse.qct_parse:main" +[tool.tox] +envlist = ["3.9", "3.10", "3.11", "3.12", "3.13"] From 2cd3f9b30d977c3bc1089eab4046fc70b8593921 Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Tue, 3 Dec 2024 14:39:23 -0600 Subject: [PATCH 34/42] QCT_PARSE_CONFIG_DIRECTORY is now optional environment variable --- pyproject.toml | 4 ++++ .../qct-parse_10bit_config.txt | 0 .../qct-parse_8bit_config.txt | 0 qct_parse/qct_parse.py | 11 ++++------- 4 files changed, 8 insertions(+), 7 deletions(-) rename qct-parse_10bit_config.txt => qct_parse/qct-parse_10bit_config.txt (100%) rename qct-parse_8bit_config.txt => qct_parse/qct-parse_8bit_config.txt (100%) diff --git a/pyproject.toml b/pyproject.toml index 1f7a928..141e843 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,10 @@ dependencies =[ [tool.setuptools] packages=['qct_parse'] +include-package-data = true + +[tool.setuptools.package-data] +qct_parse = ["./qct-parse_8bit_config.txt", "./qct-parse_10bit_config.txt"] [project.scripts] makeqctoolsreport = "qct_parse.makeqctoolsreport:main" diff --git a/qct-parse_10bit_config.txt b/qct_parse/qct-parse_10bit_config.txt similarity index 100% rename from qct-parse_10bit_config.txt rename to qct_parse/qct-parse_10bit_config.txt diff --git a/qct-parse_8bit_config.txt b/qct_parse/qct-parse_8bit_config.txt similarity index 100% rename from qct-parse_8bit_config.txt rename to qct_parse/qct-parse_8bit_config.txt diff --git a/qct_parse/qct_parse.py b/qct_parse/qct_parse.py index 0a90b4f..380eb5d 100644 --- a/qct_parse/qct_parse.py +++ b/qct_parse/qct_parse.py @@ -777,13 +777,10 @@ def main(): # setup configparser config = configparser.RawConfigParser(allow_no_value=True) dn, fn = os.path.split(os.path.abspath(__file__)) # grip the dir where ~this script~ is located, also where config.txt should be located - # assign config based on bit depth of tag values - config_file_path = os.environ.get(CONFIG_ENVIRONMENT_VARIABLE_NAME) - if not config_file_path: - raise AttributeError( - f'{CONFIG_ENVIRONMENT_VARIABLE_NAME} is not set. ' - f'This is needed to locate config files.' - ) + # assign config based on bit depth of tag values + if CONFIG_ENVIRONMENT_VARIABLE_NAME in os.environ: + print(f"Using config files in ${CONFIG_ENVIRONMENT_VARIABLE_NAME}") + dn = os.environ[CONFIG_ENVIRONMENT_VARIABLE_NAME] if bit_depth_10: config.read(os.path.join(dn,"qct-parse_10bit_config.txt")) # read in the config file From a1055a4c3f1a33f082251fe7b55cfade7bfa0aaa Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Tue, 3 Dec 2024 15:27:00 -0600 Subject: [PATCH 35/42] add initial test --- README.md | 9 +++++++++ pyproject.toml | 5 +++++ tests/test_qct_parse.py | 4 ++++ 3 files changed, 18 insertions(+) create mode 100644 tests/test_qct_parse.py diff --git a/README.md b/README.md index fb5de68..cdf03f8 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,15 @@ This repository contains scripts for automating analysis of QCTools reports. * Install Package `python -m pip install .` +### Test Code + +* Activate virtual env (see Install from source) +* Install pytest + `pip install pytest` +* Run tests + `python -m pytest` + + ### Scripts: - **`qct-parse`** diff --git a/pyproject.toml b/pyproject.toml index e7ee374..0de5939 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,3 +19,8 @@ qct-parse = "qct_parse.qct_parse:main" [tool.tox] envlist = ["3.9", "3.10", "3.11", "3.12", "3.13"] + +[tool.tox.env_run_base] +description = "Run test under {base_python}" +commands = [["pytest"]] +deps = ["pytest"] diff --git a/tests/test_qct_parse.py b/tests/test_qct_parse.py new file mode 100644 index 0000000..483d2c1 --- /dev/null +++ b/tests/test_qct_parse.py @@ -0,0 +1,4 @@ +from qct_parse import qct_parse + +def test_dts2ts(): + assert qct_parse.dts2ts("0.0330000") == '00:00:00.0330' From 751d8e5baa3161757cbd6fdf7f16325d6079892e Mon Sep 17 00:00:00 2001 From: Annie Schweikert <31541957+aeschweik@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:28:25 -0600 Subject: [PATCH 36/42] Add tag definitions to README Links out to BAVC documentation --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6a4cc51..7aaa486 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Run a single tag against a supplied value or multiple tags using a config file ( |-----------------------------|-------------------------------------------------------------------------------------------------------| | `-h`, `--help` | Show this help message and exit | | `-i`, `--input` | Path to the input `qctools.xml.gz` or `qctools.mkv` file | -| `-t`, `--tagname` | The tag name you want to test (e.g., `SATMAX`) | +| `-t`, `--tagname` | The tag name you want to test (e.g., `SATMAX`); see table of tag names below for list | | `-o`, `--over` | Threshold overage number | | `-u`, `--under` | Threshold under number | | `-p`, `--profile` | Compare frame data against tag values from `config.txt`. Use `-p default` for QCTools default values | @@ -40,6 +40,20 @@ Run a single tag against a supplied value or multiple tags using a config file ( | `-pr`, `--print` | Print over/under frame data to console (default: off) | | `-q`, `--quiet` | Suppress ffmpeg output in console (default: off) | +## Tags + +| Tag category | Tag names | +|-----------------------------|-------------------------------------------------------------------------------------------------------| +| [YUV values](https://bavc.github.io/qctools/filter_descriptions.html#yuv) | `YMIN,YLOW,YAVG,YHIGH,YMAX`
`UMIN,ULOW,UAVG,UHIGH,UMAX`
`VMIN,VLOW,VAVG,VHIGH,VMAX` | +| [YUV values (difference)](https://bavc.github.io/qctools/filter_descriptions.html#diff) | `YDIF,UDIF,VDIF` | +| [Saturation values](https://bavc.github.io/qctools/filter_descriptions.html#saturation) | `SATMIN,SATLOW,SATAVG,SATHIGH,SATMAX` | +| [Hue values](https://bavc.github.io/qctools/filter_descriptions.html#hue) | `HUEMED,HUEAVG` | +| [Temporal outliers](https://bavc.github.io/qctools/filter_descriptions.html#tout) | `TOUT` | +| [Vertical line repetitions](https://bavc.github.io/qctools/filter_descriptions.html#vrep) | `VREP` | +| [Broadcast range](https://bavc.github.io/qctools/filter_descriptions.html#brng) | `BRNG` | +| [Mean square error fields](https://bavc.github.io/qctools/filter_descriptions.html#msef) | `mse_y,mse_u,mse_v,mse_avg` | +| [Peak signal to noise ratio fields](https://bavc.github.io/qctools/filter_descriptions.html#psnrf) | `psnr_y,psnr_u,psnr_v,psnr_avg` | + ## Examples ### Run single tag tests From b7f57a8082142656053a9307f55f90071a45890b Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Tue, 3 Dec 2024 16:11:14 -0600 Subject: [PATCH 37/42] split main function into get_arg_parser and parse_qc_tools_report functions --- qct_parse/qct_parse.py | 142 +++++++++++++++++++++-------------------- 1 file changed, 74 insertions(+), 68 deletions(-) diff --git a/qct_parse/qct_parse.py b/qct_parse/qct_parse.py index 380eb5d..6de8ea6 100644 --- a/qct_parse/qct_parse.py +++ b/qct_parse/qct_parse.py @@ -613,49 +613,7 @@ def color_percentage(value): print(f"Frames With At Least One Fail:\t{overallFrameFail}\t{color}{percent_overall_string}{RESET}\t% of the total # of frames\n") print(f"{BOLD}**************************{RESET}\n") - -def main(): - """ - Main function that parses QCTools XML files, applies analysis, and optionally exports thumbnails. - - This function handles command-line arguments to process a QCTools report, extract frame data from the XML, - apply threshold analysis for broadcast values, optionally detect color bars, and export analysis results to - the console or thumbnails. - - Command-line Arguments: - -i, --input (str): Path to the input QCTools XML.gz file. - -t, --tagname (str): Tag name to analyze, e.g., SATMAX. - -o, --over (float): Overage threshold for the tag specified. - -u, --under (float): Under threshold for the tag specified. - -p, --profile (str): Profile or template name from the qct-parse_config.txt file, e.g., 'default'. - -buff, --buffSize (int): Circular buffer size. Defaults to 11, ensures odd number. - -te, --thumbExport: Export thumbnails if flag is set. - -ted, --thumbExportDelay (int): Minimum number of frames between exported thumbnails. - -tep, --thumbExportPath (str): Path to export thumbnails, defaults to input basename if not provided. - -ds, --durationStart (float): Start time in seconds for analysis. - -de, --durationEnd (float): End time in seconds for analysis. - -bd, --barsDetection: Flag to enable color bars detection. - -pr, --print: Flag to print frame data to the console. - -q, --quiet: Hide ffmpeg output if flag is set. - - Workflow: - 1. Parse command-line arguments. - 2. Optionally load reference threshold values from a profile in `qct-parse_config.txt`. - 3. Initialize buffers, frame counters, and paths for thumbnail export. - 4. Check for `pkt_dts_time` or `pkt_pts_time` in the QCTools XML file. - 5. Set the analysis duration start and end times. - 6. Perform bars detection if enabled, otherwise proceed with general analysis. - 7. Call the `analyzeIt` function to perform frame-by-frame analysis and calculate exceedances. - 8. Print results using `printresults` if applicable. - 9. Handle errors or invalid input (e.g., missing thumbnail export flag but specifying a path). - - Example usage: - python qct-parse.py -i sample.qctools.xml.gz -t SATMAX -o 5.0 -u -5.0 -te - - Returns: - None: The function processes the XML file, performs analysis, and optionally exports thumbnails and prints results to the console. - """ - #### init the stuff from the cli ######## +def get_arg_parser(): parser = argparse.ArgumentParser(description="parses QCTools XML files for frames beyond broadcast values") parser.add_argument('-i','--input',dest='i', help="the path to the input qctools.xml.gz file") parser.add_argument('-t','--tagname',dest='t', help="the tag name you want to test, e.g. SATMAX") @@ -672,18 +630,14 @@ def main(): parser.add_argument('-be','--barsEvaluation',dest='be',action ='store_true',default=False, help="turns Color Bar Evaluation on and off") parser.add_argument('-pr','--print',dest='pr',action='store_true',default=False, help="print over/under frame data to console window") parser.add_argument('-q','--quiet',dest='q',action='store_true',default=False, help="hide ffmpeg output from console window") - args = parser.parse_args() - - ## Validate required arguments - if not args.i: - parser.error("the following arguments are required: -i/--input [path to QCTools report]") - if args.o and args.u: - parser.error("Both the -o and -u options were used. Cannot set threshold for both over and under, only one at a time.") + return parser + +def parse_qc_tools_report(args): ##### Initialize variables and buffers ###### startObj = args.i.replace("\\","/") extension = os.path.splitext(startObj)[1] - # If qctools report is in an MKV attachment, extract .qctools.xml.gz report + # If qctools report is in an MKV attachment, extract .qctools.xml.gz report if extension.lower().endswith('mkv'): startObj = extract_report_mkv(startObj) buffSize = int(args.buff) # cast the input buffer as an integer @@ -693,7 +647,7 @@ def main(): overcount = 0 # init count of overs undercount = 0 # init count of unders count = 0 # init total frames counter - framesList = collections.deque(maxlen=buffSize) # init framesList + framesList = collections.deque(maxlen=buffSize) # init framesList thumbDelay = int(args.ted) # get a seconds number for the delay in the original file btw exporting tags parentDir = os.path.dirname(startObj) baseName = os.path.basename(startObj) @@ -702,7 +656,7 @@ def main(): durationEnd = args.de # we gotta find out if the qctools report has pkt_dts_time or pkt_pts_time ugh - with gzip.open(startObj) as xml: + with gzip.open(startObj) as xml: for event, elem in etree.iterparse(xml, events=('end',), tag='frame'): # iterparse the xml doc if elem.attrib['media_type'] == "video": # get just the video frames # we gotta find out if the qctools report has pkt_dts_time or pkt_pts_time ugh @@ -712,10 +666,10 @@ def main(): break ###### Initialize values from the Config Parser - # Determine if video values are 10 bit depth + # Determine if video values are 10 bit depth bit_depth_10 = detectBitdepth(startObj,pkt,framesList,buffSize) # init a dictionary where we'll store reference values from our config file - profile = {} + profile = {} # init a list of every tag available in a QCTools Report tagList = ["YMIN","YLOW","YAVG","YHIGH","YMAX","UMIN","ULOW","UAVG","UHIGH","UMAX","VMIN","VLOW","VAVG","VHIGH","VMAX","SATMIN","SATLOW","SATAVG","SATHIGH","SATMAX","HUEMED","HUEAVG","YDIF","UDIF","VDIF","TOUT","VREP","BRNG","mse_y","mse_u","mse_v","mse_avg","psnr_y","psnr_u","psnr_v","psnr_avg"] @@ -727,13 +681,13 @@ def main(): durationStart = float(args.ds) # The duration at which we start analyzing the file if no bar detection is selected elif not args.de == 99999999: durationEnd = float(args.de) # The duration at which we stop analyzing the file if no bar detection is selected - - - # set the path for the thumbnail export + + + # set the path for the thumbnail export if args.tep and not args.te: print("Buddy, you specified a thumbnail export path without specifying that you wanted to export the thumbnails. Please either add '-te' to your cli call or delete '-tep [path]'") exit() - + if args.tep: # if user supplied thumbExportPath, use that thumbPath = str(args.tep) else: @@ -744,12 +698,12 @@ def main(): thumbPath = os.path.join(parentDir, str(args.t) + "." + str(args.u)) else: # if they're using a profile, put all thumbs in 1 dir thumbPath = os.path.join(parentDir, "ThumbExports") - + if args.te: # make the thumb export path if it doesn't already exist if not os.path.exists(thumbPath): os.makedirs(thumbPath) - - + + ######## Iterate Through the XML for Bars detection ######## if args.bd: print(f"\nStarting Bars Detection on {baseName}\n") @@ -770,7 +724,7 @@ def main(): else: durationStart = "" durationEnd = "" - + if args.p is not None: # create list of profiles list_of_templates = args.p @@ -791,7 +745,7 @@ def main(): if not config.has_section(template): print(f"Profile '{template}' does not match any section in the config.") continue # Skip to the next template if section doesn't exist - for t in tagList: # loop thru every tag available and + for t in tagList: # loop thru every tag available and try: # see if it's in the config section profile[t.replace("_",".")] = config.get(template,t) # if it is, replace _ necessary for config file with . which xml attributes use, assign the value in config except: # if no config tag exists, do nothing so we can move faster @@ -801,8 +755,8 @@ def main(): print(f"\nStarting Analysis on {baseName} using assigned profile {template}\n") kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag=False) printresults(kbeyond,frameCount,overallFrameFail) - - if args.t and args.o or args.u: + + if args.t and args.o or args.u: profile = {} tag = args.t if args.o: @@ -813,11 +767,63 @@ def main(): print(f"\nStarting Analysis on {baseName} using user specified tag {tag} w/ threshold {over}\n") kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag = True) printresults(kbeyond,frameCount,overallFrameFail) - + print(f"\nFinished Processing File: {baseName}.qctools.xml.gz\n") - + return +def main(): + """ + Main function that parses QCTools XML files, applies analysis, and optionally exports thumbnails. + + This function handles command-line arguments to process a QCTools report, extract frame data from the XML, + apply threshold analysis for broadcast values, optionally detect color bars, and export analysis results to + the console or thumbnails. + + Command-line Arguments: + -i, --input (str): Path to the input QCTools XML.gz file. + -t, --tagname (str): Tag name to analyze, e.g., SATMAX. + -o, --over (float): Overage threshold for the tag specified. + -u, --under (float): Under threshold for the tag specified. + -p, --profile (str): Profile or template name from the qct-parse_config.txt file, e.g., 'default'. + -buff, --buffSize (int): Circular buffer size. Defaults to 11, ensures odd number. + -te, --thumbExport: Export thumbnails if flag is set. + -ted, --thumbExportDelay (int): Minimum number of frames between exported thumbnails. + -tep, --thumbExportPath (str): Path to export thumbnails, defaults to input basename if not provided. + -ds, --durationStart (float): Start time in seconds for analysis. + -de, --durationEnd (float): End time in seconds for analysis. + -bd, --barsDetection: Flag to enable color bars detection. + -pr, --print: Flag to print frame data to the console. + -q, --quiet: Hide ffmpeg output if flag is set. + + Workflow: + 1. Parse command-line arguments. + 2. Optionally load reference threshold values from a profile in `qct-parse_config.txt`. + 3. Initialize buffers, frame counters, and paths for thumbnail export. + 4. Check for `pkt_dts_time` or `pkt_pts_time` in the QCTools XML file. + 5. Set the analysis duration start and end times. + 6. Perform bars detection if enabled, otherwise proceed with general analysis. + 7. Call the `analyzeIt` function to perform frame-by-frame analysis and calculate exceedances. + 8. Print results using `printresults` if applicable. + 9. Handle errors or invalid input (e.g., missing thumbnail export flag but specifying a path). + + Example usage: + python qct-parse.py -i sample.qctools.xml.gz -t SATMAX -o 5.0 -u -5.0 -te + + Returns: + None: The function processes the XML file, performs analysis, and optionally exports thumbnails and prints results to the console. + """ + #### init the stuff from the cli ######## + parser = get_arg_parser() + args = parser.parse_args() + ## Validate required arguments + if not args.i: + parser.error("the following arguments are required: -i/--input [path to QCTools report]") + if args.o and args.u: + parser.error("Both the -o and -u options were used. Cannot set threshold for both over and under, only one at a time.") + parse_qc_tools_report(args) + + if __name__ == '__main__': dependencies() main() \ No newline at end of file From b610d5a963a3862318f25e0b09bb5ebd60f65417 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Tue, 3 Dec 2024 15:36:40 -0700 Subject: [PATCH 38/42] updates to README to reflect today's changes --- README.md | 60 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 7f9eb25..1d7fd43 100644 --- a/README.md +++ b/README.md @@ -8,20 +8,24 @@ This repository contains scripts for automating analysis of QCTools reports. ### Install from source: * Create a new Python Virtual Environment for qct_parse - * Unix based (Mac or Linux): - `python3 -m venv venv` - * Windows: - `py -m venv venv` + * Unix based (Mac or Linux): + `python3 -m venv name_of_env` + * Windows: + `py -m venv name_of_env` + (where 'name_of_env' is replaced with the name of your virtual environment) * Activate virtual env - * Unix based: - `source ./venv/bin/activate` - * Windows: - `venv\scripts\activate` + * Unix based (Mac or Linux): + `source ./name_of_env/bin/activate` + * Windows: + `name_of_env\scripts\activate` * Install Package + * Navigate to the repo root directory `path/to/qct-parse/` + * Run the command: `python -m pip install .` ### Test Code +If you intend to develop the code for your proposes or contribute to the open source project, a test directory is provided in the repo. * Activate virtual env (see Install from source) * Install pytest `pip install pytest` @@ -29,19 +33,16 @@ This repository contains scripts for automating analysis of QCTools reports. `python -m pytest` -### Scripts: +### Commands: - **`qct-parse`** Finds frames that exceed thresholds for QCTool tag(s). Detect color bars with the `-bd` option. - -- **`makeqctoolsreport`** - Generates a QCTools `.xml.gz` report for a given input video file. --- # `qct-parse` -Run a single tag against a supplied value or multiple tags using a config file (`qct-parse_[#]bit_config.txt`). +Run a single tag against a supplied value or multiple tags using a config file. ## Arguments @@ -82,22 +83,22 @@ Run a single tag against a supplied value or multiple tags using a config file ( ### Run single tag tests ```bash -python qct-parse.py -t SATMAX -o 235 -t YMIN -u 16 -i /path/to/report.mkv.qctools.xml.gz +qct-parse -t SATMAX -o 235 -t YMIN -u 16 -i /path/to/report.mkv.qctools.xml.gz ``` ### Run bars detection using default QCTools profile ```bash -python qct-parse.py -bd -p default -i /path/to/report.mkv.qctools.mkv +qct-parse -bd -p default -i /path/to/report.mkv.qctools.mkv ``` ### Export thumbnails of frames beyond threshold ```bash -python qct-parse.py -p default -te -tep /path/to/export/folder -i /path/to/report.mkv.qctools.xml.gz +qct-parse -p default -te -tep /path/to/export/folder -i /path/to/report.mkv.qctools.xml.gz ``` ### Use peak values from detected color bars as thresholds ```bash -python qct-parse.py -bd -be -i /path/to/report.mkv.qctools.xml.gz +qct-parse -bd -be -i /path/to/report.mkv.qctools.xml.gz ``` ## Input files @@ -112,6 +113,8 @@ If the qctools.xml.gz report is in an MKV attachment, the qctools.xml.gz report Both 8-bit and 10-bit values are supported. The bit depth will be detected automatically, and does not need to be specified. +## Config Files + If you wish to edit the profiles stored in the config.txt files, please note that there is a separate config.txt for 8-bit and 10-bit values. In order to export thumbnails, the QCTools report must be in the same directory as the video file it is describing, and must have the same file name as the report (excluding the `qctools.xml.gz`). @@ -126,13 +129,26 @@ Log files contain every instance of values over the specified threshold. For exa --- -# `makeqctoolsreport.py` +### Legacy Commands: + +Not in active development. Please file an issue if you are interested in using these. + +- **`makeqctoolsreport`** A Python port of Morgan’s [makeqctoolsreport.as](https://github.com/iamdamosuzuki/QCToolsReport), this script generates QCTools `.xml.gz` reports from input video files. ## Example Usage ```bash -python makeqctoolsreport.py /path/to/input.mxf +makeqctoolsreport /path/to/input.mxf +``` + +- **`overcatch`** + +A script from the original qct-parse development for running a report against multiple profiles. + +## Example Usage +```bash +overcatch /path/to/input.mxf ``` --- @@ -143,10 +159,7 @@ Ensure Python 3.x.x is installed. Requires FFmpeg. -Additionally, install the `lxml` library: -```bash -pip install lxml -``` +This tool uses the `lxml` python module which is automatically installed with the qct-parse package. For more information on `lxml` usage, check out the [lxml documentation](http://lxml.de/). @@ -161,3 +174,4 @@ For more information on `lxml` usage, check out the [lxml documentation](http:// ## Maintainer - [@av_morgan](https://github.com/av_morgan) +- [@eddycolloton](https://github.com/eddycolloton) From a45f9c8089329418acd5967dd51387cf7a1fe8a9 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Tue, 3 Dec 2024 17:31:09 -0700 Subject: [PATCH 39/42] more README updates --- README.md | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 1d7fd43..8d0f530 100644 --- a/README.md +++ b/README.md @@ -35,14 +35,14 @@ If you intend to develop the code for your proposes or contribute to the open so ### Commands: -- **`qct-parse`** - Finds frames that exceed thresholds for QCTool tag(s). Detect color bars with the `-bd` option. +- **`qct-parse -i/--input [path to QCTools report] [optional arguments]`** + Finds frames that exceed thresholds for QCTool tag(s). Full list of command line arguments below. --- # `qct-parse` -Run a single tag against a supplied value or multiple tags using a config file. +Run a single tag against a supplied value or multiple tags using a profile. ## Arguments @@ -115,16 +115,43 @@ Both 8-bit and 10-bit values are supported. The bit depth will be detected autom ## Config Files -If you wish to edit the profiles stored in the config.txt files, please note that there is a separate config.txt for 8-bit and 10-bit values. +The provided profiles are: +* default +* highTolerance +* midTolerance +* lowTolerance + +Each of these profiles contain the following tags with a corresponding threshold: +`YLOW, YMAX, UMIN, UMAX, VMIN, VMAX, SATMAX, TOUT, VREP` + +The profiles are stored in the config.txt files. Please note that there is a separate config.txt for 8-bit and 10-bit values. + +The process for providing user supplied profiles is in development. +Currently, if you wish to create your own profile, you will need to create your own config directory and `config.txt` file. +There is a environmental variable at the top of qct-parse.py which can be used to reset the config directory: +```bash +CONFIG_ENVIRONMENT_VARIABLE_NAME = 'QCT_PARSE_CONFIG_DIRECTORY' +``` +Simply place the full path to the user created config *directory* in place of 'QCT_PARSE_CONFIG_DIRECTORY' + +## Thumbnails + +Thumbnails of failed frames will be exported if the `-te` flag is invoked. In order to export thumbnails, the QCTools report must be in the same directory as the video file it is describing, and must have the same file name as the report (excluding the `qctools.xml.gz`). +If you would like to provide a path for exporting thumbnails, you can do so using the `-tep` flag. +Otherwise, thumbnails will automatically be created in the same directory as the video file and QCTools report, in a new directory. + +When running qct-parse with a profile, the thumbnails will be placed in a directory named `ThumbExports`. +When run against single tags the directory will be named [TAG NAME].[THRESHOLD] + ## Logging -A log file is created with the same name as the input file but with a '.log' extension. +A log file is created with the same name as the input file but with a '.log' extension. For example: `some_video_file.mkv.qctools.xml.gz.log` -Log files contain every instance of values over the specified threshold. For example: +Log files contain every instance of values over the specified threshold. For example: `2024-10-03 17:02:35,737 SATMAX is over 181.02 with a value of 698.0 at duration 00:00:16.4500` --- @@ -133,20 +160,20 @@ Log files contain every instance of values over the specified threshold. For exa Not in active development. Please file an issue if you are interested in using these. -- **`makeqctoolsreport`** +#### `makeqctoolsreport` A Python port of Morgan’s [makeqctoolsreport.as](https://github.com/iamdamosuzuki/QCToolsReport), this script generates QCTools `.xml.gz` reports from input video files. -## Example Usage +Example Usage: ```bash makeqctoolsreport /path/to/input.mxf ``` -- **`overcatch`** +#### `overcatch` A script from the original qct-parse development for running a report against multiple profiles. -## Example Usage +Example Usage: ```bash overcatch /path/to/input.mxf ``` From c568e84ee8b46bf0b38f0fe3292fd60a805dce54 Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Tue, 3 Dec 2024 20:42:02 -0600 Subject: [PATCH 40/42] qct-parse can have multiple inputs --- qct_parse/qct_parse.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/qct_parse/qct_parse.py b/qct_parse/qct_parse.py index 6de8ea6..7d8058f 100644 --- a/qct_parse/qct_parse.py +++ b/qct_parse/qct_parse.py @@ -92,7 +92,21 @@ def initLog(inputPath): logPath = inputPath + '.log' logging.basicConfig(filename=logPath,level=logging.INFO,format='%(asctime)s %(message)s') logging.info("Started QCT-Parse") - + +def set_logger(input_path): + log_path = f'{input_path}.log' + logger = logging.getLogger() + if logger.hasHandlers(): + logger.handlers.clear() + logger.setLevel(logging.INFO) + + file_handler = logging.FileHandler(filename=log_path) + file_handler.setLevel(logging.INFO) + file_handler.setFormatter(logging.Formatter('%(asctime)s %(message)s')) + logger.addHandler(file_handler) + logger.info("Started QCT-Parse") + + # finds stuff over/under threshold def threshFinder(inFrame,args,startObj,pkt,tag,over,thumbPath,thumbDelay,adhoc_tag): @@ -615,7 +629,7 @@ def color_percentage(value): def get_arg_parser(): parser = argparse.ArgumentParser(description="parses QCTools XML files for frames beyond broadcast values") - parser.add_argument('-i','--input',dest='i', help="the path to the input qctools.xml.gz file") + parser.add_argument('-i','--input',dest='i', action='append', help="the path to the input qctools.xml.gz file") parser.add_argument('-t','--tagname',dest='t', help="the tag name you want to test, e.g. SATMAX") parser.add_argument('-o','--over',dest='o', help="the threshold overage number") parser.add_argument('-u','--under',dest='u', help="the threshold under number") @@ -631,11 +645,10 @@ def get_arg_parser(): parser.add_argument('-pr','--print',dest='pr',action='store_true',default=False, help="print over/under frame data to console window") parser.add_argument('-q','--quiet',dest='q',action='store_true',default=False, help="hide ffmpeg output from console window") return parser - -def parse_qc_tools_report(args): - ##### Initialize variables and buffers ###### - startObj = args.i.replace("\\","/") + +def parse_single_qc_tools_report(input_file, args): + startObj = input_file.replace("\\","/") extension = os.path.splitext(startObj)[1] # If qctools report is in an MKV attachment, extract .qctools.xml.gz report if extension.lower().endswith('mkv'): @@ -643,7 +656,7 @@ def parse_qc_tools_report(args): buffSize = int(args.buff) # cast the input buffer as an integer if buffSize%2 == 0: buffSize = buffSize + 1 - initLog(startObj) # initialize the log + set_logger(startObj) overcount = 0 # init count of overs undercount = 0 # init count of unders count = 0 # init total frames counter @@ -770,7 +783,10 @@ def parse_qc_tools_report(args): print(f"\nFinished Processing File: {baseName}.qctools.xml.gz\n") - return +def parse_qc_tools_report(args): + ##### Initialize variables and buffers ###### + for input_file in args.i: + parse_single_qc_tools_report(input_file, args) def main(): """ From 17393071c2ffae76a6412e24b26e05188fd43eed Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Wed, 4 Dec 2024 13:26:58 -0600 Subject: [PATCH 41/42] Support creation of Sphinx documentation --- README.md | 7 +++++++ docs/Makefile | 20 ++++++++++++++++++++ docs/conf.py | 31 +++++++++++++++++++++++++++++++ docs/development.md | 29 +++++++++++++++++++++++++++++ docs/index.rst | 14 ++++++++++++++ docs/make.bat | 35 +++++++++++++++++++++++++++++++++++ docs/readme.rst | 2 ++ 7 files changed, 138 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/development.md create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 docs/readme.rst diff --git a/README.md b/README.md index 8d0f530..88930a5 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,13 @@ If you intend to develop the code for your proposes or contribute to the open so * Run tests `python -m pytest` +### Building the documentation +* Activate virtual env (see Install from source) +* Install sphinx and myst-parser + `pip install sphinx myst-parser` +* Run sphinx-build command + `sphinx-build ./docs ./dist/docs` + ### Commands: diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..58ed402 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,31 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'qct-parse' +copyright = '2024, AMIA Open-Source' +author = 'AMIA Open-Source' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ['myst_parser'] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +source_suffix = { + '.rst': 'restructuredtext', + '.txt': 'markdown', + '.md': 'markdown', +} + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'alabaster' +html_static_path = ['_static'] diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..227caeb --- /dev/null +++ b/docs/development.md @@ -0,0 +1,29 @@ +# Development Information + +## Configure Development Environment + +* Create a new Python Virtual Environment for qct_parse + * Unix based (Mac or Linux): + `python3 -m venv name_of_env` + * Windows: + `py -m venv name_of_env` + (where 'name_of_env' is replaced with the name of your virtual environment) +* Activate virtual env + * Unix based (Mac or Linux): + `source ./name_of_env/bin/activate` + * Windows: + `name_of_env\scripts\activate` +* Install Package as editable package + * Navigate to the repo root directory `path/to/qct-parse/` + * Run the command: + `python -m pip install -e .` + +## Run Tests + +If you intend to develop the code for your proposes or contribute to the open source project, a test directory is provided in the repo. +* Activate virtual env (see Configure Development Environment) +* Install pytest + `pip install pytest` +* Run tests + `python -m pytest` + diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..7ebe75d --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,14 @@ +.. qct-parse documentation master file, created by + sphinx-quickstart on Tue Dec 3 20:53:22 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +qct-parse documentation +======================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + readme + development diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..954237b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 0000000..339dc8f --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1,2 @@ +.. include:: ../README.md + :parser: myst_parser.sphinx_ From 8d119244e5b0d36ced7268bce7dc89213626f942 Mon Sep 17 00:00:00 2001 From: eddycollotonn Date: Wed, 4 Dec 2024 13:19:18 -0700 Subject: [PATCH 42/42] added csv output option w/ -csv arg --- qct_parse/qct_parse.py | 84 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/qct_parse/qct_parse.py b/qct_parse/qct_parse.py index 7d8058f..9cbd4bd 100644 --- a/qct_parse/qct_parse.py +++ b/qct_parse/qct_parse.py @@ -19,6 +19,7 @@ import time import json import shutil # dependency checking +import csv CONFIG_ENVIRONMENT_VARIABLE_NAME = 'QCT_PARSE_CONFIG_DIRECTORY' @@ -627,6 +628,65 @@ def color_percentage(value): print(f"Frames With At Least One Fail:\t{overallFrameFail}\t{color}{percent_overall_string}{RESET}\t% of the total # of frames\n") print(f"{BOLD}**************************{RESET}\n") + +def print_results_to_csv(kbeyond, frameCount, overallFrameFail, startObj, output_file): + """ + Writes the analysis results of frame data to a CSV file. + + Args: + kbeyond (dict): A dictionary where keys are tag names and values are the counts of frames + that exceed the threshold for each tag. + frameCount (int): The total number of frames analyzed. + overallFrameFail (int): The number of frames where at least one tag exceeds its threshold. + output_file (str): The name of the CSV file to save the results. + + Outputs: + A CSV file with the following structure: + - TotalFrames: [frameCount] + - By Tag: [Tag, Count, Percentage] + - Overall: Frames With At Least One Fail, Count, Percentage + + Notes: + - Percentages are formatted as whole numbers (e.g., "100"), two decimal places + (e.g., "12.34"), or "<0.01" for values less than 0.01%. + """ + + def format_percentage(value): + percent = value * 100 + if percent == 100: + return "100" + elif percent == 0: + return "0" + elif percent < 0.01: + return "<0.01" + else: + return f"{percent:.2f}" + + # Write results to CSV + with open(output_file, mode='w', newline='', encoding='utf-8') as file: + writer = csv.writer(file) + + # Title row + writer.writerow(["qct-parse summary report for file:", startObj]) + + # Total Frames + writer.writerow(["TotalFrames", frameCount]) + + # By Tag + writer.writerow(["By Tag"]) + writer.writerow(["Tag", "Count", "Percentage of Total Frames"]) + for tag, count in kbeyond.items(): + percent = count / frameCount + writer.writerow([tag, count, format_percentage(percent)]) + + # Overall + writer.writerow([]) + writer.writerow(["Overall"]) + overall_percent = overallFrameFail / frameCount + writer.writerow(["Frames With At Least One Fail", overallFrameFail, format_percentage(overall_percent)]) + + print(f"Results successfully written to {output_file}") + def get_arg_parser(): parser = argparse.ArgumentParser(description="parses QCTools XML files for frames beyond broadcast values") parser.add_argument('-i','--input',dest='i', action='append', help="the path to the input qctools.xml.gz file") @@ -643,6 +703,7 @@ def get_arg_parser(): parser.add_argument('-bd','--barsDetection',dest='bd',action ='store_true',default=False, help="turns Bar Detection on and off") parser.add_argument('-be','--barsEvaluation',dest='be',action ='store_true',default=False, help="turns Color Bar Evaluation on and off") parser.add_argument('-pr','--print',dest='pr',action='store_true',default=False, help="print over/under frame data to console window") + parser.add_argument('-csv', '--csvreport',dest='csv',action ='store_true',default=False, help="print summary results to a csv sidecar file") parser.add_argument('-q','--quiet',dest='q',action='store_true',default=False, help="hide ffmpeg output from console window") return parser @@ -667,6 +728,11 @@ def parse_single_qc_tools_report(input_file, args): baseName = baseName.replace(".qctools.xml.gz", "") durationStart = args.ds durationEnd = args.de + # set the path for the csv report + if args.csv: + result_csv_file = os.path.join(parentDir, str(baseName) + ".qct_summary_report.csv") + else: + result_csv_file = None # we gotta find out if the qctools report has pkt_dts_time or pkt_pts_time ugh with gzip.open(startObj) as xml: @@ -733,7 +799,11 @@ def parse_single_qc_tools_report(input_file, args): durationEnd = 99999999 profile = maxBarsDict kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag=False) - printresults(kbeyond,frameCount,overallFrameFail) + if args.csv: + print_results_to_csv(kbeyond, frameCount, overallFrameFail, startObj, result_csv_file) + printresults(kbeyond,frameCount,overallFrameFail) + else: + printresults(kbeyond,frameCount,overallFrameFail) else: durationStart = "" durationEnd = "" @@ -767,7 +837,11 @@ def parse_single_qc_tools_report(input_file, args): ######## Iterate Through the XML for General Analysis ######## print(f"\nStarting Analysis on {baseName} using assigned profile {template}\n") kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag=False) - printresults(kbeyond,frameCount,overallFrameFail) + if args.csv: + print_results_to_csv(kbeyond, frameCount, overallFrameFail, startObj, result_csv_file) + printresults(kbeyond,frameCount,overallFrameFail) + else: + printresults(kbeyond,frameCount,overallFrameFail) if args.t and args.o or args.u: profile = {} @@ -779,7 +853,11 @@ def parse_single_qc_tools_report(input_file, args): profile[tag] = over print(f"\nStarting Analysis on {baseName} using user specified tag {tag} w/ threshold {over}\n") kbeyond, frameCount, overallFrameFail = analyzeIt(args,profile,startObj,pkt,durationStart,durationEnd,thumbPath,thumbDelay,framesList,adhoc_tag = True) - printresults(kbeyond,frameCount,overallFrameFail) + if args.csv: + print_results_to_csv(kbeyond, frameCount, overallFrameFail, startObj, result_csv_file) + printresults(kbeyond,frameCount,overallFrameFail) + else: + printresults(kbeyond,frameCount,overallFrameFail) print(f"\nFinished Processing File: {baseName}.qctools.xml.gz\n")