From cc596f68b9d0f102451c73fe9a46eef2a84a762a Mon Sep 17 00:00:00 2001 From: Jason Patton Date: Mon, 13 Oct 2025 15:16:35 -0500 Subject: [PATCH] INF-3316 Fix htcondor import statements and remove unused imports Attempt to use htcondor2 when available, fallback to htcondor, and failing that, raise an error. --- accounting/aggregations/create_report.py | 14 +++----------- accounting/aggregations/functions.py | 12 +++++++++--- accounting/aggregations/missing_ospool_projects.py | 1 - accounting/aggregations/query.py | 11 +++-------- accounting/filters/BaseFilter.py | 1 - accounting/filters/ChtcScheddCpuFilter.py | 1 - accounting/filters/ChtcScheddCpuMonthlyFilter.py | 3 --- accounting/filters/ChtcScheddCpuOspoolFilter.py | 12 +++++++++++- .../filters/ChtcScheddCpuOspoolMonthlyFilter.py | 13 ++++++++++--- accounting/filters/ChtcScheddCpuRemovedFilter.py | 2 -- accounting/filters/ChtcScheddDSIGpuFilter.py | 3 --- accounting/filters/ChtcScheddGpuFilter.py | 2 -- accounting/filters/ChtcScheddJobDistroFilter.py | 5 ----- accounting/filters/IgwnScheddCpuFilter.py | 2 -- accounting/filters/IgwnScheddCpuMonthlyFilter.py | 1 - accounting/filters/OsgScheddCpuFilter.py | 13 +++++++++++-- accounting/filters/OsgScheddCpuHeldFilter.py | 12 ++++++++++-- accounting/filters/OsgScheddCpuMonthlyFilter.py | 1 - accounting/filters/OsgScheddCpuRemovedFilter.py | 12 +++++++++++- accounting/filters/OsgScheddCpuRetryFilter.py | 12 +++++++++++- accounting/filters/OsgScheddGpuFilter.py | 12 +++++++++++- accounting/filters/OsgScheddJobDistroFilter.py | 13 +++++++++++-- accounting/filters/OsgScheddLongJobFilter.py | 12 +++++++++++- accounting/filters/PathScheddCpuFilter.py | 1 - accounting/formatters/BaseFormatter.py | 4 ++-- .../formatters/ChtcScheddJobDistroFormatter.py | 4 +--- accounting/pull_hold_reasons.py | 1 - cache_collector_hosts.py | 13 ++++++++++++- current/ospool_gpu_job_shapes.py | 14 ++++++++++++-- 29 files changed, 139 insertions(+), 68 deletions(-) diff --git a/accounting/aggregations/create_report.py b/accounting/aggregations/create_report.py index de9d2612..a06e8d23 100644 --- a/accounting/aggregations/create_report.py +++ b/accounting/aggregations/create_report.py @@ -2,23 +2,15 @@ # parses cmd line args and handles outputs import elasticsearch -from elasticsearch_dsl import Search, Q, A, connections import argparse import sys import json -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path -from pprint import pprint from query import run_query -from functions import send_email -from report_helpers import ( - Aggregation, - add_runtime_script, - get_percent_bucket_script, - table, print_error, - generate_csv -) +from functions import send_email +from report_helpers import table, generate_csv OUTPUT_ARGS = { "--print-table" : {"action" : "store_true", "help" : "prints a CLI table, NOTE: pipe into 'less -S'"}, diff --git a/accounting/aggregations/functions.py b/accounting/aggregations/functions.py index 8799a358..1a0b11e6 100644 --- a/accounting/aggregations/functions.py +++ b/accounting/aggregations/functions.py @@ -15,9 +15,15 @@ from pathlib import Path try: - import htcondor -except ModuleNotFoundError: - htcondor = None + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + htcondor = None + from dns.resolver import query as dns_query diff --git a/accounting/aggregations/missing_ospool_projects.py b/accounting/aggregations/missing_ospool_projects.py index f8d7723a..0e820e4a 100644 --- a/accounting/aggregations/missing_ospool_projects.py +++ b/accounting/aggregations/missing_ospool_projects.py @@ -1,5 +1,4 @@ import sys -import time import json import argparse import importlib diff --git a/accounting/aggregations/query.py b/accounting/aggregations/query.py index c3b70472..86863364 100755 --- a/accounting/aggregations/query.py +++ b/accounting/aggregations/query.py @@ -1,15 +1,10 @@ import elasticsearch import argparse -import sys -import json -from functions import send_email -from elasticsearch_dsl import Search, Q, A, connections, response -from datetime import datetime, timedelta -from collections import namedtuple +from elasticsearch_dsl import Search, Q, A, response +from datetime import datetime from operator import itemgetter -from pprint import pprint -from report_helpers import Aggregation, add_runtime_script, get_percent_bucket_script, table, print_error +from report_helpers import Aggregation, add_runtime_script, get_percent_bucket_script # lists to hold aggregation objects ROWS_AGGS = [] diff --git a/accounting/filters/BaseFilter.py b/accounting/filters/BaseFilter.py index 386dc308..5b409562 100644 --- a/accounting/filters/BaseFilter.py +++ b/accounting/filters/BaseFilter.py @@ -1,5 +1,4 @@ import logging -import statistics as stats from collections import defaultdict from functools import partial from operator import itemgetter diff --git a/accounting/filters/ChtcScheddCpuFilter.py b/accounting/filters/ChtcScheddCpuFilter.py index 68454495..ee9903ba 100644 --- a/accounting/filters/ChtcScheddCpuFilter.py +++ b/accounting/filters/ChtcScheddCpuFilter.py @@ -1,5 +1,4 @@ import statistics as stats -from pathlib import Path from ast import literal_eval from .BaseFilter import BaseFilter from accounting.functions import get_job_units diff --git a/accounting/filters/ChtcScheddCpuMonthlyFilter.py b/accounting/filters/ChtcScheddCpuMonthlyFilter.py index 2774598d..36d7c4bd 100644 --- a/accounting/filters/ChtcScheddCpuMonthlyFilter.py +++ b/accounting/filters/ChtcScheddCpuMonthlyFilter.py @@ -1,9 +1,6 @@ - -import statistics as stats from collections import defaultdict from operator import itemgetter from ast import literal_eval -import elasticsearch.helpers from .BaseFilter import BaseFilter from accounting.functions import get_job_units diff --git a/accounting/filters/ChtcScheddCpuOspoolFilter.py b/accounting/filters/ChtcScheddCpuOspoolFilter.py index e0fd7edc..09b9530e 100644 --- a/accounting/filters/ChtcScheddCpuOspoolFilter.py +++ b/accounting/filters/ChtcScheddCpuOspoolFilter.py @@ -1,6 +1,6 @@ import re +import sys import pickle -import htcondor import statistics as stats from datetime import date from pathlib import Path @@ -8,6 +8,16 @@ from .BaseFilter import BaseFilter from accounting.functions import get_job_units +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 10: "Num Uniq Job Ids", diff --git a/accounting/filters/ChtcScheddCpuOspoolMonthlyFilter.py b/accounting/filters/ChtcScheddCpuOspoolMonthlyFilter.py index 092eec49..94c881b9 100644 --- a/accounting/filters/ChtcScheddCpuOspoolMonthlyFilter.py +++ b/accounting/filters/ChtcScheddCpuOspoolMonthlyFilter.py @@ -1,16 +1,23 @@ import re +import sys import pickle -import htcondor -import statistics as stats from collections import defaultdict from operator import itemgetter from ast import literal_eval -import elasticsearch.helpers from functools import lru_cache from pathlib import Path from .BaseFilter import BaseFilter from accounting.functions import get_job_units +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + MAX_INT = 2**62 DEFAULT_COLUMNS = { diff --git a/accounting/filters/ChtcScheddCpuRemovedFilter.py b/accounting/filters/ChtcScheddCpuRemovedFilter.py index 4ee0a44e..9f4702d6 100644 --- a/accounting/filters/ChtcScheddCpuRemovedFilter.py +++ b/accounting/filters/ChtcScheddCpuRemovedFilter.py @@ -1,6 +1,4 @@ -import htcondor import statistics as stats -from pathlib import Path from .BaseFilter import BaseFilter diff --git a/accounting/filters/ChtcScheddDSIGpuFilter.py b/accounting/filters/ChtcScheddDSIGpuFilter.py index cbb655ce..b48b8ad6 100644 --- a/accounting/filters/ChtcScheddDSIGpuFilter.py +++ b/accounting/filters/ChtcScheddDSIGpuFilter.py @@ -1,6 +1,3 @@ - -import statistics as stats -from pathlib import Path from .BaseFilter import BaseFilter diff --git a/accounting/filters/ChtcScheddGpuFilter.py b/accounting/filters/ChtcScheddGpuFilter.py index 274f90a3..0d014a38 100644 --- a/accounting/filters/ChtcScheddGpuFilter.py +++ b/accounting/filters/ChtcScheddGpuFilter.py @@ -1,6 +1,4 @@ - import statistics as stats -from pathlib import Path from ast import literal_eval from .BaseFilter import BaseFilter diff --git a/accounting/filters/ChtcScheddJobDistroFilter.py b/accounting/filters/ChtcScheddJobDistroFilter.py index 23740b73..ccec136f 100644 --- a/accounting/filters/ChtcScheddJobDistroFilter.py +++ b/accounting/filters/ChtcScheddJobDistroFilter.py @@ -1,8 +1,3 @@ - -import htcondor -import pickle -from pathlib import Path -from elasticsearch import Elasticsearch import elasticsearch.helpers from .BaseFilter import BaseFilter from functools import lru_cache diff --git a/accounting/filters/IgwnScheddCpuFilter.py b/accounting/filters/IgwnScheddCpuFilter.py index f3a1be07..ccf0f39b 100644 --- a/accounting/filters/IgwnScheddCpuFilter.py +++ b/accounting/filters/IgwnScheddCpuFilter.py @@ -1,6 +1,4 @@ import statistics as stats -from pathlib import Path -from ast import literal_eval from .BaseFilter import BaseFilter from accounting.functions import get_job_units diff --git a/accounting/filters/IgwnScheddCpuMonthlyFilter.py b/accounting/filters/IgwnScheddCpuMonthlyFilter.py index 6c190075..5619425e 100644 --- a/accounting/filters/IgwnScheddCpuMonthlyFilter.py +++ b/accounting/filters/IgwnScheddCpuMonthlyFilter.py @@ -1,4 +1,3 @@ -from pathlib import Path from collections import defaultdict from operator import itemgetter from .BaseFilter import BaseFilter diff --git a/accounting/filters/OsgScheddCpuFilter.py b/accounting/filters/OsgScheddCpuFilter.py index a7116178..1b12284f 100644 --- a/accounting/filters/OsgScheddCpuFilter.py +++ b/accounting/filters/OsgScheddCpuFilter.py @@ -1,6 +1,5 @@ - import re -import htcondor +import sys import pickle import statistics as stats from datetime import date @@ -8,6 +7,16 @@ from .BaseFilter import BaseFilter from accounting.functions import get_job_units, get_topology_project_data, get_topology_resource_data, get_institution_database +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 5: "% Shadw w/o Start", diff --git a/accounting/filters/OsgScheddCpuHeldFilter.py b/accounting/filters/OsgScheddCpuHeldFilter.py index b634d2ac..ae0305ca 100644 --- a/accounting/filters/OsgScheddCpuHeldFilter.py +++ b/accounting/filters/OsgScheddCpuHeldFilter.py @@ -1,11 +1,19 @@ - import re -import htcondor +import sys import pickle import statistics as stats from pathlib import Path from .BaseFilter import BaseFilter +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise HOLD_REASONS = [ "Unspecified", diff --git a/accounting/filters/OsgScheddCpuMonthlyFilter.py b/accounting/filters/OsgScheddCpuMonthlyFilter.py index 8ba77183..97ec32ef 100644 --- a/accounting/filters/OsgScheddCpuMonthlyFilter.py +++ b/accounting/filters/OsgScheddCpuMonthlyFilter.py @@ -1,4 +1,3 @@ - import re import htcondor import pickle diff --git a/accounting/filters/OsgScheddCpuRemovedFilter.py b/accounting/filters/OsgScheddCpuRemovedFilter.py index 96096ccf..2757a8f1 100644 --- a/accounting/filters/OsgScheddCpuRemovedFilter.py +++ b/accounting/filters/OsgScheddCpuRemovedFilter.py @@ -1,11 +1,21 @@ import re -import htcondor +import sys import pickle import statistics as stats from pathlib import Path from .BaseFilter import BaseFilter +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 10: "Num Uniq Job Ids", diff --git a/accounting/filters/OsgScheddCpuRetryFilter.py b/accounting/filters/OsgScheddCpuRetryFilter.py index 7f24f5f2..c0ef45b5 100644 --- a/accounting/filters/OsgScheddCpuRetryFilter.py +++ b/accounting/filters/OsgScheddCpuRetryFilter.py @@ -1,11 +1,21 @@ import re -import htcondor +import sys import pickle from pathlib import Path from .BaseFilter import BaseFilter from accounting.pull_hold_reasons import get_hold_reasons +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 10: "Num Uniq Job Ids", diff --git a/accounting/filters/OsgScheddGpuFilter.py b/accounting/filters/OsgScheddGpuFilter.py index 12468fe5..ef23fc64 100644 --- a/accounting/filters/OsgScheddGpuFilter.py +++ b/accounting/filters/OsgScheddGpuFilter.py @@ -1,6 +1,6 @@ import re -import htcondor +import sys import pickle import statistics as stats from datetime import date @@ -8,6 +8,16 @@ from .BaseFilter import BaseFilter from accounting.functions import get_topology_resource_data, get_institution_database +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 10: "Num Uniq Job Ids", diff --git a/accounting/filters/OsgScheddJobDistroFilter.py b/accounting/filters/OsgScheddJobDistroFilter.py index 8ed7a8e7..0dfb1a22 100644 --- a/accounting/filters/OsgScheddJobDistroFilter.py +++ b/accounting/filters/OsgScheddJobDistroFilter.py @@ -1,14 +1,23 @@ import re -import htcondor +import sys import pickle from pathlib import Path -from elasticsearch import Elasticsearch import elasticsearch.helpers from .BaseFilter import BaseFilter from functools import lru_cache from collections import defaultdict +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + OSG_CONNECT_APS = { "login04.osgconnect.net", diff --git a/accounting/filters/OsgScheddLongJobFilter.py b/accounting/filters/OsgScheddLongJobFilter.py index 92b6b9f8..18ef4bae 100644 --- a/accounting/filters/OsgScheddLongJobFilter.py +++ b/accounting/filters/OsgScheddLongJobFilter.py @@ -1,10 +1,20 @@ import re -import htcondor +import sys import pickle from pathlib import Path from .BaseFilter import BaseFilter +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + DEFAULT_COLUMNS = { 5 : "Project", diff --git a/accounting/filters/PathScheddCpuFilter.py b/accounting/filters/PathScheddCpuFilter.py index 350f8e09..fdcdf0bf 100644 --- a/accounting/filters/PathScheddCpuFilter.py +++ b/accounting/filters/PathScheddCpuFilter.py @@ -1,5 +1,4 @@ import statistics as stats -from pathlib import Path from ast import literal_eval from .BaseFilter import BaseFilter diff --git a/accounting/formatters/BaseFormatter.py b/accounting/formatters/BaseFormatter.py index 1740d521..38a6e6b0 100644 --- a/accounting/formatters/BaseFormatter.py +++ b/accounting/formatters/BaseFormatter.py @@ -1,6 +1,6 @@ import csv -from collections import OrderedDict, defaultdict -from datetime import datetime, timedelta +from collections import OrderedDict +from datetime import datetime from pathlib import Path diff --git a/accounting/formatters/ChtcScheddJobDistroFormatter.py b/accounting/formatters/ChtcScheddJobDistroFormatter.py index a372fbad..36115497 100644 --- a/accounting/formatters/ChtcScheddJobDistroFormatter.py +++ b/accounting/formatters/ChtcScheddJobDistroFormatter.py @@ -1,8 +1,6 @@ import csv -from collections import OrderedDict, defaultdict -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path -from decimal import Decimal def break_chars(s): diff --git a/accounting/pull_hold_reasons.py b/accounting/pull_hold_reasons.py index 94de84e8..86ed93eb 100644 --- a/accounting/pull_hold_reasons.py +++ b/accounting/pull_hold_reasons.py @@ -1,7 +1,6 @@ import pickle import tempfile import os -import time import re from urllib.request import urlopen from pathlib import Path diff --git a/cache_collector_hosts.py b/cache_collector_hosts.py index b44ba27b..da9dc885 100644 --- a/cache_collector_hosts.py +++ b/cache_collector_hosts.py @@ -1,8 +1,19 @@ import re -import htcondor +import sys import pickle from pathlib import Path +try: + import htcondor2 as htcondor +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise + + CUSTOM_MAPPING = { "osg-login2.pace.gatech.edu": {"osg-login2.pace.gatech.edu"}, "ce1.opensciencegrid.org": {"cm-2.ospool.osg-htc.org", "cm-1.ospool.osg-htc.org"}, diff --git a/current/ospool_gpu_job_shapes.py b/current/ospool_gpu_job_shapes.py index bb912d44..4a22c8fc 100644 --- a/current/ospool_gpu_job_shapes.py +++ b/current/ospool_gpu_job_shapes.py @@ -1,3 +1,4 @@ +import sys import math import time import argparse @@ -7,8 +8,17 @@ from typing import List, Tuple, Dict, Set, Union from pprint import pprint -import htcondor -import classad +try: + import htcondor2 as htcondor + import classad2 as classad +except ImportError: + print("Could not import from htcondor2, falling back to htcondor", file=sys.stderr) + try: + import htcondor + import classad + except ImportError: + print("Could not import htcondor", file=sys.stderr) + raise DEFAULT_COLLECTOR = "cm-1.ospool.osg-htc.org"