Skip to content

Import validation improvements #1850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vish-cs merged 1 commit into from
Jan 28, 2026
+132 −305
Merged

Import validation improvements #1850

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 2 additions & 8 deletions scripts/us_fed/treasury_constant_maturity_rates/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,7 @@
"source_files": [
"treasury_constant_maturity_rates.csv"
],
"cron_schedule": "15 3 * * *",
"config_override": {
"ignore_validation_status": false
}
"cron_schedule": "15 3 * * *"
},
{
"import_name": "USFed_ConstantMaturityRates",
Expand All @@ -45,10 +42,7 @@
"source_files": [
"treasury_constant_maturity_rates.csv"
],
"cron_schedule": "15 3 * * *",
"config_override": {
"ignore_validation_status": false
}
"cron_schedule": "15 3 * * *"
}
]
}
36 changes: 0 additions & 36 deletions scripts/us_fed/treasury_constant_maturity_rates/validation_config.json
View file
Open in desktop

This file was deleted.

28 changes: 2 additions & 26 deletions statvar_imports/bis/bis_central_bank_policy_rate/validation_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,35 +18,11 @@
}
},
{
"rule_id": "check_deleted_count",
"description": "Checks that the number of deleted points is within the threshold.",
"validator": "DELETED_COUNT",
"scope": {
"data_source": "differ"
},
"rule_id": "check_deleted_records_count",
"validator": "DELETED_RECORDS_COUNT",
"params": {
"threshold": 100
}
},
{
"rule_id": "check_missing_refs_count",
"validator": "MISSING_REFS_COUNT",
"scope": {
"data_source": "lint"
},
"params": {
"threshold": 0
}
},
{
"rule_id": "check_lint_error_count",
"validator": "LINT_ERROR_COUNT",
"scope": {
"data_source": "lint"
},
"params": {
"threshold": 0
}
}
]
}
26 changes: 1 addition & 25 deletions statvar_imports/school_algebra1/validation_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,36 +1,12 @@
{
"schema_version": "1.0",
"rules": [
{
"rule_id": "check_deleted_count",
"description": "Checks that the number of deleted points is within the threshold.",
"validator": "DELETED_COUNT",
"scope": {
"data_source": "differ"
},
"params": {
"threshold": 0
}
},
{
"rule_id": "check_missing_refs_count",
"validator": "MISSING_REFS_COUNT",
"scope": {
"data_source": "lint"
},
"params": {
"threshold": 457000
}
},
{
"rule_id": "check_lint_error_count",
"validator": "LINT_ERROR_COUNT",
"scope": {
"data_source": "lint"
},
"params": {
"threshold": 0
}
}
]
}
}
30 changes: 3 additions & 27 deletions statvar_imports/us_bls/cpi_category/validation_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,35 +2,11 @@
"schema_version": "1.0",
"rules": [
{
"rule_id": "check_deleted_count",
"description": "Checks that the number of deleted points is within the threshold.",
"validator": "DELETED_COUNT",
"scope": {
"data_source": "differ"
},
"rule_id": "check_deleted_records_count",
"validator": "DELETED_RECORDS_COUNT",
"params": {
"threshold": 1230
}
},
{
"rule_id": "check_missing_refs_count",
"validator": "MISSING_REFS_COUNT",
"scope": {
"data_source": "lint"
},
"params": {
"threshold": 0
}
},
{
"rule_id": "check_lint_error_count",
"validator": "LINT_ERROR_COUNT",
"scope": {
"data_source": "lint"
},
"params": {
"threshold": 0
}
}
]
}
}
16 changes: 4 additions & 12 deletions tools/import_validation/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@ Each object in the `rules` list defines a single validation check with the follo

The `scope` object specifies which data to run the validation on. It has two main parts:

- `data_source`: The key for the data source to use (`"stats"` or `"differ"`).
- `variables`: An optional filter to select a subset of StatVars from the data source.

The `variables` object can contain any of the following keys:
Expand All @@ -82,17 +81,11 @@ Here is an example of a complete configuration file:
{
"rule_id": "check_latest_date_for_all",
"validator": "MAX_DATE_LATEST",
"scope": {
"data_source": "stats"
},
"params": {}
},
{
"rule_id": "check_deleted_points_threshold",
"validator": "DELETED_COUNT",
"scope": {
"data_source": "differ"
},
"validator": "DELETED_RECORDS_COUNT",
"params": {
"threshold": 10
}
Expand All @@ -101,7 +94,6 @@ Here is an example of a complete configuration file:
"rule_id": "check_percent_max_value",
"validator": "MAX_VALUE_CHECK",
"scope": {
"data_source": "stats",
"variables": {
"contains_all": ["Percent"]
}
Expand Down Expand Up @@ -150,9 +142,9 @@ The following validations are currently supported:
| `MAX_DATE_CONSISTENT` | Checks that the latest date is the same for all StatVars. | `stats` | None |
| `MISSING_REFS_COUNT` | Checks that the total number of missing references is within a threshold. | `lint` | `threshold` (integer, defaults to 0) |
| `LINT_ERROR_COUNT` | Checks that the total number of lint errors is within a threshold. | `lint` | `threshold` (integer, defaults to 0) |
| `DELETED_COUNT` | Checks that the total number of deleted points is within a threshold. | `differ` | `threshold` (integer, defaults to 0) |
| `MODIFIED_COUNT` | Checks that the number of modified points is the same for all StatVars. | `differ` | None |
| `ADDED_COUNT` | Checks that the number of added points is the same for all StatVars. | `differ` | None |
| `DELETED_RECORDS_COUNT` | Checks that the total number of deleted points is within a threshold. | `differ` | `threshold` (integer, defaults to 0) |
| `MODIFIED_RECORDS_COUNT` | Checks that the number of modified points is the same for all StatVars. | `differ` | None |
| `ADDED_RECORDS_COUNT` | Checks that the number of added points is the same for all StatVars. | `differ` | None |
| `NUM_PLACES_CONSISTENT` | Checks that the number of places is the same for all StatVars. | `stats` | None |
| `NUM_PLACES_COUNT` | Checks that the number of places is within a defined range. | `stats` | `minimum`, `maximum`, or `value` (integer) |
| `NUM_OBSERVATIONS_CHECK` | Checks that the number of observations is within a defined range. | `stats` | `minimum`, `maximum`, or `value` (integer) |
Expand Down
29 changes: 9 additions & 20 deletions tools/import_validation/import_validation_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,7 @@ def test_successful_run(self):
"rules": [{
"rule_id": "num_places_consistent",
"validator": "NUM_PLACES_CONSISTENT",
"scope": {
"data_source": "stats"
},
"scope": {},
"params": {}
}]
}, f)
Expand Down Expand Up @@ -97,9 +95,7 @@ def test_failed_run(self):
"rules": [{
"rule_id": "num_places_consistent",
"validator": "NUM_PLACES_CONSISTENT",
"scope": {
"data_source": "stats"
},
"scope": {},
"params": {}
}]
}, f)
Expand Down Expand Up @@ -136,9 +132,7 @@ def test_missing_required_file_fails(self):
"rules": [{
"rule_id": "num_places_consistent",
"validator": "NUM_PLACES_CONSISTENT",
"scope": {
"data_source": "stats"
}
"scope": {}
}]
}, f)

Expand Down Expand Up @@ -169,7 +163,6 @@ def test_variables_filtering(self):
"rule_id": "num_places_consistent_filtered",
"validator": "NUM_PLACES_CONSISTENT",
"scope": {
"data_source": "stats",
"variables": {
"dcids": ["sv1", "sv2"]
}
Expand Down Expand Up @@ -254,11 +247,9 @@ def test_empty_differ_file_runs_validation(self):
json.dump(
{
"rules": [{
"rule_id": "check_deleted_count",
"validator": "DELETED_COUNT",
"scope": {
"data_source": "differ"
},
"rule_id": "check_deleted_records_count",
"validator": "DELETED_RECORDS_COUNT",
"scope": {},
"params": {
"threshold": 0
} # Fail if deleted count is > 0
Expand Down Expand Up @@ -299,11 +290,9 @@ def test_missing_differ_file_does_not_throw_exception(self):
json.dump(
{
"rules": [{
"rule_id": "check_deleted_count",
"validator": "DELETED_COUNT",
"scope": {
"data_source": "differ"
},
"rule_id": "check_deleted_records_count",
"validator": "DELETED_RECORDS_COUNT",
"scope": {},
"params": {
"threshold": 10
}
Expand Down
16 changes: 10 additions & 6 deletions tools/import_validation/runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,16 @@ def __init__(self, validation_config_path: str, differ_output: str,
(self.validator.validate_max_date_latest, 'stats'),
'MAX_DATE_CONSISTENT':
(self.validator.validate_max_date_consistent, 'stats'),
'DELETED_COUNT': (self.validator.validate_deleted_count, 'differ'),
'DELETED_RECORDS_COUNT':
(self.validator.validate_deleted_records_count, 'differ'),
'MISSING_REFS_COUNT':
(self.validator.validate_missing_refs_count, 'lint'),
'LINT_ERROR_COUNT':
(self.validator.validate_lint_error_count, 'lint'),
'MODIFIED_COUNT':
(self.validator.validate_modified_count, 'differ'),
'ADDED_COUNT': (self.validator.validate_added_count, 'differ'),
'MODIFIED_RECORDS_COUNT':
(self.validator.validate_modified_records_count, 'differ'),
'ADDED_RECORDS_COUNT':
(self.validator.validate_added_records_count, 'differ'),
'NUM_PLACES_CONSISTENT':
(self.validator.validate_num_places_consistent, 'stats'),
'NUM_PLACES_COUNT':
Expand Down Expand Up @@ -141,8 +143,10 @@ def _determine_required_sources(self) -> set[str]:
for rule in self.config.rules:
if not rule.get('enabled', True):
continue
if 'scope' in rule and 'data_source' in rule['scope']:
req_sources.add(rule['scope']['data_source'])

validator_name = rule.get('validator')
if validator_name in self.validation_dispatch:
req_sources.add(self.validation_dispatch[validator_name][1])
return req_sources

def run_validations(self) -> tuple[bool, list[ValidationResult]]:
Expand Down
Loading
Loading