project-template/dataset.yml at main · dataciviclab/project-template · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
schema_version: 1
root: "./_smoke_out"

dataset:
  name: "bdap_http_csv"
  years: [2022]

raw:
  output_policy: "versioned"
  sources:
    - name: "bdap_csv"
      type: "http_file"
      args:
        url: "https://bdap-opendata.rgs.mef.gov.it/export/csv/Rendiconto-Pubblicato---Serie-storica---Saldi.csv"
        filename: "bdap_rendiconto_saldi_{year}.csv"
      primary: true

clean:
  sql: "sql/clean.sql"
  read_mode: "fallback"
  read:
    source: "config_only"
    mode: "explicit"
    include:
      - "bdap_rendiconto_saldi_*.csv"
    delim: ";"
    decimal: "."
    encoding: "utf-8"
    header: true
    columns: null
  required_columns:
    - "anno"
    - "saldo_netto"
    - "indebitamento_netto"
    - "avanzo_primario"
    - "entrate_finali"
    - "spese_finali"
  validate:
    primary_key:
      - "anno"
    not_null:
      - "anno"
    min_rows: 1

mart:
  tables:
    - name: "mart_ok"
      sql: "sql/mart/mart_ok.sql"
  required_tables:
    - "mart_ok"
  validate:
    table_rules:
      mart_ok:
        required_columns:
          - "anno"
          - "saldo_netto"
          - "entrate_finali"
          - "spese_finali"
        primary_key:
          - "anno"
        not_null:
          - "anno"
        min_rows: 1

validation:
  fail_on_error: true

output:
  artifacts: "minimal"
  legacy_aliases: true