Scraper/checkDownloads.js at main · enBloc-org/Scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
import path from "path"
import fs from "fs"
import { errorLogColour } from "./utils/colours.js"
import { selectLatest } from "./model/states.js"
import { schoolDownload } from "./utils/downloader/schoolDownload.js"

const __dirname = new URL(".", import.meta.url).pathname
const downloadsDir = path.resolve(__dirname, "utils/downloader/downloads")

const statesJson = selectLatest()
const states = JSON.parse(statesJson.states_file)

const iterateChecker = async () => {
  if (!fs.existsSync(downloadsDir)) {
    console.log(errorLogColour, "No files downloaded")
    return
  }
  console.log("Checking Downloads")

  const yearValue = {
    5: "2018-19",
    6: "2019-20",
    7: "2020-21",
    8: "2021-22",
    9: "2022-23",
  }

  let filesChecked = 0

  for (const state of states) {
    for (const district of state.districts) {
      for (const block of district.blocks) {
        for (const school of block.schoolList) {
          for (let i = 5; i <= 9; i++) {
            const currentYear = i

            // only action endpoints where a download is expected to be available
            if (
              school[
                `isOperational${yearValue[currentYear].replace("-", "")}`
              ] === 0
            ) {
              const oldFileName = path.join(
                downloadsDir,
                `${school.districtId}-${school.blockId}_${
                  yearValue[currentYear]
                }_${school.schoolName.replace(/[/?<>\\:*|"\s]/g, "-")}`,
              )

              const newFileName = path.join(
                downloadsDir,
                `${school.schoolId}_${
                  yearValue[currentYear]
                }_${school.schoolName.replace(/[/?<>\\:*|"\s]/g, "-")}`,
              )

              if (fs.existsSync(oldFileName)) {
                const content = fs
                  .readFileSync(oldFileName, "utf-8")
                  .substring(0, 9)

                const isValidContent = content === "JVBERi0xL"

                // retry downloading if the file is corrupted
                if (!isValidContent && content !== "1") {
                  console.groupCollapsed(`Retrying ${school.schoolName}`)

                  await new Promise(async resolve => {
                    fs.unlinkSync(oldFileName)
                    const trigger = await schoolDownload(
                      school,
                      currentYear,
                      oldFileName,
                    )
                    resolve(trigger)
                  })

                  console.groupEnd()
                } else if (content === "1") {
                  // delete files corrupted at the source
                  fs.unlinkSync(oldFileName)
                }

                // rename the file after checking
                fs.renameSync(oldFileName, newFileName)

                filesChecked++
              } else if (fs.existsSync(newFileName)) {
                // check file under new name if it exists
                const content = fs
                  .readFileSync(newFileName, "utf-8")
                  .substring(0, 9)

                const isValidContent = content === "JVBERi0xL"

                // retry downloading if the file is corrupted
                if (!isValidContent && content !== "1") {
                  console.groupCollapsed(`Retrying ${school.schoolName}`)

                  await new Promise(async resolve => {
                    fs.unlinkSync(newFileName)
                    const trigger = await schoolDownload(
                      school,
                      currentYear,
                      newFileName,
                    )
                    resolve(trigger)
                  })

                  console.groupEnd()
                } else if (content === "1") {
                  // delete files corrupted at the source
                  fs.unlinkSync(newFileName)
                }

                filesChecked++
              }
            }
          }
        }
      }
    }
  }

  console.log(`${filesChecked} files checked`)
}

iterateChecker()