GetData/INE.R at master · puigjos/GetData · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151

get_INE <- function(){

  require(rvest)
  require(curl)
  require(httr)
  require(XML)
  require(xml2)
  require(data.table)
  require(dplyr)

  # First link --------------------------------------------------------------

  urls = 'http://www.ine.es/dyngs/INEbase/es/operacion.htm?c=Estadistica_C&cid=1254736177024&menu=resultados&idp=1254735976606'

  # sdata = read_html(iconv(urls,to = "UTF-8"), encoding = "latin1")
  sdata = read_html(urls)
  sdata = htmlParse(sdata,encoding="UTF-8")
  links <- xpathApply(sdata, "//a[contains(@href,'dynt3/inebase')]"
                      , function(x){
                        data.frame(name = xmlValue(x), href = xmlGetAttr(x, "href"))
                      })
  x = do.call(rbind.data.frame, links)
  First = data.table(id = 1:nrow(x),x)


  # Second link -------------------------------------------------------------

  u <- 'http://www.ine.es'
  msg = as.character(paste(First[[1]],':', First[[2]]))
  print(msg)
  cat('Escriba el número')
  number = scan(n=1)

  url = paste0(u, First$href[number])

  # sdata = read_html(iconv(url,to = "UTF-8"), encoding = "latin1")
  sdata = read_html(url)
  sdata = htmlParse(sdata,encoding="UTF-8")
  links <- xpathApply(sdata, "//a[contains(@href,'capsel')]"
                      , function(x){
                        data.frame(name = xmlValue(x, ignoreComments=T), href = xmlGetAttr(x, "href"))
                      })
  if(length(links)!=0){
    x = do.call(rbind.data.frame, links)
    Second = data.table(id = 1:nrow(x),x)
    su = paste0('\t', 1:40, collapse = '')
    Second$name = gsub(paste0('[\n\t1\t3',su,']') , '', Second$name)


    # Third link --------------------------------------------------------------

    u <- 'http://www.ine.es/dynt3/inebase/index.htm'
    msg = as.character(paste(Second[[1]],':', Second[[2]]))
    print(msg)
    cat('Escriba el número:')
    number = scan(n=1)

    url = paste0(u, Second$href[number])
  }
  # sdata = read_html(iconv(url,to = "UTF-8"), encoding = "latin1")
  sdata = read_html(url)
  sdata = htmlParse(sdata,encoding="UTF-8")
  links <- xpathApply(sdata, "//a[contains(@href,'/jaxi/Tabla')]"
                      , function(x){
                        data.frame(name = xmlValue(x, ignoreComments=T), href = xmlGetAttr(x, "href"))
                      })
  # links
  x = do.call(rbind.data.frame, links)
  Third = data.table(id = 1:nrow(x),x)


  # Fourth ------------------------------------------------------------------

  u <- 'http://www.ine.es'
  msg = as.character(paste(Third[[1]],':', Third[[2]]))
  print(msg)
  cat('Escriba el número')
  number = scan(n=1)

  url = paste0(u, Third$href[number])
  sdata = read_html(url)
  sdata = htmlParse(sdata,encoding="UTF-8")
  links <- xpathApply(sdata, "//a[contains(@href,'dlgExport.htm')]"
                      , function(x){
                        data.frame(name = xmlValue(x, ignoreComments=T), href = xmlGetAttr(x, "href"))
                      })
  # links
  x = do.call(rbind.data.frame, links)
  LINK = x$href


  # Se abre para descargar --------------------------------------------------

  u <- 'http://www.ine.es/jaxi/'
  url = paste0(u, LINK)
  sdata = read_html(url)
  sdata = htmlParse(sdata,encoding="UTF-8")
  links <- xpathApply(sdata, "//a[contains(@href,'xlsx')]"
                      , function(x){
                        data.frame(name = xmlValue(x, ignoreComments=T), href = xmlGetAttr(x, "href"))
                      })
  links
  x = do.call(rbind.data.frame, links)
  LINK2 = x$href

  url = paste0(u, LINK2)

  temp <- paste0(getwd(),'/f1.xlsx')
  download.file(url, temp, mode = 'wb')
  data = data.table(readxl::read_excel(temp))
  unlink(temp)

  #eliminaremos las primeras observaciones
  elim = 1:(min(which(is.na(data[[2]])==F))-1)
  data1 = data[!elim]
  header = as.character(unlist(data1[1,]))
  data1 = data1[-1]
  colnames(data1) = header
  #Lo mismo pero des de abajo
  for(i in length(data1[[2]]):1){
    if(is.na(data1[[2]])[i-1]==F){
      elim = i
      break
    }
  }
  data2 = data1[1:(elim-1)]

  nms = c()
  v = c()
  for(i in 1:nrow(data2)){
    if(is.na(data2[[2]][i])){
      v = data2[[1]][i]
    }else{
      nm <- paste0(v, ifelse(is.null(v), '', '_'), data2[[1]][i])
      nms = c(nms, nm)
    }
  }


  data2 = data2[!is.na(data2[[2]])]

  dt <- data.table(header[-1], transpose(data2[,-1]))
  names(dt) <- c('Date', as.character(nms))
  dt = dt %>% arrange(Date) %>% as.data.table()
  dt[, names(dt)[-1] := lapply(.SD, as.numeric), .SD = names(dt)[-1]]
  dt$Date <- as.yearmon(dym(dt$Date))
  return(dt)
}