---
title: "Eurostat data"
author: "F.M. Stefanini"
date: "5 aprile 2019"
output: html_document
editor_options: 
  chunk_output_type: console
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = FALSE)
library(convergEU)
library(tidyverse)
```
 
 
 
# How to download and filter Eurostat  data 
 
 
 
The heterogeneity in the structure of different indicators
requires some cares.
Below a list of covariates is shown for each indicator
and one of them must be selected to produce a tidy dataset
time by countries.
 
```{r}
# raw download
names(convergEU_glb())
names( convergEU_glb()$metaEUStat )
convergEU_glb()$metaEUStat$subSelection
convergEU_glb()$metaEUStat$selectorUser

ttmp2 <- download_indicator_EUS(
  indicator_code="labourcost_i", # metaEUStat$selectorUser
  fromTime = 2005,
  toTime = 2015,
  gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
  ageInterv = NA,
  countries =  convergEU_glb()$EU28$memberStates$codeMS,
  rawDump=T)

# ttmp2 <- mutate(ttmp2, etica= paste("unit= ",.data$unit,"; item: ",.data$na_item,sep=""))
# unique(ttmp2$etica)
# 
# filter(ttmp2, unit=="PPS", na_item=="D1_SAL_HW")




#nomi variabili di classificazione
nomiCriteria<-list() 
for(vaName in convergEU_glb()$metaEUStat$selectorUser){
  cat("\n\n----------------------------------------------------")
  cat(vaName,"\n\n")
  downTB <- download_indicator_EUS(
            indicator_code=vaName, # metaEUStat$selectorUser
            fromTime = 2005,
            toTime = 2015,
            gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
            ageInterv = NA,
            countries =  convergEU_glb()$EU28$memberStates$codeMS,
            rawDump=T)
  print(head(downTB))
  nomiCriteria[[vaName]] <- names(downTB)
}
length(nomiCriteria)

# estraggo quelli ovvi e noti: sex,geo,age,time,values
nomiCriteriaMin <- lapply(nomiCriteria,function(vx){
  setdiff(vx,c("sex","geo","age","time","values"))
})

## ora il cartesiano degli attributi popolati 

selezionaSubset <- function(vaName,nomiCriteriaMin){
  downTB <- download_indicator_EUS(
            indicator_code=vaName, # metaEUStat$selectorUser
            fromTime = 2005,
            toTime = 2015,
            gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
            ageInterv = NA,
            countries =  convergEU_glb()$EU28$memberStates$codeMS,
            rawDump=T)
  estraCarte <- nomiCriteriaMin[[vaName]]
  puntiCampionari <- lapply(estraCarte,function(vx){
        as.character(unlist(unique(downTB[,vx])))
  })
  cartesiano <- expand.grid(puntiCampionari)
  names(cartesiano)<-estraCarte
  cartesianoNum<- mutate(cartesiano,conta=NA)
  for(righeDB in 1:nrow(cartesiano)){
    contatore<- rep(TRUE,nrow(downTB))
    for(aux in estraCarte){
      contatore <- contatore & 
        (unlist(downTB[,aux]) ==  as.character(cartesiano[righeDB,aux])) 
    }
    cartesianoNum$conta[righeDB]<- sum(contatore)
  }
  # select if greater than zero
  estrattore <- which(cartesianoNum$conta >0)
  cartesianoNum<- cartesianoNum[estrattore,]
  list(paste("\n------------------\nSummary for variable:  ",
     vaName,"\n",
     "Please pick one of the following rows:\n",sep=""),
  cartesianoNum) 
}
## testa
# debug(selezionaSubset)
# resFF <- selezionaSubset("lfsa_argaed",nomiCriteriaMin)
# resFF <- selezionaSubset(convergEU_glb()$metaEUStat$selectorUser[20],nomiCriteriaMin)
# cat(resFF [[1]])
# resFF[[2]] 
```
 
 
 
 
```{r,eval=F}
resAll<- list()
punta<- 19
for(indicaName in convergEU_glb()$metaEUStat$selectorUser){
  punta<- punta +1
  print(punta)
   resAll[[indicaName]] <- selezionaSubset(indicaName,nomiCriteriaMin)
   #cat(resAll [[1]])
   #print(resAll[[2]] )
  }
 
for(indicaName in convergEU_glb()$metaEUStat$selectorUser){
    cat(resAll [[1]])
    print(resAll[[2]] )
  }
```
 
 
 
 
```{r}
# solo elem3enti singoli
selezionaSubsetSingoli <- function(vaName,nomiCriteriaMin){
  downTB <- download_indicator_EUS(
            indicator_code=vaName, # metaEUStat$selectorUser
            fromTime = 2005,
            toTime = 2015,
            gender= c(NA,"T","F","M")[1],#c("Total","Females","Males")
            ageInterv = NA,
            countries =  convergEU_glb()$EU28$memberStates$codeMS,
            rawDump=T)
  estraCarte <- nomiCriteriaMin[[vaName]]
  puntiCampionari <- lapply(estraCarte,function(vx){
        as.character(unlist(unique(downTB[,vx])))
  })
  names(puntiCampionari) <-  estraCarte
  
  # select if greater than zero
  list(paste("\n------------------\nSummary for variable:  ",
     vaName,"\n",
     "Please pick one of the following items for each list component (variable):\n",sep=""),
  puntiCampionari) 
}



bigSTRres<- " "
for(indicaName in convergEU_glb()$metaEUStat$selectorUser){
  resAll <-  selezionaSubsetSingoli(indicaName,nomiCriteriaMin) 
  if(length(resAll[[2]])>0){
    bigSTRres1<- paste(resAll[[1]], "\n")
    bigSTRres <- paste(bigSTRres,bigSTRres1) 
    for(aux in 1:length(resAll[[2]])){
      bigSTRres2<- paste("\n",names(resAll[[2]])[aux],":\n  ",
      paste(resAll[[2]][[aux]],collapse="; "),"\n\n")
      bigSTRres <- paste(bigSTRres,bigSTRres2) 
     }
    }
  }

cat(bigSTRres,file="/media/fred/STORE/PRJ/2018-TENDER-EU/STEP-1/bitbucketed/convergEU/inst/EUstat_download/toRevise.txt")

```
 
 
 
 
 
 
 
 
 
 
 
 
<!-- 
 
```{r}
# library(convergEU)
undebug(download_indicator_EUS)
debug(download_indicator_EUS)

names(convergEU_glb())
names( convergEU_glb()$metaEUStat )
convergEU_glb()$metaEUStat$subSelection
convergEU_glb()$metaEUStat$selectorUser

get_eurostat(id="labourcost_i")
search_eurostat("labourcost", fixed=T)

tmp <- search_eurostat("Labour cost",fixed=TRUE)
tmp
tmp <- search_eurostat("cost",fixed=TRUE)
tmp




#
ttmp <- download_indicator_EUS(
  indicator_code="hlth_dm060", # metaEUStat$selectorUser
  fromTime = 2005,
  toTime = 2015,
  gender= c(NA,"T","F","M")[2],#c("Total","Females","Males")
  ageInterv = "Y16-64",
  countries =  convergEU_glb()$EU28$memberStates$codeMS)
#


names(convergEU_glb())
####namesInDBEUF <- convergEU_glb()$metaEUStat$Code_in_database

# namesInDBEUS <- convergEU_glb()$metaEUStat$Official_code
metaEUStat <- convergEU_glb()$metaEUStat
#metaEUStat <- mutate(metaEUStat,Official_code_purified = Official_code)
metaEUStat$Official_code_purified <- c("lfsa_argaed", "lfsa_ergaed",
                                       "lfsa_ergacob", "lfsa_urgan",
                                       "une_educ_a", "lfsa_upgan", "edat_lfse_20", "une_rt_a", "ilc_lvhl36",
                                       "ilc_lvhl30", "lfsa_egad", "lfsa_eppgai",
                                       "lfsi_emp_a",
                                       "lfsi_pt_a",
                                       "lfsa_egised",
                                       "earn_gr_gpgr2",
                                       "lfsa_esgan", "lfsa_esgan",
                                       "lfsa_etgar", "prc_ppp_ind", "gov_10dd_edpt1", "nasa_10_ki",
                                       "gov_10a_main", "nama_10_lp_ulc", "nama_10_lp_ulc", "nama_10_lp_ulc",
                                       "earn_nt_net", "spr_exp_sum", "gov_10a_exp", "ilc_pnp3", "spr_exp_pens",
                                       "spr_pns_ben", "edat_lfse_14", "trng_lfse_01", "edat_lfse_03",
                                       "gov_10a_exp", "hlth_silc_08", "gov_10a_exp",

                                       "isoc_sk_dskl_i",

                                       "isoc_bde15b_h", "isoc_ci_im_i", "isoc_ec_ibuy", "isoc_ci_it_en2",
                                       "rd_p_perslf", "rd_e_gerdtot", "isoc_ciegi_ac", "gov_10a_exp",

                                       "lfsa_eegan2",
                                       "prc_hicp_aind",

                                       "ilc_di12", # missing

                                       "ilc_di11", "lfst_r_lmder", "demo_mlexpec", "hlth_hlye",
                                       "y16_deprindex",
                                       "hlth_dm060")


namesInDBEUS <- metaEUStat$Official_code_purified

library(eurostat)

toc <-  get_eurostat_toc()
outS <-  search_eurostat("Deprivation")
outS[,1:2]
# myES <- get_eurostat(outS[1,2], type = "label", time_format = "num")
# save(myES,file="myES.RData")
# load(file="myES.RData")
# levels(myES$na_item)
# levels(myES$geo)
ageEUS <- list()
sexEUS <- list()
geoEUS <- list()
timeEUS <- list()
for(aux in namesInDBEUS[- c(
  which("y16_deprindex" == namesInDBEUS),
  which("ilc_di12" == namesInDBEUS)
)]
){
  myES2 <- get_eurostat(aux,  time_format = "num")
  print(myES2)
  ageEUS[[aux]] <-  unique(myES2$age)
  sexEUS[[aux]] <-  unique(myES2$sex)
  geoEUS[[aux]] <-  unique(unlist(myES2$geo))
  timeEUS[[aux]] <-  c(min(myES2$time),max(myES2$time))
  #clean_eurostat_cache()
}
# clean_eurostat_cache()
myES2lab <-label_eurostat(myES2)
#unique(myES2lab$)
unique(myES2lab$sector)
unique(myES2lab$cofog99)
unique(myES2lab$na_item)
unique(myES2lab$)
```{r,eval=FALSE}
names(convergEU_glb())
####namesInDBEUF <- convergEU_glb()$metaEUStat$Code_in_database

# namesInDBEUS <- convergEU_glb()$metaEUStat$Official_code
metaEUStat <- convergEU_glb()$metaEUStat
namesInDBEUS <- metaEUStat$Official_code_purified











library(eurostat)

toc <-  get_eurostat_toc()
outS <-  search_eurostat("Deprivation Index")
outS[,1:2]

ageEUS <-  list()
sexEUS <-  list()
geoEUS <-  list()
timeEUS <- list()
for(aux in namesInDBEUS[- c(
  which("y16_deprindex" == namesInDBEUS),
  which("ilc_di12" == namesInDBEUS)]
){
  myES2 <- get_eurostat(aux,  time_format = "num")
  print(myES2)
  ageEUS[[aux]] <-  unique(myES2$age)
  sexEUS[[aux]] <-  unique(myES2$sex)
  geoEUS[[aux]] <-  unique(unlist(myES2$geo))
  timeEUS[[aux]] <-  c(min(myES2$time),max(myES2$time))
  #clean_eurostat_cache()
}
# clean_eurostat_cache()
myES2lab <-label_eurostat(myES2)
#unique(myES2lab$)
unique(myES2lab$sector)
unique(myES2lab$cofog99)
unique(myES2lab$na_item)
unique(myES2lab$)

estrattore <- which(convergEU_glb()$metaEUStat$Official_code_purified == "gov_10a_exp"  )
convergEU_glb()$metaEUStat[estrattore,]
## diverse vars dentro un dataset
# 1 gov_10a_exp   expspr_p         General government expenditure in social protect 
# 2 gov_10a_exp   expedu_p         General government expenditure in education as p 
# 3 gov_10a_exp   exphlth_p        General government expenditure in health as perc 
# 4 gov_10a_exp   exppubserv_p

estrattore <- which(convergEU_glb()$metaEUStat$Official_code_purified == "lfsa_esgan"  )
convergEU_glb()$metaEUStat[estrattore,]
#   Official_code     Code_in_database indicator     Official_code_pu  subSelection
#   <chr>             <chr>            <chr>         <chr>             <chr>
# 1 Own calculation.  selfemp_p        Self employm  lfsa_esgan        NA
# 2 Own calculation.  selfempwe_p      Self employm  lfsa_esgan        NA

estrattore <- which(convergEU_glb()$metaEUStat$Official_code_purified == "nama_10_lp_ulc"  )
convergEU_glb()$metaEUStat$Code_in_database[estrattore]
#   Official_code  Code_in_database indicator         Official_code_p  subSelection
#   <chr>          <chr>            <chr>             <chr>            <chr>
# 1 nama_10_lp_ulc labourcost_i     Nominal unit lab  nama_10_lp_ulc   NA
# 2 nama_10_lp_ulc labourprod_i     Real labour prod  nama_10_lp_ulc   NA
# 3 nama_10_lp_ulc compemp_pps      Compensation per  nama_10_lp_ulc   NA

unique(myES2$age)
unique(myES2$time)
unique(myES2$sex)
unique(myES2$isced11)


## eccezioni
ttmp <- NULL
try(ttmp <- invisible(get_eurostat("scaramacao",  time_format = "num")))
warnings()
error()

tryCatch(
  #ttmp <- invisible(get_eurostat("scaramacao",  time_format = "num")),
  ttmp <- (get_eurostat("scaramacao",  time_format = "num")),
  error = function(ee){list(res=NULL,
                            err="Error: download failed.",
                            msg= ttmp)},
  warning = function(ww){print(ww)},
  message = function(mm){print(mm)}
)

purrr::possibly(get_eurostat, NA, quiet = TRUE)("scaramacao",  time_format = "num")

purrr::possibly(get_eurostat, NA, quiet = FALSE)("ilc_di12",  time_format = "num")



mytmp <- utils::capture.output(
  tttmp <- purrr::possibly(get_eurostat, NA, quiet = FALSE)("scaramacao",
                                                            time_format = "num"), type="message")


mytmp <- utils::capture.output(
  tttmp <- purrr::possibly(get_eurostat, NA, quiet = FALSE)("ilc_di12",
                                                            time_format = "num"), type="message")


```

--> 
 
 
