importDataFromAMECOLink.R

### Usage example 1 -- not reshaped:
# > my_url <- 'https://webgate.ec.europa.eu/fastop/wq/ameco/online?fullVariable=3.0.0.0.ZEKND&countries=AUT,BEL,BGR,CYP,CZE,DEU,DNK,EA19,EA20,ESP,EST,EU27,FIN,FRA,GRC,HRV,HUN,IRL,ITA,LTU,LUX,LVA,MLT,NLD,POL,PRT,ROM,SVK,SVN,SWE&years=1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023,2024,2025,2026&lastYear=1&yearOrder=DESC'
# > x <- importDataFromAMECOLink(my_url, variables_to_columns=FALSE)
# > str(x)
# Classes ‘data.table’ and 'data.frame':	1972 obs. of  7 variables:
#  $ Country: chr  "Austria" "Austria" "Austria" "Austria" ...
#  $ CNTRY  : chr  "AUT" "AUT" "AUT" "AUT" ...
#  $ geo    : chr  "AT" "AT" "AT" "AT" ...
#  $ Label  : chr  "Capital-labour substitution: total economy" "Capital-labour substitution: total economy" "Capital-labour substitution: total economy" "Capital-labour substitution: total economy" ...
#  $ Unit   : chr  "(2020 = 100)" "(2020 = 100)" "(2020 = 100)" "(2020 = 100)" ...
#  $ year   : int  2027 2026 2025 2024 2023 2022 2021 2020 2019 2018 ...
#  $ value_ : num  NA 99.7 99.9 100.2 100.7 ...
#  - attr(*, ".internal.selfref")=<externalptr> 
#  - attr(*, "sorted")= chr "Country"
### Usage example 2 -- reshaped to a "wide" format:
# > my_other_url <- 'https://webgate.ec.europa.eu/fastop/wq/ameco/online?fullVariable=1.0.0.0.FETD,1.0.99.0.OKND&countries=AUT,BEL,BGR,CYP,CZE,DEU,DNK,EA19,EA20,ESP,EST,EU27,FIN,FRA,GRC,HRV,HUN,IRL,ITA,LTU,LUX,LVA,MLT,NLD,POL,PRT,ROM,SVK,SVN,SWE&years=1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023,2024,2025,2026&lastYear=1&yearOrder=DESC'
# > y <- importDataFromAMECOLink(my_other_url)
# > colnames(y)
# [1] "geo"                                                                                 
# [2] "year"                                                                                
# [3] "Employment, full-time equivalents: total economy (National accounts), 1000 FTEs"   
# [4] "Net capital stock at 2020 prices: total economy, Mrd ECU/EUR"                      
# [5] "Net capital stock at 2020 prices: total economy, Mrd ECU/EUR, Standard aggregation"

### The link/URL can be obtained by:
# going to AMECO online: https://ec.europa.eu/economy_finance/ameco_dashboard
# making there the needed selections
# clicking the "Next ->" button in the top-right corner
# clicking the "Web Query" link below the "Export" field
# opening the downloaded text file "ameco_online_wq.iqy".

library(magrittr)
library(rvest)
library(data.table)

# Main function

importDataFromAMECOLink <- function(AMECO_Web_Query_URL, variables_to_columns=TRUE)
    AMECO_Web_Query_URL %>% 
    read_html() %>% 
    html_node(xpath='/html/body/table') %>% 
    html_table(convert=FALSE) %>%
    as.data.table() %>% 
    melt(id.vars = c('Country','Label','Unit'),
         measure.vars = intColnames(.),
         variable.name='year',
         value.name='value_') %>%
    merge(AMECO_Eurostat_country_codes, by='Country') %>%
    .[,value_ := as.numeric(value_)] %>%
    .[,year := as.integer(as.character(year))] %>% 
    setcolorder(c('Country','CNTRY','geo')) %>% 
    `if`(variables_to_columns, reshapeAMECOtable(.), .)

# Helper functions

reshapeAMECOtable <- function(ameco_dt)
  ameco_dt %>% 
  .[, variable := paste0(Label,', ',Unit)] %>% 
  dcast(geo + year ~ variable,
        value.var='value_',
        fun.aggregate=identity,
        fill=NA_real_)
  
readMarkDownTable <- function(markdown_string)
  markdown_string %>% 
  fread(sep="|", header=TRUE, encoding='UTF-8') %>% 
  .[-1] %>% 
  .[, sapply(.,\(col) !is.logical(col)), with=FALSE]

# geo below is Eurostat's country code:
AMECO_Eurostat_country_codes <- '
| CNTRY  | geo        | Country         |
|--------|------------|-----------------|
| EU27   | EU27_2020  | European Union  |
| EA20   | EA20       | Euro area       |
| BEL    | BE         | Belgium         |
| BGR    | BG         | Bulgaria        |
| CZE    | CZ         | Czechia         |
| DNK    | DK         | Denmark         |
| DEU    | DE         | Germany         |
| EST    | EE         | Estonia         |
| IRL    | IE         | Ireland         |
| GRC    | EL         | Greece          |
| ESP    | ES         | Spain           |
| FRA    | FR         | France          |
| HRV    | HR         | Croatia         |
| ITA    | IT         | Italy           |
| CYP    | CY         | Cyprus          |
| LVA    | LV         | Latvia          |
| LTU    | LT         | Lithuania       |
| LUX    | LU         | Luxembourg      |
| HUN    | HU         | Hungary         |
| MLT    | MT         | Malta           |
| NLD    | NL         | Netherlands     |
| AUT    | AT         | Austria         |
| POL    | PL         | Poland          |
| PRT    | PT         | Portugal        |
| ROM    | RO         | Romania         |
| SVN    | SI         | Slovenia        |
| SVK    | SK         | Slovakia        |
| FIN    | FI         | Finland         |
| SWE    | SE         | Sweden          |
' %>% readMarkDownTable

intColnames <- function(dt)
  suppressWarnings(colnames(dt) %>% .[!is.na(as.integer(.))])