Sotkanet API R tools

Leo Lahti

2021-05-03

This is the sotkanet R package to access data from the Sotkanet portal that provides over 2000 demographic indicators across Finland and Europe, maintained by the National Institute for Health and Welfare (THL). For more information, see Information about Sotkanet and API description. This package is part of rOpenGov.

Installation

Release version from CRAN:

install.packages("sotkanet")

Development version from GitHub:

library(devtools)
install_github("ropengov/sotkanet")

Usage

Listing available indicators

Load sotkanet and other libraries used in this vignette

List available Sotkanet indicators:

sotkanet.indicators <- SotkanetIndicators(type = "table")
# kable(head(sotkanet.indicators))

List geographical regions with available indicators:

options(width = 60)
sotkanet.regions <- SotkanetRegions(type = "table")
kable(head(sotkanet.regions))
region region.title.fi region.code region.category region.uri
833 Etelä-Suomen AVIn alue 1 ALUEHALLINTOVIRASTO http://www.yso.fi/onto/kunnat/ahv1
834 Lounais-Suomen AVIn alue 2 ALUEHALLINTOVIRASTO http://www.yso.fi/onto/kunnat/ahv2
835 Itä-Suomen AVIn alue 3 ALUEHALLINTOVIRASTO http://www.yso.fi/onto/kunnat/ahv3
836 Länsi- ja Sisä-Suomen AVIn alue 4 ALUEHALLINTOVIRASTO http://www.yso.fi/onto/kunnat/ahv4
837 Pohjois-Suomen AVIn alue 5 ALUEHALLINTOVIRASTO http://www.yso.fi/onto/kunnat/ahv5
838 Lapin AVIn alue 6 ALUEHALLINTOVIRASTO http://www.yso.fi/onto/kunnat/ahv6

Querying SOTKAnet indicators

Get the indicator no. 10013 (Unemployment rate of young people aged 16-24 years) from Sotkanet, from Finland (Suomi) for 1990-2012 (Eurostat employment statistics youth unemployment), and plot a graph:

# Get indicator data
dat <- GetDataSotkanet(indicators = 10013, years = 1990:2012, 
                   genders = c('female', 'male', 'total'), 
               region.category = "EUROOPPA", regions = "Suomi")

# Investigate the first lines in the data
kable(head(dat)) %>% 
  kable_styling() %>%
  kableExtra::scroll_box(width = "100%")
region region.title.fi region.code region.category indicator indicator.title.fi year gender primary.value absolute.value indicator.organization.title.fi
10013.727 1022 Suomi 246 EUROOPPA 10013 (EU) Nuorisotyöttömyysaste 2008 male 17.1 NA Euroopan yhteisöjen tilastotoimisto (Eurostat)
10013.728 1022 Suomi 246 EUROOPPA 10013 (EU) Nuorisotyöttömyysaste 2004 female 19.4 NA Euroopan yhteisöjen tilastotoimisto (Eurostat)
10013.729 1022 Suomi 246 EUROOPPA 10013 (EU) Nuorisotyöttömyysaste 1996 female 31.6 NA Euroopan yhteisöjen tilastotoimisto (Eurostat)
10013.730 1022 Suomi 246 EUROOPPA 10013 (EU) Nuorisotyöttömyysaste 1999 male 20.8 NA Euroopan yhteisöjen tilastotoimisto (Eurostat)
10013.731 1022 Suomi 246 EUROOPPA 10013 (EU) Nuorisotyöttömyysaste 2011 female 18.4 NA Euroopan yhteisöjen tilastotoimisto (Eurostat)
10013.732 1022 Suomi 246 EUROOPPA 10013 (EU) Nuorisotyöttömyysaste 1997 female 28.0 NA Euroopan yhteisöjen tilastotoimisto (Eurostat)

Fetch all SOTKAnet indicators

This takes for a long time and is not recommended for regular use. Save the data on your local disk for further work.

# These indicators have problems with R routines:
problematic.indicators <- c(1575, 1743, 1826, 1861, 1882, 1924, 1952, 2000, 
                            2001, 2033, 2050, 3386, 3443)

# Get data for all indicators
datlist <- list()
for (ind in setdiff(sotkanet.indicators$indicator, problematic.indicators)) {
  datlist[[as.character(ind)]] <- GetDataSotkanet(indicators = ind, 
        years = 1990:2013, genders = c('female', 'male', 'total'))
}

# Combine tables (this may require considerable time and memory 
# for the full data set)
dat <- do.call("rbind", datlist)

Visualization

Download and visualize time series:

# Pick indicator name
indicator.name <- as.character(unique(dat$indicator.title.fi))
indicator.source <- as.character(unique(dat$indicator.organization.title.fi))

# Visualize
library(ggplot2)
theme_set(theme_bw(20)); 
p <- ggplot(dat, aes(x = year, y = primary.value, group = gender, color = gender)) 
p <- p + geom_line() + ggtitle(paste(indicator.name, indicator.source, sep = " / ")) 
p <- p + xlab("Year") + ylab("Value") 
p <- p + theme(title = element_text(size = 10))
p <- p + theme(axis.title.x = element_text(size = 20))
p <- p + theme(axis.title.y = element_text(size = 20))
p <- p + theme(legend.title = element_text(size = 15))
print(p)

Investigate the effect of municipality size on demographic variation. Smaller municipalities show more random variation as expected by statistical arguments:

selected.inds <- c(127, 178)
dat <- GetDataSotkanet(indicators = selected.inds, 
                years = 2011, genders = c('total'))
# Pick necessary fields and remove duplicates
datf <- dat[, c("region.title.fi", "indicator.title.fi", "primary.value")]
datf <- datf[!duplicated(datf),]
dw <- reshape(datf, idvar = "region.title.fi", 
                timevar = "indicator.title.fi", direction = "wide")
names(dw) <- c("Municipality", "Population", "Migration")
p <- ggplot(dw, aes(x = log10(Population), y = Migration)) +
       geom_point(size = 3)
       ggtitle("Migration vs. population size") 
#> $title
#> [1] "Migration vs. population size"
#> 
#> attr(,"class")
#> [1] "labels"
       theme(title = element_text(size = 15))
#> List of 1
#>  $ title:List of 11
#>   ..$ family       : NULL
#>   ..$ face         : NULL
#>   ..$ colour       : NULL
#>   ..$ size         : num 15
#>   ..$ hjust        : NULL
#>   ..$ vjust        : NULL
#>   ..$ angle        : NULL
#>   ..$ lineheight   : NULL
#>   ..$ margin       : NULL
#>   ..$ debug        : NULL
#>   ..$ inherit.blank: logi FALSE
#>   ..- attr(*, "class")= chr [1:2] "element_text" "element"
#>  - attr(*, "class")= chr [1:2] "theme" "gg"
#>  - attr(*, "complete")= logi FALSE
#>  - attr(*, "validate")= logi TRUE
       theme(axis.title.x = element_text(size = 20))
#> List of 1
#>  $ axis.title.x:List of 11
#>   ..$ family       : NULL
#>   ..$ face         : NULL
#>   ..$ colour       : NULL
#>   ..$ size         : num 20
#>   ..$ hjust        : NULL
#>   ..$ vjust        : NULL
#>   ..$ angle        : NULL
#>   ..$ lineheight   : NULL
#>   ..$ margin       : NULL
#>   ..$ debug        : NULL
#>   ..$ inherit.blank: logi FALSE
#>   ..- attr(*, "class")= chr [1:2] "element_text" "element"
#>  - attr(*, "class")= chr [1:2] "theme" "gg"
#>  - attr(*, "complete")= logi FALSE
#>  - attr(*, "validate")= logi TRUE
       theme(axis.title.y = element_text(size = 20))
#> List of 1
#>  $ axis.title.y:List of 11
#>   ..$ family       : NULL
#>   ..$ face         : NULL
#>   ..$ colour       : NULL
#>   ..$ size         : num 20
#>   ..$ hjust        : NULL
#>   ..$ vjust        : NULL
#>   ..$ angle        : NULL
#>   ..$ lineheight   : NULL
#>   ..$ margin       : NULL
#>   ..$ debug        : NULL
#>   ..$ inherit.blank: logi FALSE
#>   ..- attr(*, "class")= chr [1:2] "element_text" "element"
#>  - attr(*, "class")= chr [1:2] "theme" "gg"
#>  - attr(*, "complete")= logi FALSE
#>  - attr(*, "validate")= logi TRUE
       theme(legend.title = element_text(size = 15))
#> List of 1
#>  $ legend.title:List of 11
#>   ..$ family       : NULL
#>   ..$ face         : NULL
#>   ..$ colour       : NULL
#>   ..$ size         : num 15
#>   ..$ hjust        : NULL
#>   ..$ vjust        : NULL
#>   ..$ angle        : NULL
#>   ..$ lineheight   : NULL
#>   ..$ margin       : NULL
#>   ..$ debug        : NULL
#>   ..$ inherit.blank: logi FALSE
#>   ..- attr(*, "class")= chr [1:2] "element_text" "element"
#>  - attr(*, "class")= chr [1:2] "theme" "gg"
#>  - attr(*, "complete")= logi FALSE
#>  - attr(*, "validate")= logi TRUE
print(p)

Further examples

For further usage examples, see rOpenGov-blog, and takomo, and Helsinki Region Infoshare-blog

Licensing and Citations

SOTKAnet data

Cite SOTKAnet and link to https://sotkanet.fi/sotkanet/fi/index. Also mention indicator provider.

Central points:

SOTKAnet R package

This work can be freely used, modified and distributed under the Two-clause BSD license.

citation("sotkanet")
#> 
#> Kindly cite the sotkanet R package as follows:
#> 
#>   (C) Leo Lahti, Einari Happonen, Juuso Parkkinen,
#>   Joona Lehtomaki, Vesa Saaristo and Pyry Kantanen
#>   2013-2021. sotkanet: Sotkanet Open Data Access and
#>   Analysis
#> 
#> A BibTeX entry for LaTeX users is
#> 
#>   @Misc{,
#>     title = {sotkanet R package},
#>     author = {Leo Lahti and Einari Happonen and Juuso Parkkinen and Joona Lehtomaki and Vesa Saaristo and Pyry Kantanen},
#>     year = {2013},
#>   }
#> 
#> For more info, see:
#> https://github.com/ropengov/sotkanet/

Session info

This vignette was created with

sessionInfo()
#> R version 4.0.4 (2021-02-15)
#> Platform: x86_64-pc-linux-gnu (64-bit)
#> Running under: Ubuntu 20.10
#> 
#> Matrix products: default
#> BLAS:   /home/lemila/bin/R-4.0.4/lib/libRblas.so
#> LAPACK: /home/lemila/bin/R-4.0.4/lib/libRlapack.so
#> 
#> locale:
#>  [1] LC_CTYPE=en_US.UTF-8       LC_NUMERIC=C              
#>  [3] LC_TIME=en_US.UTF-8        LC_COLLATE=en_US.UTF-8    
#>  [5] LC_MONETARY=en_US.UTF-8    LC_MESSAGES=en_US.UTF-8   
#>  [7] LC_PAPER=en_US.UTF-8       LC_NAME=C                 
#>  [9] LC_ADDRESS=C               LC_TELEPHONE=C            
#> [11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C       
#> 
#> attached base packages:
#> [1] stats     graphics  grDevices utils     datasets 
#> [6] methods   base     
#> 
#> other attached packages:
#> [1] ggplot2_3.3.3    kableExtra_1.3.4 magrittr_2.0.1  
#> [4] knitr_1.33       sotkanet_0.9.76 
#> 
#> loaded via a namespace (and not attached):
#>  [1] bslib_0.2.4        compiler_4.0.4    
#>  [3] pillar_1.6.0       jquerylib_0.1.4   
#>  [5] highr_0.9          tools_4.0.4       
#>  [7] digest_0.6.27      jsonlite_1.7.2    
#>  [9] evaluate_0.14      lifecycle_1.0.0   
#> [11] tibble_3.1.1       gtable_0.3.0      
#> [13] viridisLite_0.4.0  pkgconfig_2.0.3   
#> [15] rlang_0.4.10       DBI_1.1.1         
#> [17] rstudioapi_0.13    yaml_2.2.1        
#> [19] xfun_0.22          withr_2.4.2       
#> [21] dplyr_1.0.5        httr_1.4.2        
#> [23] stringr_1.4.0      xml2_1.3.2        
#> [25] generics_0.1.0     vctrs_0.3.7       
#> [27] sass_0.3.1         systemfonts_1.0.1 
#> [29] tidyselect_1.1.0   webshot_0.5.2.9000
#> [31] grid_4.0.4         svglite_2.0.0     
#> [33] glue_1.4.2         R6_2.5.0          
#> [35] fansi_0.4.2        rmarkdown_2.7.12  
#> [37] farver_2.1.0       purrr_0.3.4       
#> [39] scales_1.1.1       htmltools_0.5.1.1 
#> [41] ellipsis_0.3.1     assertthat_0.2.1  
#> [43] rvest_1.0.0        colorspace_2.0-0  
#> [45] labeling_0.4.2     utf8_1.2.1        
#> [47] stringi_1.5.3      munsell_0.5.0     
#> [49] crayon_1.4.1       rjson_0.2.20