---
title: "Automatic report"
author: "IOHprofiler"
date: "6/28/2019"
output: 
  pdf_document:
    keep_tex: true
    fig_caption: yes
params: 
  input: NA
  dsl: NA
  figure_folder: NA
  REG: NA
bibliography: bibliography.bib
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = FALSE)
input <- params$input
dsl <- params$dsl
figure_folder <- params$figure_folder
get_data_id <- function(dsList) {
  if (is.null(dsList) | length(dsList) == 0)
    return(NULL)

  paste(get_funcId(dsList), get_dim(dsList), sep = '-')
}
REG <- params$REG
```

```{asis, echo=input$Report.Introduction}
# Introduction

This report has been automatically generated by the IOHanalyzer [@IOHprofiler].

```

# Fixed-target view

```{r overview, results='asis', eval=input$Report.RT.Overview}
asis_output("### Overview of selected function / dimension pairs")
cat('\n')
for (dim in input$`Report.RT.Overview-DIM` ) {
  for (fid in input$`Report.RT.Overview-FuncId` ) {
    x <- paste0("### F", fid, " ", dim, "D \n")
    cat(x)
    cat('\n')
    opts_current$set(label =  paste0("Overview_F", fid, " ", dim, "D"))
    dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.RT.Overview-Alg` )
    p <- kable(get_RT_overview(dsl_sub), caption = paste0("Overview of selected algorithms on F", fid, " in ", dim, "D"), format = 'latex') %>% kable_styling(latex_options=c("scale_down", "hold_position"))
    print(p)
    cat('\n')

  }
}
```

```{r statistics, results='asis', eval=input$Report.RT.Statistics}
asis_output("### Runtime statistics of selected function / dimension pairs")
cat('\n')
for (dim in input$`Report.RT.Statistics-DIM` ) {
  for (fid in input$`Report.RT.Statistics-FuncId` ) {
    x <- paste0("### F", fid, " ", dim, "D \n")
    cat(x)
    cat('\n')
    opts_current$set(label =  paste0("Statistics_F", fid, " ", dim, "D"))
    dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.RT.Statistics-Alg`)
    name <- get_data_id(dsl_sub)
    fvs <- get_funvals(dsl_sub)
    tryCatch(
      {
      fv_seq <- seq_FV(fvs, as.numeric(REG$RTSummary.Statistics.Min[[name]]), as.numeric(REG$RTSummary.Statistics.Max[[name]]), 
                       by = as.numeric(REG$RTSummary.Statistics.Step[[name]])) 
    },
    error = function(e) { fv_seq <- seq_RT(fvs, length.out = 10)}
    )
    p <- kable(get_RT_summary(dsl_sub, fv_seq), 
               caption = paste0("Runtime statistics of selected algorithms on F", fid, " in ", dim, "D"), 
               format = 'latex') %>% 
      kable_styling(latex_options=c("scale_down", "hold_position"))
    print(p)
    cat('\n')

  }
}
```
 
```{r single_erts, eval=input$Report.RT.Single_ERT, results='hide', include=F}
out <- NULL
fig_names <- NULL
for (dim in input$`Report.RT.Single_ERT-DIM` ) {
  for (fid in input$`Report.RT.Single_ERT-FuncId` ) {
    name_figure <- paste0('ERT_single_F', fid, '-', dim, 'D')
    algs = input$`Report.RT.Single_ERT-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "Single_ERT.Rmd"))))
    fig_names <- c(fig_names, name_figure)
  }
}
```

`r if(input$Report.RT.Single_ERT) paste(out, collapse='\n')`

```{r single_erts_text, eval=input$Report.RT.Single_ERT, results='asis'}
asis_output(paste0("This section shows the ERTs of the selected algorithms on a per-function and per-dimension basis. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")})))
```


```{r multi_erts, eval=input$Report.RT.Multi_ERT, results='hide', include=F}
out <- NULL
for (dim in input$`Report.RT.Multi_ERT-DIM` ) {
  name_figure <- paste0('ERT_multi_', dim, 'D')
  algs = input$`Report.RT.Multi_ERT-Alg`
  env = new.env()
  out <- invisible(c(out,knit_child(text = knit_expand(file = "Multi_ERT.Rmd"))))
}
```

`r if(input$Report.RT.Multi_ERT) paste(out, collapse='\n')`

```{r ranking_erts, eval=input$Report.RT.Rank, results='hide', include=F}
out <- NULL
for (dim in input$`Report.RT.Rank-DIM` ) {
  name_figure <- paste0('RT_Rank_', dim, 'D')
  algs = input$`Report.RT.Rank-Alg`
  env = new.env()
  out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_Rank.Rmd"))))
}
```

`r if(input$Report.RT.Rank) paste(out, collapse='\n')`

```{r rt_hist, eval=input$Report.RT.Histogram, results='hide', include=F}
out <- NULL
for (dim in input$`Report.RT.Histogram-DIM` ) {
  for (fid in input$`Report.RT.Histogram-FuncId` ) {
    name_figure <- paste0('RT_Hist_F', fid, '-', dim, 'D')
    algs = input$`Report.RT.Histogram-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_Histogram.Rmd"))))
  }
}
```

`r if(input$Report.RT.Histogram) paste(out, collapse='\n')`

```{r rt_pmf, eval=input$Report.RT.PMF, results='hide', include=F}
out <- NULL
for (dim in input$`Report.RT.PMF-DIM` ) {
  for (fid in input$`Report.RT.PMF-FuncId` ) {
    name_figure <- paste0('RT_PMF_F', fid, '-', dim, 'D')
    algs = input$`Report.RT.PMF-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "RT_PMF.Rmd"))))
  }
}
```

`r if(input$Report.RT.PMF) paste(out, collapse='\n')`

```{r ecdf_single_target, eval=input$Report.RT.ECDF_Single_Target, results='hide', include=F}
out <- NULL
fig_names <- NULL
for (dim in input$`Report.RT.ECDF_Single_Target-DIM` ) {
  for (fid in input$`Report.RT.ECDF_Single_Target-FuncId` ) {
    name_figure <- paste0('ECDF_Single_Target', fid, '-', dim, 'D')
    algs = input$`Report.RT.ECDF_Single_Target-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_Single_Target.Rmd"))))
    fig_names <- c(fig_names, name_figure)
  }
}
```

`r if(input$Report.RT.ECDF_Single_Target) paste(out, collapse='\n')`

```{r ecdf_single_target_text, eval=input$Report.RT.ECDF_Single_Target, results='asis'}
asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")})))
```

```{r ecdf_single_function, eval=input$Report.RT.ECDF_Single_Function, results='hide', include=F}
out <- NULL
fig_names <- NULL
for (dim in input$`Report.RT.ECDF_Single_Function-DIM` ) {
  for (fid in input$`Report.RT.ECDF_Single_Function-FuncId` ) {
    name_figure <- paste0('ECDF_Single_Function', fid, '-', dim, 'D')
    algs = input$`Report.RT.ECDF_Single_Function-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_Single_Function.Rmd"))))
    fig_names <- c(fig_names, name_figure)
  }
}
```

`r if(input$Report.RT.ECDF_Single_Function) paste(out, collapse='\n')`

```{r ecdf_single_function_text, eval=input$Report.RT.ECDF_Single_Function, results='asis'}
asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")})))
```

```{r ecdf_aggr, eval=input$Report.RT.ECDF_Aggregated, results='hide', include=F}
algs = input$`Report.RT.ECDF_Aggregated-Alg`
env = new.env()
out <- knit_child(text = knit_expand(file = "ECDF_Aggregated.Rmd"))

```

`r if(input$Report.RT.ECDF_Aggregated) paste(out, collapse='\n')`

```{r ecdf_aggr_text, eval=input$Report.RT.ECDF_Aggregated, results='asis'}
asis_output(paste0("Filler text. This can be seen in Figure \\ref{fig:aggregated_ecdf}"))
```

```{r ecdf_auc, eval=input$Report.RT.ECDF_AUC, results='hide', include=F}
out <- NULL
fig_names <- NULL
for (dim in input$`Report.RT.ECDF_AUC-DIM` ) {
  for (fid in input$`Report.RT.ECDF_AUC-FuncId` ) {
    name_figure <- paste0('ECDF_AUC', fid, '-', dim, 'D')
    algs = input$`Report.RT.ECDF_AUC-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "ECDF_AUC.Rmd"))))
    fig_names <- c(fig_names, name_figure)
  }
}
```

`r if(input$Report.RT.ECDF_AUC) paste(out, collapse='\n')`

```{r ecdf_auc_text, eval=input$Report.RT.ECDF_AUC, results='asis'}
asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")})))
```

# Fixed-budget view

```{r fv_overview, results='asis', eval=input$Report.FV.Overview}
asis_output("### Overview of selected function / dimension pairs")
cat('\n')
for (dim in input$`Report.FV.Overview-DIM` ) {
  for (fid in input$`Report.FV.Overview-FuncId` ) {
    x <- paste0("### F", fid, " ", dim, "D \n")
    cat(x)
    cat('\n')
    opts_current$set(label =  paste0("Overview_FV_F", fid, " ", dim, "D"))
    dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.FV.Overview-Alg` )
    p <- kable(get_FV_overview(dsl_sub), caption = paste0("Overview of selected algorithms on F", fid, " in ", dim, "D"), format = 'latex') %>% kable_styling(latex_options=c("scale_down", "hold_position"))
    print(p)
    cat('\n')

  }
}
```

```{r fv_statistics, results='asis', eval=input$Report.FV.Statistics}
asis_output("### Runtime statistics of selected function / dimension pairs")
cat('\n')
for (dim in input$`Report.FV.Statistics-DIM` ) {
  for (fid in input$`Report.FV.Statistics-FuncId` ) {
    x <- paste0("### F", fid, " ", dim, "D \n")
    cat(x)
    cat('\n')
    opts_current$set(label =  paste0("Statistics_FV_F", fid, " ", dim, "D"))
    dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.FV.Statistics-Alg`)
    name <- get_data_id(dsl_sub)
    rts <- get_runtimes(dsl_sub)
    tryCatch(
      {
      rt_seq <- seq_RT(rts, as.numeric(REG$FCESummary.Statistics.Min[[name]]), as.numeric(REG$FCESummary.Statistics.Max[[name]]), 
                       by = as.numeric(REG$FCESummary.Statistics.Step[[name]])) 
    },
    error = function(e) { rt_seq <- seq_RT(rts, length.out = 10)}
    )
    p <- kable(get_FV_summary(dsl_sub, rt_seq), 
               caption = paste0("Runtime statistics of selected algorithms on F", fid, " in ", dim, "D"), 
               format = 'latex') %>% 
      kable_styling(latex_options=c("scale_down", "hold_position"))
    print(p)
    cat('\n')

  }
}
```
 
```{r single_fce, eval=input$Report.FV.Single_FCE, results='hide', include=F}
out <- NULL
fig_names <- NULL
for (dim in input$`Report.FV.Single_FCE-DIM` ) {
  for (fid in input$`Report.FV.Single_FCE-FuncId` ) {
    name_figure <- paste0('FCE_single_F', fid, '-', dim, 'D')
    algs = input$`Report.FV.Single_FCE-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "Single_FCE.Rmd"))))
    fig_names <- c(fig_names, name_figure)
  }
}
```

`r if(input$Report.FV.Single_FCE) paste(out, collapse='\n')`

```{r single_fce_text, eval=input$Report.FV.Single_FCE, results='asis'}
asis_output(paste0("This section shows the FCEs of the selected algorithms on a per-function and per-dimension basis. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")})))
```

```{r multi_fces, eval=input$Report.FV.Multi_FCE, results='hide', include=F}
out <- NULL
for (dim in input$`Report.FV.Multi_FCE-DIM` ) {
  name_figure <- paste0('FCE_multi_', dim, 'D')
  algs = input$`Report.FV.Multi_FCE-Alg`
  env = new.env()
  out <- invisible(c(out,knit_child(text = knit_expand(file = "Multi_FCE.Rmd"))))
}
```

`r if(input$Report.FV.Multi_FCE) paste(out, collapse='\n')`

```{r ranking_FVs, eval=input$Report.FV.Rank, results='hide', include=F}
out <- NULL
for (dim in input$`Report.FV.Rank-DIM` ) {
  name_figure <- paste0('FV_Rank_', dim, 'D')
  algs = input$`Report.FV.Rank-Alg`
  env = new.env()
  out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_Rank.Rmd"))))
}
```

`r if(input$Report.FV.Rank) paste(out, collapse='\n')`

```{r fv_hist, eval=input$Report.FV.Histogram, results='hide', include=F}
out <- NULL
for (dim in input$`Report.FV.Histogram-DIM` ) {
  for (fid in input$`Report.FV.Histogram-FuncId` ) {
    name_figure <- paste0('FV_Hist_F', fid, '-', dim, 'D')
    algs = input$`Report.FV.Histogram-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_Histogram.Rmd"))))
  }
}
```

`r if(input$Report.FV.Histogram) paste(out, collapse='\n')`

```{r fv_pmf, eval=input$Report.FV.PMF, results='hide', include=F}
out <- NULL
for (dim in input$`Report.FV.PMF-DIM` ) {
  for (fid in input$`Report.FV.PMF-FuncId` ) {
    name_figure <- paste0('FV_PMF_F', fid, '-', dim, 'D')
    algs = input$`Report.FV.PMF-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_PMF.Rmd"))))
  }
}
```

`r if(input$Report.FV.PMF) paste(out, collapse='\n')`

```{r fv_ecdf_single_target, eval=input$Report.FV.ECDF_Single_Target, results='hide', include=F}
out <- NULL
fig_names <- NULL
for (dim in input$`Report.FV.ECDF_Single_Target-DIM` ) {
  for (fid in input$`Report.FV.ECDF_Single_Target-FuncId` ) {
    name_figure <- paste0('FV_ECDF_Single_Target', fid, '-', dim, 'D')
    algs = input$`Report.FV.ECDF_Single_Target-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_Single_Target.Rmd"))))
    fig_names <- c(fig_names, name_figure)
  }
}
```

`r if(input$Report.FV.ECDF_Single_Target) paste(out, collapse='\n')`

```{r fv_ecdf_single_target_text, eval=input$Report.FV.ECDF_Single_Target, results='asis'}
asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")})))
```

```{r fv_ecdf_single_function, eval=input$Report.FV.ECDF_Single_Function, results='hide', include=F}
out <- NULL
fig_names <- NULL
for (dim in input$`Report.FV.ECDF_Single_Function-DIM` ) {
  for (fid in input$`Report.FV.ECDF_Single_Function-FuncId` ) {
    name_figure <- paste0('FV_ECDF_Single_Function', fid, '-', dim, 'D')
    algs = input$`Report.FV.ECDF_Single_Function-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_Single_Function.Rmd"))))
    fig_names <- c(fig_names, name_figure)
  }
}
```

`r if(input$Report.FV.ECDF_Single_Function) paste(out, collapse='\n')`

```{r fv_ecdf_single_function_text, eval=input$Report.FV.ECDF_Single_Function, results='asis'}
asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")})))
```

```{r fv_ecdf_auc, eval=input$Report.FV.ECDF_AUC, results='hide', include=F}
out <- NULL
fig_names <- NULL
for (dim in input$`Report.FV.ECDF_AUC-DIM` ) {
  for (fid in input$`Report.FV.ECDF_AUC-FuncId` ) {
    name_figure <- paste0('FV_ECDF_AUC', fid, '-', dim, 'D')
    algs = input$`Report.FV.ECDF_AUC-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "FV_ECDF_AUC.Rmd"))))
    fig_names <- c(fig_names, name_figure)
  }
}
```

`r if(input$Report.FV.ECDF_AUC) paste(out, collapse='\n')`

```{r fv_ecdf_auc_text, eval=input$Report.FV.ECDF_AUC, results='asis'}
asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")})))
```

```{r param_statistics, results='asis', eval=input$Report.Param.Statistics}
asis_output("### Statistics of parameters of selected function / dimension pairs")
cat('\n')
for (dim in input$`Report.Param.Statistics-DIM` ) {
  for (fid in input$`Report.Param.Statistics-FuncId` ) {
    x <- paste0("### F", fid, " ", dim, "D \n")
    cat(x)
    cat('\n')
    opts_current$set(label =  paste0("Statistics_Param_F", fid, " ", dim, "D"))
    dsl_sub <- subset(dsl, DIM == dim && funcId == fid && algId %in% input$`Report.Param.Statistics-Alg`)
    name <- get_data_id(dsl_sub)
    fvs <- get_funvals(dsl_sub)
    tryCatch(
      {
      fv_seq <- seq_FV(fvs, as.numeric(REG$PAR.Summary.Min[[name]]), as.numeric(REG$PAR.Summary.Max[[name]]), 
                       by = as.numeric(REG$PAR.Summary.Step[[name]])) 
    },
    error = function(e) { fv_seq <- seq_FV(fvs, length.out = 10) }
    )
    p <- kable(get_PAR_summary(dsl_sub, fv_seq), 
               caption = paste0("Statistics of parameters of selected algorithms on F", fid, " in ", dim, "D"), 
               format = 'latex') %>% 
      kable_styling(latex_options=c("scale_down", "hold_position"))
    print(p)
    cat('\n')
  }
}
```

```{r param_plot, eval=input$Report.Param.Plot, results='hide', include=F}
out <- NULL
fig_names <- NULL
for (dim in input$`Report.Param.Plot-DIM` ) {
  for (fid in input$`Report.Param.Plot-FuncId` ) {
    name_figure <- paste0('Param_Plot', fid, '-', dim, 'D')
    algs = input$`Report.Param.Plot-Alg`
    env = new.env()
    out <- invisible(c(out,knit_child(text = knit_expand(file = "Param_plot.Rmd"))))
    fig_names <- c(fig_names, name_figure)
  }
}
```

`r if(input$Report.Param.Plot) paste(out, collapse='\n')`

```{r param_plot_text, eval=input$Report.Param.Plot, results='asis'}
asis_output(paste0("Filler text. This can be seen in Figures ", sapply(fig_names, function(f_name) { paste0("\\ref{fig:", f_name, "},")})))
```

# References
