<!--
%\VignetteEngine{knitr::knitr}
%\VignetteIndexEntry{Search}
%\VignetteEncoding{UTF-8}
-->

```{r, eval=TRUE, echo=FALSE}
knitr::opts_chunk$set(
  comment = "#>",
  warning = FALSE,
  message = FALSE
)
```

elastic searching
======

## Load elastic

```{r}
library("elastic")
```

## The Search function

The main interface to searching documents in your Elasticsearch store is the function `Search()`. I nearly always develop R software using all lowercase, but R has a function called `search()`, and I wanted to avoid collision with that function.

`Search()` is an interface to both the HTTP search API (in which queries are passed in the URI of the request, meaning queries have to be relatively simple), as well as the POST API, or the Query DSL, in which queries are passed in the body of the request (so can be much more complex).

There are a huge amount of ways you can search Elasticsearch documents - this tutorial covers some of them, and highlights the ways in which you interact with the R outputs.

```{r}
x <- connect()
```

### Search an index

```{r}
out <- Search(x, index="shakespeare")
out$hits$total
```

```{r}
out$hits$hits[[1]]
```

### Search an index by type

```{r}
Search(x, index = "shakespeare", type = "line")$hits$hits[[1]]
```

### Return certain fields

```{r}
Search(x, index = "shakespeare", body = '{
  "_source": ["play_name", "speaker"]
}')$hits$hits[[1]]
```


### Paging

```{r}
Search(x, index="shakespeare", size=1, from=1)$hits
```

### Queries

Using the `q` parameter you can pass in a query, which gets passed in the URI of the query. This type of query is less powerful than the below query passed in the body of the request, using the `body` parameter.

```{r}
Search(x, index="shakespeare", type="line", q="speaker:KING HENRY IV")$hits$total
```

#### More complex queries

Here, query for values from 10 to 20 in the field `line_id`

```{r}
Search(x, index="shakespeare", q="line_id:[10 TO 20]")$hits$total
```

### Get version number for each document

Version number usually is not returned.

```{r}
sapply(Search(x, index="shakespeare", version=TRUE, size=2)$hits$hits, "[[", "_version")
```

### Get raw data

```{r}
Search(x, index="shakespeare", type="line", raw=TRUE)
```

### Curl debugging

Common options are `verbose=TRUE`, `timeout_ms=1`, `followlocation=TRUE`.

```{r}
out <- Search(x, index="shakespeare", type="line", verbose = TRUE)
```

### Query DSL searches - queries sent in the body of the request

Pass in as an R list

```{r}
mapping_create(x, "shakespeare", "line", update_all_types = TRUE, body = '{
   "properties": {
     "text_entry": {
       "type":     "text",
       "fielddata": true
    }
  }
}')
aggs <- list(aggs = list(stats = list(terms = list(field = "text_entry"))))
Search(x, index="shakespeare", body=aggs)$hits$hits[[1]]
```

Or pass in as json query with newlines, easy to read

```{r}
aggs <- '{
    "aggs": {
        "stats" : {
            "terms" : {
                "field" : "text_entry"
            }
        }
    }
}'
Search(x, index="shakespeare", body=aggs)$hits$hits[[1]]
```

Or pass in collapsed json string

```{r}
aggs <- '{"aggs":{"stats":{"terms":{"field":"text_entry"}}}}'
Search(x, index="shakespeare", body=aggs)$hits$hits[[1]]
```

### Aggregations

Histograms

```{r}
aggs <- '{
    "aggs": {
        "latbuckets" : {
           "histogram" : {
               "field" : "decimalLatitude",
               "interval" : 5
           }
        }
    }
}'
Search(x, index="gbif", body=aggs, size=0)$aggregations$latbuckets$buckets[1:3]
```

### A bool query

```{r}
mmatch <- '{
 "query": {
   "bool" : {
     "must_not" : {
       "range" : {
         "speech_number" : {
           "from" : 1, "to": 5
}}}}}}'
sapply(Search(x, index="shakespeare", body=mmatch)$hits$hits, function(x) x$`_source`$speech_number)
```

### Fuzzy query

Fuzzy query on numerics

```{r}
fuzzy <- list(query = list(fuzzy = list(text_entry = "arms")))
Search(x, index="shakespeare", body = fuzzy)$hits$total
```

```{r}
fuzzy <- list(query = list(fuzzy = list(text_entry = list(value = "arms", fuzziness = 4))))
Search(x, index="shakespeare", body=fuzzy)$hits$total
```

### Range query

With numeric

```{r}
body <- list(query=list(range=list(decimalLongitude=list(gte=1, lte=3))))
Search(x, 'gbif', body=body)$hits$total
```

```{r}
body <- list(query=list(range=list(decimalLongitude=list(gte=2.9, lte=10))))
Search(x, 'gbif', body=body)$hits$total
```

With dates

```{r}
body <- list(query=list(range=list(eventDate=list(gte="2012-01-01", lte="now"))))
Search(x, 'gbif', body=body)$hits$total
```

```{r}
body <- list(query=list(range=list(eventDate=list(gte="2014-01-01", lte="now"))))
Search(x, 'gbif', body=body)$hits$total
```

### More-like-this query (more_like_this can be shortened to mlt)

```{r}
body <- '{
 "query": {
   "more_like_this": {
     "fields": ["abstract","title"],
     "like": "and then",
     "min_term_freq": 1,
     "max_query_terms": 12
   }
 }
}'
Search(x, 'plos', body=body)$hits$total
```

```{r}
body <- '{
 "query": {
   "more_like_this": {
     "fields": ["abstract","title"],
     "like": "cell",
     "min_term_freq": 1,
     "max_query_terms": 12
   }
 }
}'
Search(x, 'plos', body=body)$hits$total
```


### Highlighting

```{r}
body <- '{
 "query": {
   "query_string": {
     "query" : "cell"
   }
 },
 "highlight": {
   "fields": {
     "title": {"number_of_fragments": 2}
   }
 }
}'
out <- Search(x, 'plos', 'article', body=body)
out$hits$total
```

```{r}
sapply(out$hits$hits, function(x) x$highlight$title[[1]])[8:10]
```

### Scrolling search - instead of paging

```{r}
Search(x, 'shakespeare', q="a*")$hits$total
res <- Search(x, index = 'shakespeare', q="a*", time_scroll = "1m")
length(scroll(x, res$`_scroll_id`, time_scroll = "1m")$hits$hits)
```

```{r}
res <- Search(x, index = 'shakespeare', q = "a*", time_scroll = "5m")
out <- res$hits$hits
hits <- 1
while (hits != 0) {
  res <- scroll(x, res$`_scroll_id`)
  hits <- length(res$hits$hits)
  if (hits > 0)
    out <- c(out, res$hits$hits)
}
length(out)
res$hits$total
```

Woohoo! Collected all `r length(out)` documents in very little time.
