From 42975da2185c4669f7db4704c2a77496f2740f20 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Sat, 10 Oct 2015 11:03:55 +0200 Subject: [PATCH 01/30] Add HTTP Basic Auth option; fix startup healthcheck This commit backports the `SetBasicAuth` option already available in elastic.v3 into elastic.v2. You can now use `SetBasicAuth("username", "password")` as an option when setting up a new client (closes #146). This commit also fixes an issue with the startup healthcheck. We now use the `http.Client` specified via options in `startupHealthcheck` instead of setting up our own. The latter would go wrong if the `http.Client` specified when creating the client is special, e.g. when it uses HTTP Basic Auth (see #145). --- client.go | 30 +++++++++++++++++++++++++++--- client_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/client.go b/client.go index 8e899cd45..13d4270ec 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.12" + Version = "2.0.13" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. @@ -118,6 +118,9 @@ type Client struct { snifferInterval time.Duration // interval between sniffing snifferStop chan bool // notify sniffer to stop, and notify back decoder Decoder // used to decode data sent from Elasticsearch + basicAuth bool // indicates whether to send HTTP Basic Auth credentials + basicAuthUsername string // username for HTTP Basic Auth + basicAuthPassword string // password for HTTP Basic Auth } // NewClient creates a new client to work with Elasticsearch. @@ -129,7 +132,8 @@ type Client struct { // // client, err := elastic.NewClient( // elastic.SetURL("http://localhost:9200", "http://localhost:9201"), -// elastic.SetMaxRetries(10)) +// elastic.SetMaxRetries(10), +// elastic.SetBasicAuth("user", "secret")) // // If no URL is configured, Elastic uses DefaultURL by default. // @@ -243,6 +247,17 @@ func SetHttpClient(httpClient *http.Client) ClientOptionFunc { } } +// SetBasicAuth can be used to specify the HTTP Basic Auth credentials to +// use when making HTTP requests to Elasticsearch. +func SetBasicAuth(username, password string) ClientOptionFunc { + return func(c *Client) error { + c.basicAuthUsername = username + c.basicAuthPassword = password + c.basicAuth = c.basicAuthUsername != "" || c.basicAuthPassword != "" + return nil + } +} + // SetURL defines the URL endpoints of the Elasticsearch nodes. Notice that // when sniffing is enabled, these URLs are used to initially sniff the // cluster on startup. @@ -752,7 +767,12 @@ func (c *Client) startupHealthcheck(timeout time.Duration) error { // If we don't get a connection after "timeout", we bail. start := time.Now() for { - cl := &http.Client{Timeout: timeout} + // Make a copy of the HTTP client provided via options to respect + // settings like Basic Auth or a user-specified http.Transport. + cl := new(http.Client) + *cl = *c.c + cl.Timeout = timeout + for _, url := range urls { res, err := cl.Head(url) if err == nil && res != nil && res.StatusCode >= 200 && res.StatusCode < 300 { @@ -864,6 +884,10 @@ func (c *Client) PerformRequest(method, path string, params url.Values, body int return nil, err } + if c.basicAuth { + ((*http.Request)(req)).SetBasicAuth(c.basicAuthUsername, c.basicAuthPassword) + } + // Set body if body != nil { switch b := body.(type) { diff --git a/client_test.go b/client_test.go index 705a48223..0c39c8c25 100644 --- a/client_test.go +++ b/client_test.go @@ -55,6 +55,15 @@ func TestClientDefaults(t *testing.T) { if client.snifferInterval != DefaultSnifferInterval { t.Errorf("expected sniffer interval = %v, got: %v", DefaultSnifferInterval, client.snifferInterval) } + if client.basicAuth != false { + t.Errorf("expected no basic auth; got: %v", client.basicAuth) + } + if client.basicAuthUsername != "" { + t.Errorf("expected no basic auth username; got: %q", client.basicAuthUsername) + } + if client.basicAuthPassword != "" { + t.Errorf("expected no basic auth password; got: %q", client.basicAuthUsername) + } } func TestClientWithoutURL(t *testing.T) { @@ -109,6 +118,22 @@ func TestClientWithMultipleURLs(t *testing.T) { } } +func TestClientWithBasicAuth(t *testing.T) { + client, err := NewClient(SetBasicAuth("user", "secret")) + if err != nil { + t.Fatal(err) + } + if client.basicAuth != true { + t.Errorf("expected basic auth; got: %v", client.basicAuth) + } + if got, want := client.basicAuthUsername, "user"; got != want { + t.Errorf("expected basic auth username %q; got: %q", want, got) + } + if got, want := client.basicAuthPassword, "secret"; got != want { + t.Errorf("expected basic auth password %q; got: %q", want, got) + } +} + func TestClientSniffSuccess(t *testing.T) { client, err := NewClient(SetURL("http://localhost:19200", "http://localhost:9200")) if err != nil { @@ -589,6 +614,10 @@ func (tr *failingTransport) RoundTrip(r *http.Request) (*http.Response, error) { return http.DefaultTransport.RoundTrip(r) } +// CancelRequest is required in a http.Transport to support timeouts. +func (tr *failingTransport) CancelRequest(req *http.Request) { +} + func TestPerformRequestWithMaxRetries(t *testing.T) { var numFailedReqs int fail := func(r *http.Request) (*http.Response, error) { From a012ec77008d11bd94d744fffbfa3ed77c614675 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Thu, 15 Oct 2015 10:27:38 +0200 Subject: [PATCH 02/30] Provide a lightweight client for use in App Engine In the normal use case, you create one Client in your application on startup. You then share this Client throughout your application, elastic will automatically mark nodes as healthy or dead etc. However, in restricted environments like App Engine, one cannot create such a shared client. You must create a new client for every request. So we cannot afford to keep Goroutines around and must make the creation of a client as efficient as possible. So if you now create a new client with the sniffer and healthchecker disabled (`SetSniff(false)` and `SetHealthcheck(false)`), the `NewClient` function will skip most of the sanity checks it does on normal startup. It will also not start any Goroutines. So to use elastic with restricted environments like App Engine, be sure to disable sniffer and healthchecker via the options `SetSniff(false)` and `SetHealthcheck(false)`. --- client.go | 44 ++++++++++++++++++++++++++++++-------------- client_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/client.go b/client.go index 13d4270ec..4a0841d67 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.13" + Version = "2.0.14" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. @@ -201,8 +201,10 @@ func NewClient(options ...ClientOptionFunc) (*Client, error) { c.urls = canonicalize(c.urls...) // Check if we can make a request to any of the specified URLs - if err := c.startupHealthcheck(c.healthcheckTimeoutStartup); err != nil { - return nil, err + if c.healthcheckEnabled { + if err := c.startupHealthcheck(c.healthcheckTimeoutStartup); err != nil { + return nil, err + } } if c.snifferEnabled { @@ -217,15 +219,21 @@ func NewClient(options ...ClientOptionFunc) (*Client, error) { } } - // Perform an initial health check and - // ensure that we have at least one connection available - c.healthcheck(c.healthcheckTimeoutStartup, true) + if c.healthcheckEnabled { + // Perform an initial health check + c.healthcheck(c.healthcheckTimeoutStartup, true) + } + // Ensure that we have at least one connection available if err := c.mustActiveConn(); err != nil { return nil, err } - go c.sniffer() // periodically update cluster information - go c.healthchecker() // start goroutine periodically ping all nodes of the cluster + if c.snifferEnabled { + go c.sniffer() // periodically update cluster information + } + if c.healthcheckEnabled { + go c.healthchecker() // start goroutine periodically ping all nodes of the cluster + } c.mu.Lock() c.running = true @@ -450,8 +458,12 @@ func (c *Client) Start() { } c.mu.RUnlock() - go c.sniffer() - go c.healthchecker() + if c.snifferEnabled { + go c.sniffer() + } + if c.healthcheckEnabled { + go c.healthchecker() + } c.mu.Lock() c.running = true @@ -473,11 +485,15 @@ func (c *Client) Stop() { } c.mu.RUnlock() - c.healthcheckStop <- true - <-c.healthcheckStop + if c.healthcheckEnabled { + c.healthcheckStop <- true + <-c.healthcheckStop + } - c.snifferStop <- true - <-c.snifferStop + if c.snifferEnabled { + c.snifferStop <- true + <-c.snifferStop + } c.mu.Lock() c.running = false diff --git a/client_test.go b/client_test.go index 0c39c8c25..875be5097 100644 --- a/client_test.go +++ b/client_test.go @@ -237,6 +237,46 @@ func TestClientStartAndStop(t *testing.T) { } } +func TestClientStartAndStopWithSnifferAndHealthchecksDisabled(t *testing.T) { + client, err := NewClient(SetSniff(false), SetHealthcheck(false)) + if err != nil { + t.Fatal(err) + } + + running := client.IsRunning() + if !running { + t.Fatalf("expected background processes to run; got: %v", running) + } + + // Stop + client.Stop() + running = client.IsRunning() + if running { + t.Fatalf("expected background processes to be stopped; got: %v", running) + } + + // Stop again => no-op + client.Stop() + running = client.IsRunning() + if running { + t.Fatalf("expected background processes to be stopped; got: %v", running) + } + + // Start + client.Start() + running = client.IsRunning() + if !running { + t.Fatalf("expected background processes to run; got: %v", running) + } + + // Start again => no-op + client.Start() + running = client.IsRunning() + if !running { + t.Fatalf("expected background processes to run; got: %v", running) + } +} + // -- Sniffing -- func TestClientSniffNode(t *testing.T) { From 1de06a68b064a4185f388a50a5f4371d855c2229 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Thu, 15 Oct 2015 10:34:17 +0200 Subject: [PATCH 03/30] Provide a lightweight client for use in App Engine In the normal use case, you create one Client in your application on startup. You then share this Client throughout your application, elastic will automatically mark nodes as healthy or dead etc. However, in restricted environments like App Engine, one cannot create such a shared client. You must create a new client for every request. So we cannot afford to keep Goroutines around and must make the creation of a client as efficient as possible. So if you now create a new client with the sniffer and healthchecker disabled (`SetSniff(false)` and `SetHealthcheck(false)`), the `NewClient` function will skip most of the sanity checks it does on normal startup. It will also not start any Goroutines. So to use elastic with restricted environments like App Engine, be sure to disable sniffer and healthchecker via the options `SetSniff(false)` and `SetHealthcheck(false)`. --- client.go | 44 ++++++++++++++++++++++++++++++-------------- client_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/client.go b/client.go index 13d4270ec..4a0841d67 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.13" + Version = "2.0.14" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. @@ -201,8 +201,10 @@ func NewClient(options ...ClientOptionFunc) (*Client, error) { c.urls = canonicalize(c.urls...) // Check if we can make a request to any of the specified URLs - if err := c.startupHealthcheck(c.healthcheckTimeoutStartup); err != nil { - return nil, err + if c.healthcheckEnabled { + if err := c.startupHealthcheck(c.healthcheckTimeoutStartup); err != nil { + return nil, err + } } if c.snifferEnabled { @@ -217,15 +219,21 @@ func NewClient(options ...ClientOptionFunc) (*Client, error) { } } - // Perform an initial health check and - // ensure that we have at least one connection available - c.healthcheck(c.healthcheckTimeoutStartup, true) + if c.healthcheckEnabled { + // Perform an initial health check + c.healthcheck(c.healthcheckTimeoutStartup, true) + } + // Ensure that we have at least one connection available if err := c.mustActiveConn(); err != nil { return nil, err } - go c.sniffer() // periodically update cluster information - go c.healthchecker() // start goroutine periodically ping all nodes of the cluster + if c.snifferEnabled { + go c.sniffer() // periodically update cluster information + } + if c.healthcheckEnabled { + go c.healthchecker() // start goroutine periodically ping all nodes of the cluster + } c.mu.Lock() c.running = true @@ -450,8 +458,12 @@ func (c *Client) Start() { } c.mu.RUnlock() - go c.sniffer() - go c.healthchecker() + if c.snifferEnabled { + go c.sniffer() + } + if c.healthcheckEnabled { + go c.healthchecker() + } c.mu.Lock() c.running = true @@ -473,11 +485,15 @@ func (c *Client) Stop() { } c.mu.RUnlock() - c.healthcheckStop <- true - <-c.healthcheckStop + if c.healthcheckEnabled { + c.healthcheckStop <- true + <-c.healthcheckStop + } - c.snifferStop <- true - <-c.snifferStop + if c.snifferEnabled { + c.snifferStop <- true + <-c.snifferStop + } c.mu.Lock() c.running = false diff --git a/client_test.go b/client_test.go index 0c39c8c25..875be5097 100644 --- a/client_test.go +++ b/client_test.go @@ -237,6 +237,46 @@ func TestClientStartAndStop(t *testing.T) { } } +func TestClientStartAndStopWithSnifferAndHealthchecksDisabled(t *testing.T) { + client, err := NewClient(SetSniff(false), SetHealthcheck(false)) + if err != nil { + t.Fatal(err) + } + + running := client.IsRunning() + if !running { + t.Fatalf("expected background processes to run; got: %v", running) + } + + // Stop + client.Stop() + running = client.IsRunning() + if running { + t.Fatalf("expected background processes to be stopped; got: %v", running) + } + + // Stop again => no-op + client.Stop() + running = client.IsRunning() + if running { + t.Fatalf("expected background processes to be stopped; got: %v", running) + } + + // Start + client.Start() + running = client.IsRunning() + if !running { + t.Fatalf("expected background processes to run; got: %v", running) + } + + // Start again => no-op + client.Start() + running = client.IsRunning() + if !running { + t.Fatalf("expected background processes to run; got: %v", running) + } +} + // -- Sniffing -- func TestClientSniffNode(t *testing.T) { From 84daa6c3319af439e49f119898a0052e2e9f7eab Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Fri, 23 Oct 2015 13:01:00 +0200 Subject: [PATCH 04/30] Test against 1.7.3; drop Travis test against 1.6 --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 881994539..0de95148f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,7 @@ go: env: matrix: - - ES_VERSION=1.6.2 - - ES_VERSION=1.7.2 + - ES_VERSION=1.7.3 before_script: - mkdir ${HOME}/elasticsearch From d83073710ba60ed5a65db25bbae541a2cfe3aaae Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Wed, 28 Oct 2015 21:11:07 +0100 Subject: [PATCH 05/30] Update README with release of 3.0 As Elasticsearch 2.0.0 is released, we must update the README. --- .travis.yml | 2 +- README.md | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0de95148f..16e372b56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ sudo: false language: go go: - - 1.5 + - 1.5.1 - tip env: diff --git a/README.md b/README.md index 2b7579a54..28ab44b2c 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ Elastic is an [Elasticsearch](http://www.elasticsearch.org/) client for the [Go](http://www.golang.org/) programming language. -[![Build Status](https://travis-ci.org/olivere/elastic.svg?branch=master)](https://travis-ci.org/olivere/elastic) -[![Godoc](http://img.shields.io/badge/godoc-reference-blue.svg?style=flat)](https://godoc.org/github.com/olivere/elastic) +[![Build Status](https://travis-ci.org/olivere/elastic.svg?branch=release-branch.v3)](https://travis-ci.org/olivere/elastic) +[![Godoc](http://img.shields.io/badge/godoc-reference-blue.svg?style=flat)](https://godoc.org/gopkg.in/olivere/elastic.v2) [![license](http://img.shields.io/badge/license-MIT-red.svg?style=flat)](https://raw.githubusercontent.com/olivere/elastic/master/LICENSE) See the [wiki](https://github.com/olivere/elastic/wiki) for additional information about Elastic. @@ -18,13 +18,13 @@ Here's the version matrix: Elasticsearch version | Elastic version -| Package URL ----------------------|------------------|------------ -2.x | 3.0 **beta** | [`gopkg.in/olivere/elastic.v3-unstable`](https://gopkg.in/olivere/elastic.v3-unstable) ([source](https://github.com/olivere/elastic/tree/release-branch.v3) [doc](http://godoc.org/gopkg.in/olivere/elastic.v3-unstable)) +2.x | 3.0 | [`gopkg.in/olivere/elastic.v3`](https://gopkg.in/olivere/elastic.v3) ([source](https://github.com/olivere/elastic/tree/release-branch.v3) [doc](http://godoc.org/gopkg.in/olivere/elastic.v3)) 1.x | 2.0 | [`gopkg.in/olivere/elastic.v2`](https://gopkg.in/olivere/elastic.v2) ([source](https://github.com/olivere/elastic/tree/release-branch.v2) [doc](http://godoc.org/gopkg.in/olivere/elastic.v2)) 0.9-1.3 | 1.0 | [`gopkg.in/olivere/elastic.v1`](https://gopkg.in/olivere/elastic.v1) ([source](https://github.com/olivere/elastic/tree/release-branch.v1) [doc](http://godoc.org/gopkg.in/olivere/elastic.v1)) **Example:** -You have Elasticsearch 1.6.0 installed and want to use Elastic. As listed above, you should use Elastic 2.0. So you first install Elastic 2.0. +You have Elasticsearch 1.7.3 installed and want to use Elastic. As listed above, you should use Elastic 2.0. So you first install Elastic 2.0. ```sh $ go get gopkg.in/olivere/elastic.v2 @@ -38,9 +38,9 @@ import "gopkg.in/olivere/elastic.v2" ### Elastic 3.0 -Elastic 3.0 targets Elasticsearch 2.x and is currently under [active development](https://github.com/olivere/elastic/tree/release-branch.v3). It is not published to gokpg yet. +Elastic 3.0 targets Elasticsearch 2.0 and later. Elasticsearch 2.0.0 was [released on 28th October 2015](https://www.elastic.co/blog/elasticsearch-2-0-0-released). -There are a lot of [breaking changes in Elasticsearch 2.0](https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-2.0.html) and we will use this as an opportunity to [clean up and refactor Elastic as well](https://github.com/olivere/elastic/blob/release-branch.v3/CHANGELOG-3.0.md). +Notice that there are a lot of [breaking changes in Elasticsearch 2.0](https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking-changes-2.0.html) and we used this as an opportunity to [clean up and refactor Elastic as well](https://github.com/olivere/elastic/blob/release-branch.v3/CHANGELOG-3.0.md). ### Elastic 2.0 @@ -69,8 +69,7 @@ More often than not it's renaming APIs and adding/removing features so that we are in sync with the Elasticsearch API. Elastic has been used in production with the following Elasticsearch versions: -0.90, 1.0, 1.1, 1.2, 1.3, 1.4, and 1.5. -Furthermore, we use [Travis CI](https://travis-ci.org/) +0.90, 1.0-1.7. Furthermore, we use [Travis CI](https://travis-ci.org/) to test Elastic with the most recent versions of Elasticsearch and Go. See the [.travis.yml](https://github.com/olivere/elastic/blob/master/.travis.yml) file for the exact matrix and [Travis](https://travis-ci.org/olivere/elastic) From d9a1579285e3ce7a145990a47e6d68ef86334d2f Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Wed, 28 Oct 2015 21:15:55 +0100 Subject: [PATCH 06/30] Update README as version 3.0.0 is available As Elasticsearch 2.0.0 is released, we update the README accordingly. --- .travis.yml | 2 +- README.md | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0de95148f..16e372b56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ sudo: false language: go go: - - 1.5 + - 1.5.1 - tip env: diff --git a/README.md b/README.md index 2365b8ca9..f9784252d 100644 --- a/README.md +++ b/README.md @@ -18,13 +18,13 @@ Here's the version matrix: Elasticsearch version | Elastic version -| Package URL ----------------------|------------------|------------ -2.x | 3.0 **beta** | [`gopkg.in/olivere/elastic.v3-unstable`](https://gopkg.in/olivere/elastic.v3-unstable) ([source](https://github.com/olivere/elastic/tree/release-branch.v3) [doc](http://godoc.org/gopkg.in/olivere/elastic.v3-unstable)) +2.x | 3.0 | [`gopkg.in/olivere/elastic.v3`](https://gopkg.in/olivere/elastic.v3) ([source](https://github.com/olivere/elastic/tree/release-branch.v3) [doc](http://godoc.org/gopkg.in/olivere/elastic.v3)) 1.x | 2.0 | [`gopkg.in/olivere/elastic.v2`](https://gopkg.in/olivere/elastic.v2) ([source](https://github.com/olivere/elastic/tree/release-branch.v2) [doc](http://godoc.org/gopkg.in/olivere/elastic.v2)) 0.9-1.3 | 1.0 | [`gopkg.in/olivere/elastic.v1`](https://gopkg.in/olivere/elastic.v1) ([source](https://github.com/olivere/elastic/tree/release-branch.v1) [doc](http://godoc.org/gopkg.in/olivere/elastic.v1)) **Example:** -You have Elasticsearch 1.7.1 installed and want to use Elastic. As listed above, you should use Elastic 2.0. So you first install Elastic 2.0. +You have Elasticsearch 1.7.3 installed and want to use Elastic. As listed above, you should use Elastic 2.0. So you first install Elastic 2.0. ```sh $ go get gopkg.in/olivere/elastic.v2 @@ -38,9 +38,9 @@ import "gopkg.in/olivere/elastic.v2" ### Elastic 3.0 -Elastic 3.0 targets Elasticsearch 2.x and is currently under [active development](https://github.com/olivere/elastic/tree/release-branch.v3). It is not published to gokpg yet. +Elastic 3.0 targets Elasticsearch 2.0 and later. Elasticsearch 2.0.0 was [released on 28th October 2015](https://www.elastic.co/blog/elasticsearch-2-0-0-released). -There are a lot of [breaking changes in Elasticsearch 2.0](https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-2.0.html) and we will use this as an opportunity to [clean up and refactor Elastic as well](https://github.com/olivere/elastic/blob/release-branch.v3/CHANGELOG-3.0.md). +Notice that there are a lot of [breaking changes in Elasticsearch 2.0](https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking-changes-2.0.html) and we used this as an opportunity to [clean up and refactor Elastic as well](https://github.com/olivere/elastic/blob/release-branch.v3/CHANGELOG-3.0.md). ### Elastic 2.0 @@ -69,8 +69,7 @@ More often than not it's renaming APIs and adding/removing features so that we are in sync with the Elasticsearch API. Elastic has been used in production with the following Elasticsearch versions: -0.90, 1.0, 1.1, 1.2, 1.3, 1.4, and 1.5. -Furthermore, we use [Travis CI](https://travis-ci.org/) +0.90, 1.0-1.7. Furthermore, we use [Travis CI](https://travis-ci.org/) to test Elastic with the most recent versions of Elasticsearch and Go. See the [.travis.yml](https://github.com/olivere/elastic/blob/master/.travis.yml) file for the exact matrix and [Travis](https://travis-ci.org/olivere/elastic) From 0c2b945e30b682b0070c5b3bc0cb4a907b763bd6 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Thu, 29 Oct 2015 16:07:32 +0100 Subject: [PATCH 07/30] Add context queries to SuggestField This is a backport of [this commit](https://github.com/olivere/elastic/commit/e310ae73e331c7ee8636489e1c718e9201a00846) in elastic.v3. --- CONTRIBUTORS | 1 + client.go | 2 +- suggest_field.go | 26 ++++++++++++++++++++++---- suggest_field_test.go | 30 ++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 suggest_field_test.go diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 268b4ac3b..d9026e93f 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -9,6 +9,7 @@ Alexey Sharov [@nizsheanez](https://github.com/nizsheanez) Conrad Pankoff [@deoxxa](https://github.com/deoxxa) Corey Scott [@corsc](https://github.com/corsc) +Daniel Heckrath [@DanielHeckrath](https://github.com/DanielHeckrath) Gerhard Häring [@ghaering](https://github.com/ghaering) Guilherme Silveira [@guilherme-santos](https://github.com/guilherme-santos) Jack Lindamood [@cep21](https://github.com/cep21) diff --git a/client.go b/client.go index 4a0841d67..0b13b7565 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.14" + Version = "2.0.15" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. diff --git a/suggest_field.go b/suggest_field.go index 60f94818f..a6e0ff6a2 100644 --- a/suggest_field.go +++ b/suggest_field.go @@ -12,10 +12,11 @@ import ( // at index time. For a detailed example, see e.g. // http://www.elasticsearch.org/blog/you-complete-me/. type SuggestField struct { - inputs []string - output *string - payload interface{} - weight int + inputs []string + output *string + payload interface{} + weight int + contextQueries []SuggesterContextQuery } func NewSuggestField() *SuggestField { @@ -45,6 +46,11 @@ func (f *SuggestField) Weight(weight int) *SuggestField { return f } +func (f *SuggestField) ContextQuery(queries ...SuggesterContextQuery) *SuggestField { + f.contextQueries = append(f.contextQueries, queries...) + return f +} + // MarshalJSON encodes SuggestField into JSON. func (f *SuggestField) MarshalJSON() ([]byte, error) { source := make(map[string]interface{}) @@ -70,5 +76,17 @@ func (f *SuggestField) MarshalJSON() ([]byte, error) { source["weight"] = f.weight } + switch len(f.contextQueries) { + case 0: + case 1: + source["context"] = f.contextQueries[0].Source() + default: + var ctxq []interface{} + for _, query := range f.contextQueries { + ctxq = append(ctxq, query.Source()) + } + source["context"] = ctxq + } + return json.Marshal(source) } diff --git a/suggest_field_test.go b/suggest_field_test.go new file mode 100644 index 000000000..b01cf0af0 --- /dev/null +++ b/suggest_field_test.go @@ -0,0 +1,30 @@ +// Copyright 2012-2015 Oliver Eilhard. All rights reserved. +// Use of this source code is governed by a MIT-license. +// See http://olivere.mit-license.org/license.txt for details. + +package elastic + +import ( + "encoding/json" + "testing" +) + +func TestSuggestField(t *testing.T) { + field := NewSuggestField(). + Input("Welcome to Golang and Elasticsearch.", "Golang and Elasticsearch"). + Output("Golang and Elasticsearch: An introduction."). + Weight(1). + ContextQuery( + NewSuggesterCategoryMapping("color").FieldName("color_field").DefaultValues("red", "green", "blue"), + NewSuggesterGeoMapping("location").Precision("5m").Neighbors(true).DefaultLocations(GeoPointFromLatLon(52.516275, 13.377704)), + ) + data, err := json.Marshal(field) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"context":[{"color":{"default":["red","green","blue"],"path":"color_field","type":"category"}},{"location":{"default":{"lat":52.516275,"lon":13.377704},"neighbors":true,"precision":["5m"],"type":"geo"}}],"input":["Welcome to Golang and Elasticsearch.","Golang and Elasticsearch"],"output":"Golang and Elasticsearch: An introduction.","weight":1}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +} From 1ecca9e9b181d3b0fefdae86a422803879b83610 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Fri, 30 Oct 2015 19:21:02 +0100 Subject: [PATCH 08/30] Handle 404 as an error on failed update This commit changes Update in that a 404 response from ES yields an error from `Update.Do`. See [PR-153](#153). --- client.go | 2 +- errors.go | 12 ++++++++---- update.go | 5 +++++ update_test.go | 31 +++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/client.go b/client.go index 0b13b7565..a1ff425ae 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.15" + Version = "2.0.16" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. diff --git a/errors.go b/errors.go index abbb09c6c..cfddda536 100644 --- a/errors.go +++ b/errors.go @@ -35,18 +35,22 @@ func checkResponse(res *http.Response) error { if err != nil { return fmt.Errorf("elastic: Error %d (%s) when reading body: %v", res.StatusCode, http.StatusText(res.StatusCode), err) } + return createResponseError(res.StatusCode, slurp) +} + +func createResponseError(statusCode int, data []byte) error { errReply := new(Error) - err = json.Unmarshal(slurp, errReply) + err := json.Unmarshal(data, errReply) if err != nil { - return fmt.Errorf("elastic: Error %d (%s)", res.StatusCode, http.StatusText(res.StatusCode)) + return fmt.Errorf("elastic: Error %d (%s)", statusCode, http.StatusText(statusCode)) } if errReply != nil { if errReply.Status == 0 { - errReply.Status = res.StatusCode + errReply.Status = statusCode } return errReply } - return fmt.Errorf("elastic: Error %d (%s)", res.StatusCode, http.StatusText(res.StatusCode)) + return fmt.Errorf("elastic: Error %d (%s)", statusCode, http.StatusText(statusCode)) } type Error struct { diff --git a/update.go b/update.go index d2595a44b..a6b51c183 100644 --- a/update.go +++ b/update.go @@ -7,6 +7,7 @@ package elastic import ( "encoding/json" "fmt" + "net/http" "net/url" "strings" @@ -332,6 +333,10 @@ func (b *UpdateService) Do() (*UpdateResult, error) { if err != nil { return nil, err } + // 404 indicates an error for failed updates + if res.StatusCode == http.StatusNotFound { + return nil, createResponseError(res.StatusCode, res.Body) + } // Return result ret := new(UpdateResult) diff --git a/update_test.go b/update_test.go index eb648eb0c..fdb0d260c 100644 --- a/update_test.go +++ b/update_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "net/url" "testing" + "time" ) func TestUpdateViaScript(t *testing.T) { @@ -311,3 +312,33 @@ func TestUpdateViaScriptIntegration(t *testing.T) { t.Errorf("expected Tweet.Retweets to be %d; got %d", tweet1.Retweets+increment, tweetGot.Retweets) } } + +func TestUpdateReturnsErrorOnFailure(t *testing.T) { + client := setupTestClientAndCreateIndex(t) + + // Travis lags sometimes + if isTravis() { + time.Sleep(2 * time.Second) + } + + // Ensure that no tweet with id #1 exists + exists, err := client.Exists().Index(testIndexName).Type("tweet").Id("1").Do() + if err != nil { + t.Fatal(err) + } + if exists { + t.Fatalf("expected no document; got: %v", exists) + } + + // Update (non-existent) tweet with id #1 + update, err := client.Update(). + Index(testIndexName).Type("tweet").Id("1"). + Doc(map[string]interface{}{"retweets": 42}). + Do() + if err == nil { + t.Fatalf("expected error to be != nil; got: %v", err) + } + if update != nil { + t.Fatalf("expected update to be == nil; got %v", update) + } +} From 8c9c72ef207f43741787de4e1946fa9ab06410e0 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Wed, 4 Nov 2015 09:19:19 +0100 Subject: [PATCH 09/30] Add missing properties to Count API The Count service was missing a few properties from the REST API. Closes #158. --- client.go | 2 +- count.go | 298 +++++++++++++++++++++++++++++++++++++++----------- count_test.go | 41 +++++++ 3 files changed, 278 insertions(+), 63 deletions(-) diff --git a/client.go b/client.go index a1ff425ae..d5817864c 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.16" + Version = "2.0.17" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. diff --git a/count.go b/count.go index bb4c0ac26..f5097bc9d 100644 --- a/count.go +++ b/count.go @@ -17,130 +17,298 @@ import ( // number of documents in an index. Use SearchService with // a SearchType of count for counting with queries etc. type CountService struct { - client *Client - indices []string - types []string - query Query - pretty bool -} - -// CountResult is the result returned from using the Count API -// (http://www.elasticsearch.org/guide/reference/api/count/) -type CountResult struct { - Count int64 `json:"count"` - Shards shardsInfo `json:"_shards,omitempty"` + client *Client + pretty bool + index []string + typ []string + allowNoIndices *bool + analyzeWildcard *bool + analyzer string + defaultOperator string + df string + expandWildcards string + ignoreUnavailable *bool + lenient *bool + lowercaseExpandedTerms *bool + minScore interface{} + preference string + q string + query Query + routing string + bodyJson interface{} + bodyString string } +// NewCountService creates a new CountService. func NewCountService(client *Client) *CountService { - builder := &CountService{ + return &CountService{ client: client, + index: make([]string, 0), + typ: make([]string, 0), } - return builder } +// Index sets the name of the index to use to restrict the results. func (s *CountService) Index(index string) *CountService { - if s.indices == nil { - s.indices = make([]string, 0) + if s.index == nil { + s.index = make([]string, 0) } - s.indices = append(s.indices, index) + s.index = append(s.index, index) return s } +// Indices sets the names of the indices to restrict the results. func (s *CountService) Indices(indices ...string) *CountService { - if s.indices == nil { - s.indices = make([]string, 0) + if s.index == nil { + s.index = make([]string, 0) } - s.indices = append(s.indices, indices...) + s.index = append(s.index, indices...) return s } +// Type sets the type to use to restrict the results. func (s *CountService) Type(typ string) *CountService { - if s.types == nil { - s.types = make([]string, 0) + if s.typ == nil { + s.typ = make([]string, 0) } - s.types = append(s.types, typ) + s.typ = append(s.typ, typ) return s } +// Types sets the types to use to restrict the results. func (s *CountService) Types(types ...string) *CountService { - if s.types == nil { - s.types = make([]string, 0) + if s.typ == nil { + s.typ = make([]string, 0) } - s.types = append(s.types, types...) + s.typ = append(s.typ, types...) return s } +// AllowNoIndices indicates whether to ignore if a wildcard indices +// expression resolves into no concrete indices. (This includes "_all" string +// or when no indices have been specified). +func (s *CountService) AllowNoIndices(allowNoIndices bool) *CountService { + s.allowNoIndices = &allowNoIndices + return s +} + +// AnalyzeWildcard specifies whether wildcard and prefix queries should be +// analyzed (default: false). +func (s *CountService) AnalyzeWildcard(analyzeWildcard bool) *CountService { + s.analyzeWildcard = &analyzeWildcard + return s +} + +// Analyzer specifies the analyzer to use for the query string. +func (s *CountService) Analyzer(analyzer string) *CountService { + s.analyzer = analyzer + return s +} + +// DefaultOperator specifies the default operator for query string query (AND or OR). +func (s *CountService) DefaultOperator(defaultOperator string) *CountService { + s.defaultOperator = defaultOperator + return s +} + +// Df specifies the field to use as default where no field prefix is given +// in the query string. +func (s *CountService) Df(df string) *CountService { + s.df = df + return s +} + +// ExpandWildcards indicates whether to expand wildcard expression to +// concrete indices that are open, closed or both. +func (s *CountService) ExpandWildcards(expandWildcards string) *CountService { + s.expandWildcards = expandWildcards + return s +} + +// IgnoreUnavailable indicates whether specified concrete indices should be +// ignored when unavailable (missing or closed). +func (s *CountService) IgnoreUnavailable(ignoreUnavailable bool) *CountService { + s.ignoreUnavailable = &ignoreUnavailable + return s +} + +// Lenient specifies whether format-based query failures (such as +// providing text to a numeric field) should be ignored. +func (s *CountService) Lenient(lenient bool) *CountService { + s.lenient = &lenient + return s +} + +// LowercaseExpandedTerms specifies whether query terms should be lowercased. +func (s *CountService) LowercaseExpandedTerms(lowercaseExpandedTerms bool) *CountService { + s.lowercaseExpandedTerms = &lowercaseExpandedTerms + return s +} + +// MinScore indicates to include only documents with a specific `_score` +// value in the result. +func (s *CountService) MinScore(minScore interface{}) *CountService { + s.minScore = minScore + return s +} + +// Preference specifies the node or shard the operation should be +// performed on (default: random). +func (s *CountService) Preference(preference string) *CountService { + s.preference = preference + return s +} + +// Q in the Lucene query string syntax. You can also use Query to pass +// a Query struct. +func (s *CountService) Q(q string) *CountService { + s.q = q + return s +} + +// Query specifies the query to pass. You can also pass a query string with Q. func (s *CountService) Query(query Query) *CountService { s.query = query return s } +// Routing specifies the routing value. +func (s *CountService) Routing(routing string) *CountService { + s.routing = routing + return s +} + +// Pretty indicates that the JSON response be indented and human readable. func (s *CountService) Pretty(pretty bool) *CountService { s.pretty = pretty return s } -func (s *CountService) Do() (int64, error) { - var err error +// BodyJson specifies the query to restrict the results specified with the +// Query DSL (optional). The interface{} will be serialized to a JSON document, +// so use a map[string]interface{}. +func (s *CountService) BodyJson(body interface{}) *CountService { + s.bodyJson = body + return s +} - // Build url - path := "/" +// Body specifies a query to restrict the results specified with +// the Query DSL (optional). +func (s *CountService) BodyString(body string) *CountService { + s.bodyString = body + return s +} + +// buildURL builds the URL for the operation. +func (s *CountService) buildURL() (string, url.Values, error) { + var err error + var path string - // Indices part - indexPart := make([]string, 0) - for _, index := range s.indices { - index, err = uritemplates.Expand("{index}", map[string]string{ - "index": index, + if len(s.index) > 0 && len(s.typ) > 0 { + path, err = uritemplates.Expand("/{index}/{type}/_count", map[string]string{ + "index": strings.Join(s.index, ","), + "type": strings.Join(s.typ, ","), }) - if err != nil { - return 0, err - } - indexPart = append(indexPart, index) + } else if len(s.index) > 0 { + path, err = uritemplates.Expand("/{index}/_count", map[string]string{ + "index": strings.Join(s.index, ","), + }) + } else if len(s.typ) > 0 { + path, err = uritemplates.Expand("/_all/{type}/_count", map[string]string{ + "type": strings.Join(s.typ, ","), + }) + } else { + path = "/_all/_count" } - if len(indexPart) > 0 { - path += strings.Join(indexPart, ",") + if err != nil { + return "", url.Values{}, err } - // Types part - typesPart := make([]string, 0) - for _, typ := range s.types { - typ, err = uritemplates.Expand("{type}", map[string]string{ - "type": typ, - }) - if err != nil { - return 0, err - } - typesPart = append(typesPart, typ) + // Add query string parameters + params := url.Values{} + if s.pretty { + params.Set("pretty", "1") + } + if s.allowNoIndices != nil { + params.Set("allow_no_indices", fmt.Sprintf("%v", *s.allowNoIndices)) + } + if s.analyzeWildcard != nil { + params.Set("analyze_wildcard", fmt.Sprintf("%v", *s.analyzeWildcard)) + } + if s.analyzer != "" { + params.Set("analyzer", s.analyzer) + } + if s.defaultOperator != "" { + params.Set("default_operator", s.defaultOperator) + } + if s.df != "" { + params.Set("df", s.df) } - if len(typesPart) > 0 { - path += "/" + strings.Join(typesPart, ",") + if s.expandWildcards != "" { + params.Set("expand_wildcards", s.expandWildcards) } + if s.ignoreUnavailable != nil { + params.Set("ignore_unavailable", fmt.Sprintf("%v", *s.ignoreUnavailable)) + } + if s.lenient != nil { + params.Set("lenient", fmt.Sprintf("%v", *s.lenient)) + } + if s.lowercaseExpandedTerms != nil { + params.Set("lowercase_expanded_terms", fmt.Sprintf("%v", *s.lowercaseExpandedTerms)) + } + if s.minScore != nil { + params.Set("min_score", fmt.Sprintf("%v", s.minScore)) + } + if s.preference != "" { + params.Set("preference", s.preference) + } + if s.q != "" { + params.Set("q", s.q) + } + if s.routing != "" { + params.Set("routing", s.routing) + } + return path, params, nil +} - // Search - path += "/_count" +// Validate checks if the operation is valid. +func (s *CountService) Validate() error { + return nil +} - // Parameters - params := make(url.Values) - if s.pretty { - params.Set("pretty", fmt.Sprintf("%v", s.pretty)) +// Do executes the operation. +func (s *CountService) Do() (int64, error) { + // Check pre-conditions + if err := s.Validate(); err != nil { + return 0, err + } + + // Get URL for request + path, params, err := s.buildURL() + if err != nil { + return 0, err } - // Set body if there is a query specified + // Setup HTTP request body var body interface{} if s.query != nil { query := make(map[string]interface{}) query["query"] = s.query.Source() body = query + } else if s.bodyJson != nil { + body = s.bodyJson + } else if s.bodyString != "" { + body = s.bodyString } - // Get response + // Get HTTP response res, err := s.client.PerformRequest("POST", path, params, body) if err != nil { return 0, err } // Return result - ret := new(CountResult) + ret := new(CountResponse) if err := json.Unmarshal(res.Body, ret); err != nil { return 0, err } @@ -150,3 +318,9 @@ func (s *CountService) Do() (int64, error) { return int64(0), nil } + +// CountResponse is the response of using the Count API. +type CountResponse struct { + Count int64 `json:"count"` + Shards shardsInfo `json:"_shards,omitempty"` +} diff --git a/count_test.go b/count_test.go index 65bf6fd18..44ecadf22 100644 --- a/count_test.go +++ b/count_test.go @@ -6,6 +6,47 @@ package elastic import "testing" +func TestCountURL(t *testing.T) { + client := setupTestClientAndCreateIndex(t) + + tests := []struct { + Indices []string + Types []string + Expected string + }{ + { + []string{}, + []string{}, + "/_all/_count", + }, + { + []string{}, + []string{"tweet"}, + "/_all/tweet/_count", + }, + { + []string{"twitter-*"}, + []string{"tweet", "follower"}, + "/twitter-%2A/tweet%2Cfollower/_count", + }, + { + []string{"twitter-2014", "twitter-2015"}, + []string{"tweet", "follower"}, + "/twitter-2014%2Ctwitter-2015/tweet%2Cfollower/_count", + }, + } + + for _, test := range tests { + path, _, err := client.Count().Indices(test.Indices...).Types(test.Types...).buildURL() + if err != nil { + t.Fatal(err) + } + if path != test.Expected { + t.Errorf("expected %q; got: %q", test.Expected, path) + } + } +} + func TestCount(t *testing.T) { client := setupTestClientAndCreateIndex(t) From 90bba92211f7a57fcecbdb349e0b4992207be52c Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Mon, 9 Nov 2015 10:46:24 +0100 Subject: [PATCH 10/30] Add test for FSC without excludes option --- fetch_source_context_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/fetch_source_context_test.go b/fetch_source_context_test.go index f329fee8d..ae15a10ec 100644 --- a/fetch_source_context_test.go +++ b/fetch_source_context_test.go @@ -48,6 +48,19 @@ func TestFetchSourceContextFetchSource(t *testing.T) { } } +func TestFetchSourceContextFetchSourceWithIncludesOnly(t *testing.T) { + builder := NewFetchSourceContext(true).Include("a", "b") + data, err := json.Marshal(builder.Source()) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"excludes":[],"includes":["a","b"]}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +} + func TestFetchSourceContextFetchSourceWithIncludesAndExcludes(t *testing.T) { builder := NewFetchSourceContext(true).Include("a", "b").Exclude("c") data, err := json.Marshal(builder.Source()) From 9f744c4a57dd7c636101aef9678f51dddc83b068 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Fri, 13 Nov 2015 19:07:30 +0100 Subject: [PATCH 11/30] Use HTTP basic auth everywhere When using HTTP Basic Authentication e.g. with Shield, not only user requests must be authenticated but also the administrative requests as well (e.g. sniffing and health checks). --- client.go | 33 +++++++++++++++++++++++++++++---- request.go | 4 ++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/client.go b/client.go index d5817864c..7b41b63c4 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.17" + Version = "2.0.18" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. @@ -639,6 +639,12 @@ func (c *Client) sniffNode(url string) []*conn { return nodes } + c.mu.RLock() + if c.basicAuth { + req.SetBasicAuth(c.basicAuthUsername, c.basicAuthPassword) + } + c.mu.RUnlock() + res, err := c.c.Do((*http.Request)(req)) if err != nil { return nodes @@ -742,6 +748,9 @@ func (c *Client) healthcheck(timeout time.Duration, force bool) { c.connsMu.RLock() conns := c.conns + basicAuth := c.basicAuth + basicAuthUsername := c.basicAuthUsername + basicAuthPassword := c.basicAuthPassword c.connsMu.RUnlock() timeoutInMillis := int64(timeout / time.Millisecond) @@ -751,6 +760,9 @@ func (c *Client) healthcheck(timeout time.Duration, force bool) { params.Set("timeout", fmt.Sprintf("%dms", timeoutInMillis)) req, err := NewRequest("HEAD", conn.URL()+"/?"+params.Encode()) if err == nil { + if basicAuth { + req.SetBasicAuth(basicAuthUsername, basicAuthPassword) + } res, err := c.c.Do((*http.Request)(req)) if err == nil { if res.Body != nil { @@ -778,6 +790,9 @@ func (c *Client) healthcheck(timeout time.Duration, force bool) { func (c *Client) startupHealthcheck(timeout time.Duration) error { c.mu.Lock() urls := c.urls + basicAuth := c.basicAuth + basicAuthUsername := c.basicAuthUsername + basicAuthPassword := c.basicAuthPassword c.mu.Unlock() // If we don't get a connection after "timeout", we bail. @@ -790,7 +805,14 @@ func (c *Client) startupHealthcheck(timeout time.Duration) error { cl.Timeout = timeout for _, url := range urls { - res, err := cl.Head(url) + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return err + } + if basicAuth { + req.SetBasicAuth(basicAuthUsername, basicAuthPassword) + } + res, err := cl.Do(req) if err == nil && res != nil && res.StatusCode >= 200 && res.StatusCode < 300 { return nil } @@ -855,6 +877,9 @@ func (c *Client) PerformRequest(method, path string, params url.Values, body int c.mu.RLock() timeout := c.healthcheckTimeout retries := c.maxRetries + basicAuth := c.basicAuth + basicAuthUsername := c.basicAuthUsername + basicAuthPassword := c.basicAuthPassword c.mu.RUnlock() var err error @@ -900,8 +925,8 @@ func (c *Client) PerformRequest(method, path string, params url.Values, body int return nil, err } - if c.basicAuth { - ((*http.Request)(req)).SetBasicAuth(c.basicAuthUsername, c.basicAuthPassword) + if basicAuth { + req.SetBasicAuth(basicAuthUsername, basicAuthPassword) } // Set body diff --git a/request.go b/request.go index eb5a3b13a..75a96dc37 100644 --- a/request.go +++ b/request.go @@ -27,6 +27,10 @@ func NewRequest(method, url string) (*Request, error) { return (*Request)(req), nil } +func (r *Request) SetBasicAuth(username, password string) { + ((*http.Request)(r)).SetBasicAuth(username, password) +} + func (r *Request) SetBodyJson(data interface{}) error { body, err := json.Marshal(data) if err != nil { From eb2ba0d1b663a6c5be75be71c23262f2ff6989be Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Mon, 16 Nov 2015 19:22:26 +0100 Subject: [PATCH 12/30] Add SetSendGetBodyAs option Some environments do not allow to send a HTTP GET request with a body. The official clients have a SetSendGetBodyAs option for this. It changes HTTP GET requests with a body to the method specified by SetSendGetBodyAs. Example: client, err := elastic.NewClient(elastic.SetSendGetBodyAs("POST")) --- CONTRIBUTORS | 1 + client.go | 22 +++++++++++++++++++++- client_test.go | 3 +++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index d9026e93f..edd843b6f 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -7,6 +7,7 @@ # Please keep this list sorted. Alexey Sharov [@nizsheanez](https://github.com/nizsheanez) +Braden Bassingthwaite [@bbassingthwaite-va](https://github.com/bbassingthwaite-va) Conrad Pankoff [@deoxxa](https://github.com/deoxxa) Corey Scott [@corsc](https://github.com/corsc) Daniel Heckrath [@DanielHeckrath](https://github.com/DanielHeckrath) diff --git a/client.go b/client.go index 7b41b63c4..2622e5816 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.18" + Version = "2.0.19" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. @@ -72,6 +72,10 @@ const ( // Elastic will give up and return an error. It is zero by default, so // retry is disabled by default. DefaultMaxRetries = 0 + + // DefaultSendGetBodyAs is the HTTP method to use when elastic is sending + // a GET request with a body. + DefaultSendGetBodyAs = "GET" ) var ( @@ -121,6 +125,7 @@ type Client struct { basicAuth bool // indicates whether to send HTTP Basic Auth credentials basicAuthUsername string // username for HTTP Basic Auth basicAuthPassword string // password for HTTP Basic Auth + sendGetBodyAs string // override for when sending a GET with a body } // NewClient creates a new client to work with Elasticsearch. @@ -186,6 +191,7 @@ func NewClient(options ...ClientOptionFunc) (*Client, error) { snifferTimeout: DefaultSnifferTimeout, snifferInterval: DefaultSnifferInterval, snifferStop: make(chan bool), + sendGetBodyAs: DefaultSendGetBodyAs, } // Run the options on it @@ -421,6 +427,15 @@ func SetTraceLog(logger *log.Logger) func(*Client) error { } } +// SendGetBodyAs specifies the HTTP method to use when sending a GET request +// with a body. It is GET by default. +func SetSendGetBodyAs(httpMethod string) func(*Client) error { + return func(c *Client) error { + c.sendGetBodyAs = httpMethod + return nil + } +} + // String returns a string representation of the client status. func (c *Client) String() string { c.connsMu.Lock() @@ -892,6 +907,11 @@ func (c *Client) PerformRequest(method, path string, params url.Values, body int // TODO: Make this configurable, including the jitter. retryWaitMsec := int64(100 + (rand.Intn(20) - 10)) + // Change method if sendGetBodyAs is specified. + if method == "GET" && body != nil && c.sendGetBodyAs != "GET" { + method = c.sendGetBodyAs + } + for { pathWithParams := path if len(params) > 0 { diff --git a/client_test.go b/client_test.go index 875be5097..8a15de62f 100644 --- a/client_test.go +++ b/client_test.go @@ -64,6 +64,9 @@ func TestClientDefaults(t *testing.T) { if client.basicAuthPassword != "" { t.Errorf("expected no basic auth password; got: %q", client.basicAuthUsername) } + if client.sendGetBodyAs != "GET" { + t.Errorf("expected sendGetBodyAs to be GET; got: %q", client.sendGetBodyAs) + } } func TestClientWithoutURL(t *testing.T) { From 0523deb696c86a24a20b1871ef164aeb9393be1b Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Mon, 16 Nov 2015 19:31:45 +0100 Subject: [PATCH 13/30] Fix race condition --- client.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client.go b/client.go index 2622e5816..ab31d7a09 100644 --- a/client.go +++ b/client.go @@ -895,6 +895,7 @@ func (c *Client) PerformRequest(method, path string, params url.Values, body int basicAuth := c.basicAuth basicAuthUsername := c.basicAuthUsername basicAuthPassword := c.basicAuthPassword + sendGetBodyAs := c.sendGetBodyAs c.mu.RUnlock() var err error @@ -908,8 +909,8 @@ func (c *Client) PerformRequest(method, path string, params url.Values, body int retryWaitMsec := int64(100 + (rand.Intn(20) - 10)) // Change method if sendGetBodyAs is specified. - if method == "GET" && body != nil && c.sendGetBodyAs != "GET" { - method = c.sendGetBodyAs + if method == "GET" && body != nil && sendGetBodyAs != "GET" { + method = sendGetBodyAs } for { From 444684342d4c12c7c05bed13bd249034370840a9 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Fri, 20 Nov 2015 16:33:50 +0100 Subject: [PATCH 14/30] Add optional gzip compression setting Use `elastic.SetGzip(true)` to enable gzip compression of HTTP request/response with Elasticsearch. Notice that you need to enable gzip compression in Elasticsearch first by adding `http.compression: true` to `elasticsearch.yml` (see [docs](https://www.elastic.co/guide/en/elasticsearch/reference/1.7/modules-http.html)). --- CONTRIBUTORS | 1 + client.go | 24 +++++++++++------- request.go | 70 ++++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 81 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index edd843b6f..10f198283 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -6,6 +6,7 @@ # # Please keep this list sorted. +Adam Weiner [@adamweiner](https://github.com/adamweiner) Alexey Sharov [@nizsheanez](https://github.com/nizsheanez) Braden Bassingthwaite [@bbassingthwaite-va](https://github.com/bbassingthwaite-va) Conrad Pankoff [@deoxxa](https://github.com/deoxxa) diff --git a/client.go b/client.go index ab31d7a09..c8ba0c25e 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.19" + Version = "2.0.20" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. @@ -76,6 +76,9 @@ const ( // DefaultSendGetBodyAs is the HTTP method to use when elastic is sending // a GET request with a body. DefaultSendGetBodyAs = "GET" + + // DefaultGzipEnabled specifies if gzip compression is enabled by default. + DefaultGzipEnabled = false ) var ( @@ -126,6 +129,7 @@ type Client struct { basicAuthUsername string // username for HTTP Basic Auth basicAuthPassword string // password for HTTP Basic Auth sendGetBodyAs string // override for when sending a GET with a body + gzipEnabled bool // gzip compression enabled or disabled (default) } // NewClient creates a new client to work with Elasticsearch. @@ -387,6 +391,14 @@ func SetMaxRetries(maxRetries int) func(*Client) error { } } +// SetGzip enables or disables gzip compression (disabled by default). +func SetGzip(enabled bool) ClientOptionFunc { + return func(c *Client) error { + c.gzipEnabled = enabled + return nil + } +} + // SetDecoder sets the Decoder to use when decoding data from Elasticsearch. // DefaultDecoder is used by default. func SetDecoder(decoder Decoder) func(*Client) error { @@ -896,6 +908,7 @@ func (c *Client) PerformRequest(method, path string, params url.Values, body int basicAuthUsername := c.basicAuthUsername basicAuthPassword := c.basicAuthPassword sendGetBodyAs := c.sendGetBodyAs + gzipEnabled := c.gzipEnabled c.mu.RUnlock() var err error @@ -952,14 +965,7 @@ func (c *Client) PerformRequest(method, path string, params url.Values, body int // Set body if body != nil { - switch b := body.(type) { - case string: - req.SetBodyString(b) - break - default: - req.SetBodyJson(body) - break - } + req.SetBody(body, gzipEnabled) } // Tracing diff --git a/request.go b/request.go index 75a96dc37..1347e1b6f 100644 --- a/request.go +++ b/request.go @@ -6,6 +6,7 @@ package elastic import ( "bytes" + "compress/gzip" "encoding/json" "io" "io/ioutil" @@ -17,6 +18,7 @@ import ( // Elasticsearch-specific HTTP request type Request http.Request +// NewRequest is a http.Request and adds features such as encoding the body. func NewRequest(method, url string) (*Request, error) { req, err := http.NewRequest(method, url, nil) if err != nil { @@ -27,25 +29,83 @@ func NewRequest(method, url string) (*Request, error) { return (*Request)(req), nil } +// SetBasicAuth wraps http.Request's SetBasicAuth. func (r *Request) SetBasicAuth(username, password string) { ((*http.Request)(r)).SetBasicAuth(username, password) } -func (r *Request) SetBodyJson(data interface{}) error { +// SetBody encodes the body in the request. Optionally, it performs GZIP compression. +func (r *Request) SetBody(body interface{}, gzipCompress bool) error { + switch b := body.(type) { + case string: + if gzipCompress { + return r.setBodyGzip(b) + } else { + return r.setBodyString(b) + } + default: + if gzipCompress { + return r.setBodyGzip(body) + } else { + return r.setBodyJson(body) + } + } +} + +// setBodyJson encodes the body as a struct to be marshaled via json.Marshal. +func (r *Request) setBodyJson(data interface{}) error { body, err := json.Marshal(data) if err != nil { return err } - r.SetBody(bytes.NewReader(body)) r.Header.Set("Content-Type", "application/json") + r.setBodyReader(bytes.NewReader(body)) return nil } -func (r *Request) SetBodyString(body string) error { - return r.SetBody(strings.NewReader(body)) +// setBodyString encodes the body as a string. +func (r *Request) setBodyString(body string) error { + return r.setBodyReader(strings.NewReader(body)) +} + +// setBodyGzip gzip's the body. It accepts both strings and structs as body. +// The latter will be encoded via json.Marshal. +func (r *Request) setBodyGzip(body interface{}) error { + switch b := body.(type) { + case string: + buf := new(bytes.Buffer) + w := gzip.NewWriter(buf) + if _, err := w.Write([]byte(b)); err != nil { + return err + } + if err := w.Close(); err != nil { + return err + } + r.Header.Add("Content-Encoding", "gzip") + r.Header.Add("Vary", "Accept-Encoding") + return r.setBodyReader(bytes.NewReader(buf.Bytes())) + default: + data, err := json.Marshal(b) + if err != nil { + return err + } + buf := new(bytes.Buffer) + w := gzip.NewWriter(buf) + if _, err := w.Write(data); err != nil { + return err + } + if err := w.Close(); err != nil { + return err + } + r.Header.Add("Content-Encoding", "gzip") + r.Header.Add("Vary", "Accept-Encoding") + r.Header.Set("Content-Type", "application/json") + return r.setBodyReader(bytes.NewReader(buf.Bytes())) + } } -func (r *Request) SetBody(body io.Reader) error { +// setBodyReader writes the body from an io.Reader. +func (r *Request) setBodyReader(body io.Reader) error { rc, ok := body.(io.ReadCloser) if !ok && body != nil { rc = ioutil.NopCloser(body) From c5d889654c9dedef7656e74b13a087541d2507b4 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Sat, 28 Nov 2015 12:05:52 +0100 Subject: [PATCH 15/30] Add constant score query See https://www.elastic.co/guide/en/elasticsearch/reference/1.7/query-dsl-constant-score-query.html?q=constant_score. --- README.md | 2 +- search_queries_constant.go | 57 +++++++++++++++++++++++++++++++++ search_queries_constant_test.go | 40 +++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 search_queries_constant.go create mode 100644 search_queries_constant_test.go diff --git a/README.md b/README.md index 28ab44b2c..5aedc665f 100644 --- a/README.md +++ b/README.md @@ -279,7 +279,7 @@ on the command line. - [x] `bool` - [x] `boosting` - [ ] `common_terms` -- [ ] `constant_score` +- [x] `constant_score` - [x] `dis_max` - [x] `filtered` - [x] `fuzzy_like_this_query` (`flt`) diff --git a/search_queries_constant.go b/search_queries_constant.go new file mode 100644 index 000000000..f11e6c3e1 --- /dev/null +++ b/search_queries_constant.go @@ -0,0 +1,57 @@ +// Copyright 2012-2015 Oliver Eilhard. All rights reserved. +// Use of this source code is governed by a MIT-license. +// See http://olivere.mit-license.org/license.txt for details. + +package elastic + +// ConstantScoreQuery wraps a filter or another query and simply returns +// a constant score equal to the query boost for every document in the filter. +// +// For more details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/1.7/query-dsl-constant-score-query.html +type ConstantScoreQuery struct { + query Query + filter Filter + boost float64 +} + +// NewConstantScoreQuery creates a new constant score query. +func NewConstantScoreQuery() ConstantScoreQuery { + return ConstantScoreQuery{ + boost: -1, + } +} + +func (q ConstantScoreQuery) Query(query Query) ConstantScoreQuery { + q.query = query + q.filter = nil + return q +} + +func (q ConstantScoreQuery) Filter(filter Filter) ConstantScoreQuery { + q.query = nil + q.filter = filter + return q +} + +func (q ConstantScoreQuery) Boost(boost float64) ConstantScoreQuery { + q.boost = boost + return q +} + +// Source returns JSON for the function score query. +func (q ConstantScoreQuery) Source() interface{} { + source := make(map[string]interface{}) + query := make(map[string]interface{}) + source["constant_score"] = query + + if q.query != nil { + query["query"] = q.query.Source() + } else if q.filter != nil { + query["filter"] = q.filter.Source() + } + if q.boost != -1 { + query["boost"] = q.boost + } + return source +} diff --git a/search_queries_constant_test.go b/search_queries_constant_test.go new file mode 100644 index 000000000..f3d70e1b5 --- /dev/null +++ b/search_queries_constant_test.go @@ -0,0 +1,40 @@ +// Copyright 2012-2015 Oliver Eilhard. All rights reserved. +// Use of this source code is governed by a MIT-license. +// See http://olivere.mit-license.org/license.txt for details. + +package elastic + +import ( + "encoding/json" + "testing" +) + +func TestConstantScoreQueryWithQuery(t *testing.T) { + q := NewConstantScoreQuery(). + Query(NewTermQuery("user", "kimchy")). + Boost(1.2) + data, err := json.Marshal(q.Source()) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"constant_score":{"boost":1.2,"query":{"term":{"user":"kimchy"}}}}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +} + +func TestConstantScoreQueryWithFilter(t *testing.T) { + q := NewConstantScoreQuery(). + Filter(NewTermFilter("user", "kimchy")). + Boost(1.2) + data, err := json.Marshal(q.Source()) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"constant_score":{"boost":1.2,"filter":{"term":{"user":"kimchy"}}}}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +} From 8eaa0553bff8ba58b8ddeb6b012e20536687ab78 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Sat, 28 Nov 2015 12:15:20 +0100 Subject: [PATCH 16/30] Update client version to 2.0.21 --- client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.go b/client.go index c8ba0c25e..b7907f0c9 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.20" + Version = "2.0.21" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. From 85c2b73a0ac9cfdc45a6a231af091239a01cd29f Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Sat, 28 Nov 2015 12:26:03 +0100 Subject: [PATCH 17/30] Fix constant query filename and documentation --- client.go | 2 +- ...stant.go => search_queries_constant_score.go | 17 ++++++++++------- ....go => search_queries_constant_score_test.go | 0 3 files changed, 11 insertions(+), 8 deletions(-) rename search_queries_constant.go => search_queries_constant_score.go (77%) rename search_queries_constant_test.go => search_queries_constant_score_test.go (100%) diff --git a/client.go b/client.go index b7907f0c9..4c398234f 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.21" + Version = "2.0.22" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. diff --git a/search_queries_constant.go b/search_queries_constant_score.go similarity index 77% rename from search_queries_constant.go rename to search_queries_constant_score.go index f11e6c3e1..60407dced 100644 --- a/search_queries_constant.go +++ b/search_queries_constant_score.go @@ -12,30 +12,33 @@ package elastic type ConstantScoreQuery struct { query Query filter Filter - boost float64 + boost *float64 } // NewConstantScoreQuery creates a new constant score query. func NewConstantScoreQuery() ConstantScoreQuery { - return ConstantScoreQuery{ - boost: -1, - } + return ConstantScoreQuery{} } +// Query to wrap in this constant score query. func (q ConstantScoreQuery) Query(query Query) ConstantScoreQuery { q.query = query q.filter = nil return q } +// Filter to wrap in this constant score query. func (q ConstantScoreQuery) Filter(filter Filter) ConstantScoreQuery { q.query = nil q.filter = filter return q } +// Boost sets the boost for this query. Documents matching this query +// will (in addition to the normal weightings) have their score multiplied +// by the boost provided. func (q ConstantScoreQuery) Boost(boost float64) ConstantScoreQuery { - q.boost = boost + q.boost = &boost return q } @@ -50,8 +53,8 @@ func (q ConstantScoreQuery) Source() interface{} { } else if q.filter != nil { query["filter"] = q.filter.Source() } - if q.boost != -1 { - query["boost"] = q.boost + if q.boost != nil { + query["boost"] = *q.boost } return source } diff --git a/search_queries_constant_test.go b/search_queries_constant_score_test.go similarity index 100% rename from search_queries_constant_test.go rename to search_queries_constant_score_test.go From 5a3c5ae9e9d5272296fb42923ed185b916bdda59 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Thu, 3 Dec 2015 14:44:51 +0100 Subject: [PATCH 18/30] Fix auth on ping Close #178 --- CONTRIBUTORS | 1 + ping.go | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 10f198283..f6e94fd19 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -14,6 +14,7 @@ Corey Scott [@corsc](https://github.com/corsc) Daniel Heckrath [@DanielHeckrath](https://github.com/DanielHeckrath) Gerhard Häring [@ghaering](https://github.com/ghaering) Guilherme Silveira [@guilherme-santos](https://github.com/guilherme-santos) +Guillaume J. Charmes [@creack](https://github.com/creack) Jack Lindamood [@cep21](https://github.com/cep21) Junpei Tsuji [@jun06t](https://github.com/jun06t) Maciej Lisiewski [@c2h5oh](https://github.com/c2h5oh) diff --git a/ping.go b/ping.go index 84a2438de..44390d19a 100644 --- a/ping.go +++ b/ping.go @@ -73,6 +73,12 @@ func (s *PingService) Pretty(pretty bool) *PingService { // Do returns the PingResult, the HTTP status code of the Elasticsearch // server, and an error. func (s *PingService) Do() (*PingResult, int, error) { + s.client.mu.RLock() + basicAuth := s.client.basicAuth + basicAuthUsername := s.client.basicAuthUsername + basicAuthPassword := s.client.basicAuthPassword + s.client.mu.RUnlock() + url_ := s.url + "/" params := make(url.Values) @@ -99,6 +105,10 @@ func (s *PingService) Do() (*PingResult, int, error) { return nil, 0, err } + if basicAuth { + req.SetBasicAuth(basicAuthUsername, basicAuthPassword) + } + res, err := s.client.c.Do((*http.Request)(req)) if err != nil { return nil, 0, err From a5a533113833d0a6b8c0bb4f8f16ff2b999868f8 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Thu, 3 Dec 2015 14:50:18 +0100 Subject: [PATCH 19/30] Fix CONTRIBUTORS list --- CONTRIBUTORS | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index f6e94fd19..597385c7f 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -15,6 +15,7 @@ Daniel Heckrath [@DanielHeckrath](https://github.com/DanielHeckrath) Gerhard Häring [@ghaering](https://github.com/ghaering) Guilherme Silveira [@guilherme-santos](https://github.com/guilherme-santos) Guillaume J. Charmes [@creack](https://github.com/creack) +Isaac Saldana [@isaldana](https://github.com/isaldana) Jack Lindamood [@cep21](https://github.com/cep21) Junpei Tsuji [@jun06t](https://github.com/jun06t) Maciej Lisiewski [@c2h5oh](https://github.com/c2h5oh) From 0a75270f31a58cd2d5e1da8fd29673c453e54630 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Fri, 4 Dec 2015 10:04:42 +0100 Subject: [PATCH 20/30] Add Termvector service See #179 on GitHub. --- CONTRIBUTORS | 1 + client.go | 8 ++ termvector.go | 321 +++++++++++++++++++++++++++++++++++++++++++++ termvector_test.go | 119 +++++++++++++++++ 4 files changed, 449 insertions(+) create mode 100644 termvector.go create mode 100644 termvector_test.go diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 597385c7f..1f3679e96 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -25,4 +25,5 @@ Nicholas Wolff [@nwolff](https://github.com/nwolff) Orne Brocaar [@brocaar](https://github.com/brocaar) Sacheendra talluri [@sacheendra](https://github.com/sacheendra) Sean DuBois [@Sean-Der](https://github.com/Sean-Der) +Tetsuya Morimoto [@t2y](https://github.com/t2y) zakthomas [@zakthomas](https://github.com/zakthomas) diff --git a/client.go b/client.go index 4c398234f..153684e1b 100644 --- a/client.go +++ b/client.go @@ -1381,3 +1381,11 @@ func (c *Client) WaitForGreenStatus(timeout string) error { func (c *Client) WaitForYellowStatus(timeout string) error { return c.WaitForStatus("yellow", timeout) } + +// TermVector returns information and statistics on terms in the fields +// of a particular document. +func (c *Client) TermVector(index, typ string) *TermvectorService { + builder := NewTermvectorService(c) + builder = builder.Index(index).Type(typ) + return builder +} diff --git a/termvector.go b/termvector.go new file mode 100644 index 000000000..5feb3df5f --- /dev/null +++ b/termvector.go @@ -0,0 +1,321 @@ +// Copyright 2012-2015 Oliver Eilhard. All rights reserved. +// Use of this source code is governed by a MIT-license. +// See http://olivere.mit-license.org/license.txt for details. + +package elastic + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + + "gopkg.in/olivere/elastic.v2/uritemplates" +) + +// TermvectorService returns information and statistics on terms in the +// fields of a particular document. The document could be stored in the +// index or artificially provided by the user. +// +// See https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-termvectors.html +// for documentation. +type TermvectorService struct { + client *Client + pretty bool + index string + typ string + id string + doc interface{} + fieldStatistics *bool + fields []string + perFieldAnalyzer map[string]string + offsets *bool + parent string + payloads *bool + positions *bool + preference string + realtime *bool + routing string + termStatistics *bool + bodyJson interface{} + bodyString string +} + +// NewTermvectorService creates a new TermvectorService. +func NewTermvectorService(client *Client) *TermvectorService { + return &TermvectorService{ + client: client, + } +} + +// Index in which the document resides. +func (s *TermvectorService) Index(index string) *TermvectorService { + s.index = index + return s +} + +// Type of the document. +func (s *TermvectorService) Type(typ string) *TermvectorService { + s.typ = typ + return s +} + +// Id of the document. +func (s *TermvectorService) Id(id string) *TermvectorService { + s.id = id + return s +} + +// Doc is the document to analyze. +func (s *TermvectorService) Doc(doc interface{}) *TermvectorService { + s.doc = doc + return s +} + +// FieldStatistics specifies if document count, sum of document frequencies +// and sum of total term frequencies should be returned. +func (s *TermvectorService) FieldStatistics(fieldStatistics bool) *TermvectorService { + s.fieldStatistics = &fieldStatistics + return s +} + +// Fields a list of fields to return. +func (s *TermvectorService) Fields(fields ...string) *TermvectorService { + if s.fields == nil { + s.fields = make([]string, 0) + } + s.fields = append(s.fields, fields...) + return s +} + +// PerFieldAnalyzer allows to specify a different analyzer than the one +// at the field. +func (s *TermvectorService) PerFieldAnalyzer(perFieldAnalyzer map[string]string) *TermvectorService { + s.perFieldAnalyzer = perFieldAnalyzer + return s +} + +// Offsets specifies if term offsets should be returned. +func (s *TermvectorService) Offsets(offsets bool) *TermvectorService { + s.offsets = &offsets + return s +} + +// Parent id of documents. +func (s *TermvectorService) Parent(parent string) *TermvectorService { + s.parent = parent + return s +} + +// Payloads specifies if term payloads should be returned. +func (s *TermvectorService) Payloads(payloads bool) *TermvectorService { + s.payloads = &payloads + return s +} + +// Positions specifies if term positions should be returned. +func (s *TermvectorService) Positions(positions bool) *TermvectorService { + s.positions = &positions + return s +} + +// Preference specify the node or shard the operation +// should be performed on (default: random). +func (s *TermvectorService) Preference(preference string) *TermvectorService { + s.preference = preference + return s +} + +// Realtime specifies if request is real-time as opposed to +// near-real-time (default: true). +func (s *TermvectorService) Realtime(realtime bool) *TermvectorService { + s.realtime = &realtime + return s +} + +// Routing is a specific routing value. +func (s *TermvectorService) Routing(routing string) *TermvectorService { + s.routing = routing + return s +} + +// TermStatistics specifies if total term frequency and document frequency +// should be returned. +func (s *TermvectorService) TermStatistics(termStatistics bool) *TermvectorService { + s.termStatistics = &termStatistics + return s +} + +// Pretty indicates that the JSON response be indented and human readable. +func (s *TermvectorService) Pretty(pretty bool) *TermvectorService { + s.pretty = pretty + return s +} + +// BodyJson defines the body parameters. See documentation. +func (s *TermvectorService) BodyJson(body interface{}) *TermvectorService { + s.bodyJson = body + return s +} + +// BodyString defines the body parameters as a string. See documentation. +func (s *TermvectorService) BodyString(body string) *TermvectorService { + s.bodyString = body + return s +} + +// buildURL builds the URL for the operation. +func (s *TermvectorService) buildURL() (string, url.Values, error) { + var pathParam = map[string]string{ + "index": s.index, + "type": s.typ, + } + var path string + var err error + + // Build URL + if s.id != "" { + pathParam["id"] = s.id + path, err = uritemplates.Expand("/{index}/{type}/{id}/_termvector", pathParam) + } else { + path, err = uritemplates.Expand("/{index}/{type}/_termvector", pathParam) + } + + if err != nil { + return "", url.Values{}, err + } + + // Add query string parameters + params := url.Values{} + if s.pretty { + params.Set("pretty", "1") + } + if s.fieldStatistics != nil { + params.Set("field_statistics", fmt.Sprintf("%v", *s.fieldStatistics)) + } + if len(s.fields) > 0 { + params.Set("fields", strings.Join(s.fields, ",")) + } + if s.offsets != nil { + params.Set("offsets", fmt.Sprintf("%v", *s.offsets)) + } + if s.parent != "" { + params.Set("parent", s.parent) + } + if s.payloads != nil { + params.Set("payloads", fmt.Sprintf("%v", *s.payloads)) + } + if s.positions != nil { + params.Set("positions", fmt.Sprintf("%v", *s.positions)) + } + if s.preference != "" { + params.Set("preference", s.preference) + } + if s.realtime != nil { + params.Set("realtime", fmt.Sprintf("%v", *s.realtime)) + } + if s.routing != "" { + params.Set("routing", s.routing) + } + if s.termStatistics != nil { + params.Set("term_statistics", fmt.Sprintf("%v", *s.termStatistics)) + } + return path, params, nil +} + +// Validate checks if the operation is valid. +func (s *TermvectorService) Validate() error { + var invalid []string + if s.index == "" { + invalid = append(invalid, "Index") + } + if s.typ == "" { + invalid = append(invalid, "Type") + } + if len(invalid) > 0 { + return fmt.Errorf("missing required fields: %v", invalid) + } + return nil +} + +// Do executes the operation. +func (s *TermvectorService) Do() (*TermvectorResponse, error) { + // Check pre-conditions + if err := s.Validate(); err != nil { + return nil, err + } + + // Get URL for request + path, params, err := s.buildURL() + if err != nil { + return nil, err + } + + // Setup HTTP request body + var body interface{} + if s.bodyJson != nil { + body = s.bodyJson + } else if s.bodyString != "" { + body = s.bodyString + } else if s.doc != nil || s.perFieldAnalyzer != nil { + data := make(map[string]interface{}) + if s.doc != nil { + data["doc"] = s.doc + } + if len(s.perFieldAnalyzer) > 0 { + data["per_field_analyzer"] = s.perFieldAnalyzer + } + body = data + } else { + body = "" + } + + // Get HTTP response + res, err := s.client.PerformRequest("GET", path, params, body) + if err != nil { + return nil, err + } + + // Return operation response + ret := new(TermvectorResponse) + if err := json.Unmarshal(res.Body, ret); err != nil { + return nil, err + } + return ret, nil +} + +type TokenInfo struct { + StartOffset int64 `json:"start_offset"` + EndOffset int64 `json:"end_offset"` + Position int64 `json:"position"` + Payload string `json:"payload"` +} + +type TermsInfo struct { + DocFreq int64 `json:"doc_freq"` + TermFreq int64 `json:"term_freq"` + Ttf int64 `json:"ttf"` + Tokens []TokenInfo `json:"tokens"` +} + +type FieldStatistics struct { + DocCount int64 `json:"doc_count"` + SumDocFreq int64 `json:"sum_doc_freq"` + SumTtf int64 `json:"sum_ttf"` +} + +type TermVectorsFieldInfo struct { + FieldStatistics FieldStatistics `json:"field_statistics"` + Terms map[string]TermsInfo `json:"terms"` +} + +// TermvectorResponse is the response of TermvectorService.Do. +type TermvectorResponse struct { + Index string `json:"_index"` + Type string `json:"_type"` + Id string `json:"_id,omitempty"` + Version int `json:"_version"` + Found bool `json:"found"` + Took int64 `json:"took"` + TermVectors map[string]TermVectorsFieldInfo `json:"term_vectors"` +} diff --git a/termvector_test.go b/termvector_test.go new file mode 100644 index 000000000..e8071fcfe --- /dev/null +++ b/termvector_test.go @@ -0,0 +1,119 @@ +// Copyright 2012-2015 Oliver Eilhard. All rights reserved. +// Use of this source code is governed by a MIT-license. +// See http://olivere.mit-license.org/license.txt for details. + +package elastic + +import "testing" + +func TestTermVectorBuildURL(t *testing.T) { + client := setupTestClientAndCreateIndex(t) + + tests := []struct { + Index string + Type string + Id string + Expected string + }{ + { + "twitter", + "tweet", + "", + "/twitter/tweet/_termvector", + }, + { + "twitter", + "tweet", + "1", + "/twitter/tweet/1/_termvector", + }, + } + + for _, test := range tests { + builder := client.TermVector(test.Index, test.Type) + if test.Id != "" { + builder = builder.Id(test.Id) + } + path, _, err := builder.buildURL() + if err != nil { + t.Fatal(err) + } + if path != test.Expected { + t.Errorf("expected %q; got: %q", test.Expected, path) + } + } +} + +func TestTermVectorWithId(t *testing.T) { + client := setupTestClientAndCreateIndex(t) + + tweet1 := tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch."} + + // Add a document + indexResult, err := client.Index(). + Index(testIndexName). + Type("tweet"). + Id("1"). + BodyJson(&tweet1). + Refresh(true). + Do() + if err != nil { + t.Fatal(err) + } + if indexResult == nil { + t.Errorf("expected result to be != nil; got: %v", indexResult) + } + + // TermVectors by specifying ID + field := "Message" + result, err := client.TermVector(testIndexName, "tweet"). + Id("1"). + Fields(field). + FieldStatistics(true). + TermStatistics(true). + Do() + if err != nil { + t.Fatal(err) + } + if result == nil { + t.Fatal("expected to return information and statistics") + } + if !result.Found { + t.Errorf("expected found to be %v; got: %v", true, result.Found) + } + if result.Took <= 0 { + t.Errorf("expected took in millis > 0; got: %v", result.Took) + } +} + +func TestTermVectorWithDoc(t *testing.T) { + client := setupTestClientAndCreateIndex(t) + + // TermVectors by specifying Doc + var doc = map[string]interface{}{ + "fullname": "John Doe", + "text": "twitter test test test", + } + var perFieldAnalyzer = map[string]string{ + "fullname": "keyword", + } + + result, err := client.TermVector(testIndexName, "tweet"). + Doc(doc). + PerFieldAnalyzer(perFieldAnalyzer). + FieldStatistics(true). + TermStatistics(true). + Do() + if err != nil { + t.Fatal(err) + } + if result == nil { + t.Fatal("expected to return information and statistics") + } + if !result.Found { + t.Errorf("expected found to be %v; got: %v", true, result.Found) + } + if result.Took <= 0 { + t.Errorf("expected took in millis > 0; got: %v", result.Took) + } +} From 0bdcb4e46bd40680536d74a110b117b3cdc044e3 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Fri, 4 Dec 2015 10:06:23 +0100 Subject: [PATCH 21/30] Update README and version --- README.md | 2 +- client.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5aedc665f..aabfc1ac4 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ Here's the current API status. - [x] Multi Get - [x] Bulk - [ ] Bulk UDP -- [ ] Term vectors +- [x] Term vectors - [ ] Multi term vectors - [x] Count - [ ] Validate diff --git a/client.go b/client.go index 153684e1b..67f22a5d9 100644 --- a/client.go +++ b/client.go @@ -22,7 +22,7 @@ import ( const ( // Version is the current version of Elastic. - Version = "2.0.22" + Version = "2.0.23" // DefaultUrl is the default endpoint of Elasticsearch on the local machine. // It is used e.g. when initializing a new Client without a specific URL. From 8f9ada00ec7ce7c64daeb24b68bd932ec9f7cb64 Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Fri, 4 Dec 2015 10:09:26 +0100 Subject: [PATCH 22/30] Try to make test succeed on Travis --- termvector_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/termvector_test.go b/termvector_test.go index e8071fcfe..dfb732b34 100644 --- a/termvector_test.go +++ b/termvector_test.go @@ -4,7 +4,10 @@ package elastic -import "testing" +import ( + "testing" + "time" +) func TestTermVectorBuildURL(t *testing.T) { client := setupTestClientAndCreateIndex(t) @@ -89,6 +92,11 @@ func TestTermVectorWithId(t *testing.T) { func TestTermVectorWithDoc(t *testing.T) { client := setupTestClientAndCreateIndex(t) + // Travis lags sometimes + if isTravis() { + time.Sleep(2 * time.Second) + } + // TermVectors by specifying Doc var doc = map[string]interface{}{ "fullname": "John Doe", From 15d4018b5e547aef3e99f95fe6964e47cf790412 Mon Sep 17 00:00:00 2001 From: John Barker Date: Sat, 7 May 2016 18:01:25 -0400 Subject: [PATCH 23/30] Add a test for parent in BulkDeleteRequest --- bulk_delete_request_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bulk_delete_request_test.go b/bulk_delete_request_test.go index 73abfcd40..d8f0d6d66 100644 --- a/bulk_delete_request_test.go +++ b/bulk_delete_request_test.go @@ -20,6 +20,12 @@ func TestBulkDeleteRequestSerialization(t *testing.T) { `{"delete":{"_id":"1","_index":"index1","_type":"tweet"}}`, }, }, + { + Request: NewBulkDeleteRequest().Index("index1").Type("tweet").Id("1").Parent("2"), + Expected: []string{ + `{"delete":{"_id":"1","_index":"index1","_parent":"2","_type":"tweet"}}`, + }, + }, } for i, test := range tests { From 69df0c2b4f28f9bf5b03502ea1a8e94bb862ce6e Mon Sep 17 00:00:00 2001 From: John Barker Date: Fri, 6 May 2016 14:52:44 -0400 Subject: [PATCH 24/30] BulkDeleteRequest#Parent, needed for deleting child records --- bulk_delete_request.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bulk_delete_request.go b/bulk_delete_request.go index 0ea372209..b2ef2f24a 100644 --- a/bulk_delete_request.go +++ b/bulk_delete_request.go @@ -18,6 +18,7 @@ type BulkDeleteRequest struct { index string typ string id string + parent string routing string refresh *bool version int64 // default is MATCH_ANY @@ -43,6 +44,11 @@ func (r *BulkDeleteRequest) Id(id string) *BulkDeleteRequest { return r } +func (r *BulkDeleteRequest) Parent(parent string) *BulkDeleteRequest { + r.parent = parent + return r +} + func (r *BulkDeleteRequest) Routing(routing string) *BulkDeleteRequest { r.routing = routing return r @@ -87,6 +93,9 @@ func (r *BulkDeleteRequest) Source() ([]string, error) { if r.id != "" { deleteCommand["_id"] = r.id } + if r.parent != "" { + deleteCommand["_parent"] = r.parent + } if r.routing != "" { deleteCommand["_routing"] = r.routing } From 7c6dd2b405c9ee9892a640fa2afc90b4eff5e8fc Mon Sep 17 00:00:00 2001 From: Oliver Eilhard Date: Sun, 15 May 2016 13:04:24 +0200 Subject: [PATCH 25/30] Revert "Feature/bulk delete request parent" --- bulk_delete_request.go | 9 --------- bulk_delete_request_test.go | 6 ------ 2 files changed, 15 deletions(-) diff --git a/bulk_delete_request.go b/bulk_delete_request.go index b2ef2f24a..0ea372209 100644 --- a/bulk_delete_request.go +++ b/bulk_delete_request.go @@ -18,7 +18,6 @@ type BulkDeleteRequest struct { index string typ string id string - parent string routing string refresh *bool version int64 // default is MATCH_ANY @@ -44,11 +43,6 @@ func (r *BulkDeleteRequest) Id(id string) *BulkDeleteRequest { return r } -func (r *BulkDeleteRequest) Parent(parent string) *BulkDeleteRequest { - r.parent = parent - return r -} - func (r *BulkDeleteRequest) Routing(routing string) *BulkDeleteRequest { r.routing = routing return r @@ -93,9 +87,6 @@ func (r *BulkDeleteRequest) Source() ([]string, error) { if r.id != "" { deleteCommand["_id"] = r.id } - if r.parent != "" { - deleteCommand["_parent"] = r.parent - } if r.routing != "" { deleteCommand["_routing"] = r.routing } diff --git a/bulk_delete_request_test.go b/bulk_delete_request_test.go index d8f0d6d66..73abfcd40 100644 --- a/bulk_delete_request_test.go +++ b/bulk_delete_request_test.go @@ -20,12 +20,6 @@ func TestBulkDeleteRequestSerialization(t *testing.T) { `{"delete":{"_id":"1","_index":"index1","_type":"tweet"}}`, }, }, - { - Request: NewBulkDeleteRequest().Index("index1").Type("tweet").Id("1").Parent("2"), - Expected: []string{ - `{"delete":{"_id":"1","_index":"index1","_parent":"2","_type":"tweet"}}`, - }, - }, } for i, test := range tests { From e6f1f5f7cf26c8c5a6bccdffa4d39263c079d583 Mon Sep 17 00:00:00 2001 From: Shawn Smith Date: Wed, 5 Oct 2016 09:19:27 +0900 Subject: [PATCH 26/30] fix typo --- example_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_test.go b/example_test.go index 7789e729c..765e163ed 100644 --- a/example_test.go +++ b/example_test.go @@ -378,7 +378,7 @@ func ExampleAggregations() { // Access "timeline" aggregate in search result. agg, found := searchResult.Aggregations.Terms("timeline") if !found { - log.Fatalf("we sould have a terms aggregation called %q", "timeline") + log.Fatalf("we should have a terms aggregation called %q", "timeline") } for _, userBucket := range agg.Buckets { // Every bucket should have the user field as key. From c8eec81b3f8f050957e8bacc8134646531565ce3 Mon Sep 17 00:00:00 2001 From: Shawn Smith Date: Wed, 5 Oct 2016 09:19:46 +0900 Subject: [PATCH 27/30] fix typo --- scroll_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scroll_test.go b/scroll_test.go index 4a5c48111..58347e831 100644 --- a/scroll_test.go +++ b/scroll_test.go @@ -92,7 +92,7 @@ func TestScroll(t *testing.T) { scrollId = searchResult.ScrollId if scrollId == "" { - t.Errorf("expeced scrollId in results; got %q", scrollId) + t.Errorf("expected scrollId in results; got %q", scrollId) } } From eef0835dfe88c92b16646a4f5994bc246f93d5bd Mon Sep 17 00:00:00 2001 From: Shawn Smith Date: Wed, 5 Oct 2016 09:20:15 +0900 Subject: [PATCH 28/30] fix typo --- search_filters_prefix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search_filters_prefix.go b/search_filters_prefix.go index a2f527357..de19ebd47 100644 --- a/search_filters_prefix.go +++ b/search_filters_prefix.go @@ -4,7 +4,7 @@ package elastic -// Filters documents that have fiels containing terms +// Filters documents that have fields containing terms // with a specified prefix (not analyzed). // For details, see: // http://www.elasticsearch.org/guide/reference/query-dsl/prefix-filter.html From 518f200712e1ca9fb3d26933b38c1d00310c60fd Mon Sep 17 00:00:00 2001 From: Eli Reisman <32776521+elireisman@users.noreply.github.com> Date: Wed, 16 Jan 2019 23:52:34 -0800 Subject: [PATCH 29/30] Fix typo in HighlightQuery API call --- highlight.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/highlight.go b/highlight.go index dab8c45b6..8ee8da2ac 100644 --- a/highlight.go +++ b/highlight.go @@ -120,7 +120,7 @@ func (hl *Highlight) Fragmenter(fragmenter string) *Highlight { return hl } -func (hl *Highlight) HighlighQuery(highlightQuery Query) *Highlight { +func (hl *Highlight) HighlightQuery(highlightQuery Query) *Highlight { hl.highlightQuery = highlightQuery return hl } From bada83c0c856515c9eaeb7bf48c0708bd4c31e27 Mon Sep 17 00:00:00 2001 From: Diego Becciolini Date: Tue, 25 Jun 2019 11:43:09 +0100 Subject: [PATCH 30/30] take into account URL changes on connection update --- client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.go b/client.go index 67f22a5d9..5852be0bc 100644 --- a/client.go +++ b/client.go @@ -723,7 +723,7 @@ func (c *Client) updateConns(conns []*conn) { for _, conn := range conns { var found bool for _, oldConn := range c.conns { - if oldConn.NodeID() == conn.NodeID() { + if oldConn.NodeID() == conn.NodeID() && oldConn.URL() == conn.URL() { // Take over the old connection newConns = append(newConns, oldConn) found = true