From e761a3cbce80e791cb14f240be9c931ae118e2f3 Mon Sep 17 00:00:00 2001 From: Cas Donoghue Date: Wed, 15 Jan 2025 08:11:46 -0800 Subject: [PATCH 1/3] Remove irrelevant warning (#1202) * Remove irrelevant warning This commit removes an old warning which is no longer relevant to any supported stack version. It is noisy and there is no action for the user to take. * Add changelog entry for log removal * 11.22.11 release prep Update version in preparation for a release. --- CHANGELOG.md | 3 +++ lib/logstash/outputs/elasticsearch/http_client/pool.rb | 9 +-------- logstash-output-elasticsearch.gemspec | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19c27494..c9d9ae6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 11.22.11 + - Remove irrelevant log warning about elastic stack version [#1202](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1202) + ## 11.22.10 - Add `x-elastic-product-origin` header to Elasticsearch requests [#1195](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1195) diff --git a/lib/logstash/outputs/elasticsearch/http_client/pool.rb b/lib/logstash/outputs/elasticsearch/http_client/pool.rb index fb3194cc..68715066 100644 --- a/lib/logstash/outputs/elasticsearch/http_client/pool.rb +++ b/lib/logstash/outputs/elasticsearch/http_client/pool.rb @@ -515,20 +515,13 @@ def set_last_es_version(version, url) major = major_version(version) if @maximum_seen_major_version.nil? @logger.info("Elasticsearch version determined (#{version})", es_version: major) - set_maximum_seen_major_version(major) + @maximum_seen_major_version = major elsif major > @maximum_seen_major_version warn_on_higher_major_version(major, url) @maximum_seen_major_version = major end end - def set_maximum_seen_major_version(major) - if major >= 6 - @logger.warn("Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type", es_version: major) - end - @maximum_seen_major_version = major - end - def warn_on_higher_major_version(major, url) @logger.warn("Detected a node with a higher major version than previously observed, " + "this could be the result of an Elasticsearch cluster upgrade", diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec index d2e88631..e8d5f858 100644 --- a/logstash-output-elasticsearch.gemspec +++ b/logstash-output-elasticsearch.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-elasticsearch' - s.version = '11.22.10' + s.version = '11.22.11' s.licenses = ['apache-2.0'] s.summary = "Stores logs in Elasticsearch" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 0261f27071c5f8f029de81dbb7cf8b66c4a7bb16 Mon Sep 17 00:00:00 2001 From: Cas Donoghue Date: Tue, 28 Jan 2025 08:38:08 -0800 Subject: [PATCH 2/3] Properly handle 413 Payload Too Large errors (#1199) (#1206) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Properly handle `413` Payload Too Large errors Previously when Elasticsearch responds with a 413 (Payload Too Large) status, the manticore adapter raises an error before the response can be processed by the bulk_send error handling. This commit refactors the way `BadErrorResponse` codes are handled. Previously we had logic in the manticore adaptor which special cased raising errors on some codes. This commit refactors such that the adaptor raises on any error status and the caller is now responsible for special case handling the code. * 12.0.2 release prep * Use `error_code` instead of `code` when handling BadResponseCodeError Previously a few bugs spotted in code review were being obfuscated by the combinations of tests not running in CI and the incorrect method for retrieving a code from a BadResponseCodeError. This commit updates the method names and addresses the feedback from code review. --------- Co-authored-by: João Duarte --- CHANGELOG.md | 2 + docs/index.asciidoc | 17 ++++++- .../outputs/elasticsearch/http_client.rb | 49 ++++++++++--------- .../http_client/manticore_adapter.rb | 5 +- .../outputs/elasticsearch/http_client/pool.rb | 24 ++++----- logstash-output-elasticsearch.gemspec | 2 +- spec/unit/outputs/elasticsearch_spec.rb | 7 ++- 7 files changed, 61 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9d9ae6f..1a46b00a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 11.22.12 + - Properly handle http code 413 (Payload Too Large) [#1199](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1199) ## 11.22.11 - Remove irrelevant log warning about elastic stack version [#1202](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1202) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 514722a3..07f7efdc 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -196,7 +196,22 @@ This plugin uses the Elasticsearch bulk API to optimize its imports into Elastic either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP request are handled differently than error codes for individual documents. -HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely, +including 413 (Payload Too Large) responses. + +If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead: + +[source,ruby] +----- +output { + elasticsearch { + hosts => ["localhost:9200"] + dlq_custom_codes => [413] # Send 413 errors to DLQ instead of retrying + } +----- + +This will capture oversized payloads in the DLQ for analysis rather than retrying them. The following document errors are handled as follows: diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index e0b70e36..120d3e67 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -182,22 +182,20 @@ def join_bulk_responses(bulk_responses) def bulk_send(body_stream, batch_actions) params = compression_level? ? {:headers => {"Content-Encoding" => "gzip"}} : {} - response = @pool.post(@bulk_path, params, body_stream.string) - - @bulk_response_metrics.increment(response.code.to_s) - - case response.code - when 200 # OK - LogStash::Json.load(response.body) - when 413 # Payload Too Large + begin + response = @pool.post(@bulk_path, params, body_stream.string) + @bulk_response_metrics.increment(response.code.to_s) + rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e + @bulk_response_metrics.increment(e.response_code.to_s) + raise e unless e.response_code == 413 + # special handling for 413, treat it as a document level issue logger.warn("Bulk request rejected: `413 Payload Too Large`", :action_count => batch_actions.size, :content_length => body_stream.size) - emulate_batch_error_response(batch_actions, response.code, 'payload_too_large') - else - url = ::LogStash::Util::SafeURI.new(response.final_url) - raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new( - response.code, url, body_stream.to_s, response.body - ) + return emulate_batch_error_response(batch_actions, 413, 'payload_too_large') + rescue => e # it may be a network issue instead, re-raise + raise e end + + LogStash::Json.load(response.body) end def emulate_batch_error_response(actions, http_code, reason) @@ -411,6 +409,9 @@ def host_to_url(h) def exists?(path, use_get=false) response = use_get ? @pool.get(path) : @pool.head(path) response.code >= 200 && response.code <= 299 + rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e + return false if e.response_code == 404 + raise e end def template_exists?(template_endpoint, name) @@ -421,6 +422,8 @@ def template_put(template_endpoint, name, template) path = "#{template_endpoint}/#{name}" logger.info("Installing Elasticsearch template", name: name) @pool.put(path, nil, LogStash::Json.dump(template)) + rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e + raise e unless e.response_code == 404 end # ILM methods @@ -432,17 +435,15 @@ def rollover_alias_exists?(name) # Create a new rollover alias def rollover_alias_put(alias_name, alias_definition) - begin - @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition)) - logger.info("Created rollover alias", name: alias_name) - # If the rollover alias already exists, ignore the error that comes back from Elasticsearch - rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e - if e.response_code == 400 - logger.info("Rollover alias already exists, skipping", name: alias_name) - return - end - raise e + @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition)) + logger.info("Created rollover alias", name: alias_name) + # If the rollover alias already exists, ignore the error that comes back from Elasticsearch + rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e + if e.response_code == 400 + logger.info("Rollover alias already exists, skipping", name: alias_name) + return end + raise e end def get_xpack_info diff --git a/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb b/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb index c9e49ec7..11f85b53 100644 --- a/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +++ b/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb @@ -76,11 +76,8 @@ def perform_request(url, method, path, params={}, body=nil) raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(e, request_uri_as_string) end - # 404s are excluded because they are valid codes in the case of - # template installation. We might need a better story around this later - # but for our current purposes this is correct code = resp.code - if code < 200 || code > 299 && code != 404 + if code < 200 || code > 299 # assume anything not 2xx is an error that the layer above needs to interpret raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(code, request_uri, body, resp.body) end diff --git a/lib/logstash/outputs/elasticsearch/http_client/pool.rb b/lib/logstash/outputs/elasticsearch/http_client/pool.rb index 68715066..1ef9d0f9 100644 --- a/lib/logstash/outputs/elasticsearch/http_client/pool.rb +++ b/lib/logstash/outputs/elasticsearch/http_client/pool.rb @@ -253,13 +253,11 @@ def get_license(url) def health_check_request(url) logger.debug("Running health check to see if an Elasticsearch connection is working", :healthcheck_url => url.sanitized.to_s, :path => @healthcheck_path) - begin - response = perform_request_to_url(url, :head, @healthcheck_path) - return response, nil - rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e - logger.warn("Health check failed", code: e.response_code, url: e.url, message: e.message) - return nil, e - end + response = perform_request_to_url(url, :head, @healthcheck_path) + return response, nil + rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e + logger.warn("Health check failed", code: e.response_code, url: e.url, message: e.message) + return nil, e end def healthcheck!(register_phase = true) @@ -312,13 +310,11 @@ def healthcheck!(register_phase = true) end def get_root_path(url, params={}) - begin - resp = perform_request_to_url(url, :get, ROOT_URI_PATH, params) - return resp, nil - rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e - logger.warn("Elasticsearch main endpoint returns #{e.response_code}", message: e.message, body: e.response_body) - return nil, e - end + resp = perform_request_to_url(url, :get, ROOT_URI_PATH, params) + return resp, nil + rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e + logger.warn("Elasticsearch main endpoint returns #{e.response_code}", message: e.message, body: e.response_body) + return nil, e end def test_serverless_connection(url, root_response) diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec index e8d5f858..1909085c 100644 --- a/logstash-output-elasticsearch.gemspec +++ b/logstash-output-elasticsearch.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-elasticsearch' - s.version = '11.22.11' + s.version = '11.22.12' s.licenses = ['apache-2.0'] s.summary = "Stores logs in Elasticsearch" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/unit/outputs/elasticsearch_spec.rb b/spec/unit/outputs/elasticsearch_spec.rb index b71e8720..35c90804 100644 --- a/spec/unit/outputs/elasticsearch_spec.rb +++ b/spec/unit/outputs/elasticsearch_spec.rb @@ -915,7 +915,12 @@ allow(elasticsearch_output_instance.client.pool).to receive(:post) do |path, params, body| if body.length > max_bytes max_bytes *= 2 # ensure a successful retry - double("Response", :code => 413, :body => "") + raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new( + 413, + "test-url", + body, + "" + ) else double("Response", :code => 200, :body => '{"errors":false,"items":[{"index":{"status":200,"result":"created"}}]}') end From f019f62780d1177337973084105af81810b4428b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Fri, 1 Aug 2025 19:36:35 +0100 Subject: [PATCH 3/3] Backport bulk count and uncompressed size headers to 11.x (+ ci/test fixes) (#1219) Adds two new headers to each bulk request: * "X-Elastic-Event-Count": number of actions / documents in that bulk request * "X-Elastic-Uncompressed-Request-Length": size in bytes of the request body before compression X-Elastic-Uncompressed-Request-Length is equal to Content-Length when compression is disabled. Backport fixes: * Tolerate the elasticsearch-ruby v8 client in integration tests. (#1208) * elasticsearch-ruby client got updated to v8 in LS core. This plugin uses it in integration tests. This change tolerates both elasticsearch-ruby v7 and v8 client versions. * Fix the ILM spec issue where method was removed, restored internally. Co-authored-by: Mashhur <99575341+mashhurs@users.noreply.github.com> --- .ci/docker-compose.override.yml | 2 - .ci/logstash-run.sh | 4 +- .travis.yml | 22 +++-- CHANGELOG.md | 4 + .../outputs/elasticsearch/http_client.rb | 22 +++-- logstash-output-elasticsearch.gemspec | 2 +- spec/es_spec_helper.rb | 56 ++++++++----- spec/fixtures/test_certs/GENERATED_AT | 2 +- spec/fixtures/test_certs/ca.crt | 57 +++++++------ spec/fixtures/test_certs/ca.der.sha256 | 2 +- spec/fixtures/test_certs/renew.sh | 5 +- spec/fixtures/test_certs/test.crt | 57 ++++++------- spec/fixtures/test_certs/test.der.sha256 | 2 +- spec/fixtures/test_certs/test.p12 | Bin 4053 -> 4291 bytes .../outputs/compressed_indexing_spec.rb | 4 +- spec/integration/outputs/delete_spec.rb | 8 +- spec/integration/outputs/ilm_spec.rb | 30 ++++--- spec/integration/outputs/index_spec.rb | 14 +++- .../integration/outputs/index_version_spec.rb | 14 ++-- .../outputs/painless_update_spec.rb | 21 ++--- .../outputs/unsupported_actions_spec.rb | 25 +++--- spec/integration/outputs/update_spec.rb | 20 +++-- spec/spec_helper.rb | 8 ++ .../elasticsearch/api/actions/get_alias.rb | 18 ---- .../elasticsearch/api/actions/put_alias.rb | 24 ------ .../outputs/elasticsearch/http_client_spec.rb | 77 ++++++++++++++++++ spec/unit/outputs/elasticsearch_spec.rb | 2 +- spec/unit/outputs/error_whitelist_spec.rb | 1 - 28 files changed, 305 insertions(+), 198 deletions(-) delete mode 100644 spec/support/elasticsearch/api/actions/get_alias.rb delete mode 100644 spec/support/elasticsearch/api/actions/put_alias.rb diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml index 260b0e50..7608b786 100644 --- a/.ci/docker-compose.override.yml +++ b/.ci/docker-compose.override.yml @@ -1,5 +1,3 @@ -version: '3' - services: logstash: diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh index 0801282b..4c231345 100755 --- a/.ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -13,7 +13,7 @@ else fi # CentOS 7 using curl defaults does not enable TLSv1.3 -CURL_OPTS="-k --tlsv1.2 --tls-max 1.3" +CURL_OPTS="-k -u admin:elastic --tlsv1.2 --tls-max 1.3" wait_for_es() { count=120 @@ -22,7 +22,7 @@ wait_for_es() { [[ $count -eq 0 ]] && exit 1 sleep 1 done - echo $(curl $CURL_OPTS -vi $ES_URL | jq -r .version.number) + echo $(curl $CURL_OPTS $ES_URL | jq -r .version.number) } if [[ "$INTEGRATION" != "true" ]]; then diff --git a/.travis.yml b/.travis.yml index 112673ac..5e29addb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,13 +4,23 @@ import: jobs: include: - stage: "Integration Tests" - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current + env: INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current + - env: INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.previous + - env: INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current + - env: INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.previous - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.next - - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.future + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.next + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=main - stage: "Secure Integration Tests" - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current SNAPSHOT=true - - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current - - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current ES_SSL_KEY_INVALID=true - - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3 + env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current ES_SSL_KEY_INVALID=true + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3 + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current ES_SSL_KEY_INVALID=true + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3 + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info SNAPSHOT=true ELASTIC_STACK_VERSION=8.next + - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info SNAPSHOT=true ELASTIC_STACK_VERSION=9.next diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a46b00a..2d0591b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ +## 11.22.13 + - Add headers reporting uncompressed size and doc count for bulk requests [#1217](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1217) + ## 11.22.12 - Properly handle http code 413 (Payload Too Large) [#1199](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1199) + ## 11.22.11 - Remove irrelevant log warning about elastic stack version [#1202](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1202) diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index 120d3e67..8d6e02cb 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -21,7 +21,8 @@ module LogStash; module Outputs; class ElasticSearch; # We wound up agreeing that a number greater than 10 MiB and less than 100MiB # made sense. We picked one on the lowish side to not use too much heap. TARGET_BULK_BYTES = 20 * 1024 * 1024 # 20MiB - + EVENT_COUNT_HEADER = "X-Elastic-Event-Count".freeze + UNCOMPRESSED_LENGTH_HEADER = "X-Elastic-Uncompressed-Request-Length".freeze class HttpClient attr_reader :client, :options, :logger, :pool, :action_count, :recv_count @@ -143,7 +144,11 @@ def bulk(actions) :payload_size => stream_writer.pos, :content_length => body_stream.size, :batch_offset => (index + 1 - batch_actions.size)) - bulk_responses << bulk_send(body_stream, batch_actions) + headers = { + EVENT_COUNT_HEADER => batch_actions.size.to_s, + UNCOMPRESSED_LENGTH_HEADER => stream_writer.pos.to_s + } + bulk_responses << bulk_send(body_stream, batch_actions, headers) body_stream.truncate(0) && body_stream.seek(0) stream_writer = gzip_writer(body_stream) if compression_level? batch_actions.clear @@ -159,7 +164,14 @@ def bulk(actions) :payload_size => stream_writer.pos, :content_length => body_stream.size, :batch_offset => (actions.size - batch_actions.size)) - bulk_responses << bulk_send(body_stream, batch_actions) if body_stream.size > 0 + + if body_stream.size > 0 + headers = { + EVENT_COUNT_HEADER => batch_actions.size.to_s, + UNCOMPRESSED_LENGTH_HEADER => stream_writer.pos.to_s + } + bulk_responses << bulk_send(body_stream, batch_actions, headers) + end body_stream.close unless compression_level? join_bulk_responses(bulk_responses) @@ -179,8 +191,8 @@ def join_bulk_responses(bulk_responses) } end - def bulk_send(body_stream, batch_actions) - params = compression_level? ? {:headers => {"Content-Encoding" => "gzip"}} : {} + def bulk_send(body_stream, batch_actions, headers = {}) + params = compression_level? ? {:headers => headers.merge("Content-Encoding" => "gzip") } : { :headers => headers } begin response = @pool.post(@bulk_path, params, body_stream.string) diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec index 1909085c..10f737a4 100644 --- a/logstash-output-elasticsearch.gemspec +++ b/logstash-output-elasticsearch.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-elasticsearch' - s.version = '11.22.12' + s.version = '11.22.13' s.licenses = ['apache-2.0'] s.summary = "Stores logs in Elasticsearch" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 6981a9c1..0f86f549 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -1,15 +1,18 @@ require_relative './spec_helper' require 'elasticsearch' -require_relative "support/elasticsearch/api/actions/delete_ilm_policy" -require_relative "support/elasticsearch/api/actions/get_alias" -require_relative "support/elasticsearch/api/actions/put_alias" -require_relative "support/elasticsearch/api/actions/get_ilm_policy" -require_relative "support/elasticsearch/api/actions/put_ilm_policy" require 'json' require 'cabin' +# remove this condition and support package once plugin starts consuming elasticsearch-ruby v8 client +# in elasticsearch-ruby v7, ILM APIs were in a separate xpack gem, now directly available +unless elastic_ruby_v8_client_available? + require_relative "support/elasticsearch/api/actions/delete_ilm_policy" + require_relative "support/elasticsearch/api/actions/get_ilm_policy" + require_relative "support/elasticsearch/api/actions/put_ilm_policy" +end + module ESHelper def get_host_port if ENV["INTEGRATION"] == "true" @@ -20,8 +23,12 @@ def get_host_port end def get_client - Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client| - allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking + if elastic_ruby_v8_client_available? + Elasticsearch::Client.new(:hosts => [get_host_port]) + else + Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client| + allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking + end end end @@ -128,31 +135,36 @@ def get_cluster_settings(client) end def get_policy(client, policy_name) - client.get_ilm_policy(name: policy_name) + if elastic_ruby_v8_client_available? + client.index_lifecycle_management.get_lifecycle(policy: policy_name) + else + client.get_ilm_policy(name: policy_name) + end end def put_policy(client, policy_name, policy) - client.put_ilm_policy({:name => policy_name, :body=> policy}) - end - - def put_alias(client, the_alias, index) - body = { - "aliases" => { - index => { - "is_write_index"=> true - } - } - } - client.put_alias({name: the_alias, body: body}) + if elastic_ruby_v8_client_available? + client.index_lifecycle_management.put_lifecycle({:policy => policy_name, :body=> policy}) + else + client.put_ilm_policy({:name => policy_name, :body=> policy}) + end end def clean_ilm(client) - client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key) if key =~ /logstash-policy/ } + if elastic_ruby_v8_client_available? + client.index_lifecycle_management.get_lifecycle.each_key { |key| client.index_lifecycle_management.delete_lifecycle(policy: key) if key =~ /logstash-policy/ } + else + client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key) if key =~ /logstash-policy/ } + end end def supports_ilm?(client) begin - client.get_ilm_policy + if elastic_ruby_v8_client_available? + client.index_lifecycle_management.get_lifecycle + else + client.get_ilm_policy + end true rescue false diff --git a/spec/fixtures/test_certs/GENERATED_AT b/spec/fixtures/test_certs/GENERATED_AT index 79bce0d8..83f8b6c2 100644 --- a/spec/fixtures/test_certs/GENERATED_AT +++ b/spec/fixtures/test_certs/GENERATED_AT @@ -1 +1 @@ -2024-06-25T21:50:58+01:00 +2025-07-22T11:15:03+01:00 diff --git a/spec/fixtures/test_certs/ca.crt b/spec/fixtures/test_certs/ca.crt index a732ad14..686c13d8 100644 --- a/spec/fixtures/test_certs/ca.crt +++ b/spec/fixtures/test_certs/ca.crt @@ -1,29 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFDDCCAvQCAQEwDQYJKoZIhvcNAQELBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV -BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD -DAZSb290Q0EwHhcNMjQwNjI1MjA1MDU4WhcNMjUwNjI1MjA1MDU4WjBMMQswCQYD -VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF -TXlMYWIxDzANBgNVBAMMBlJvb3RDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC -AgoCggIBAMtTMqAWuH17b9XqPa5L3HNqgnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh -5er5JF2iomYOHiD5JncYr4YmRQKuYfD6B1WI5FuQthD/OlA1/RHqtbY27J33SaO6 -6ro8gal7vjHrXKQkefVYRwdfO6DqqbhV6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQq -GZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju4jPhp+43BkGwEimI5km7jNC1nwjiHtxD -sY/s93AKa/vLktXKUK5nA3jjJOhAbRTVnbOAgxFt0YbX98xW/aUqscgBUVs9J/My -TRMwVKJ7Vsmth1PdJQksUASuzESlSPl09dMjTQ+MXzJDt0JvX8SIJPmbBng78MSa -CUhpOZiii1l2mBfPWejx20I/SMCUNmzbwm2w9JD50Jv2iX4l4ge4H1CIK1/orW1p -dY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aLojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv -5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nxUjCbgn0RABojRoGdhhY3QDipgwmSgFZx -r064RFr1bt/Ml3MJmPf535mSwPdk/j/zw4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMP -LEdi73Qjzmr3ENIrir4O86wNz81YRfYkg9ZX8yKJK9LBAUrYCjJ3AgMBAAEwDQYJ -KoZIhvcNAQELBQADggIBABym9LMyS9W9lvpcH4OK1YLfBPJwrhZ+4keiriY4zWOo -pB+v2Q35neMMXSlTDpeIwPdMkqsh8VZprOWURF80JGvpJ6fBfi05rCDWp/ol1ZKi -snCA+dE2zDK7Z3+F0MbakT5oBi5WgkXSvRvlJEJ/gBD7WC1wq0kxCMK+M5w2RPAT -nnV/iozNBkwExxyJA7BpS6F/v0XjwK7fm5Kpql7zKlh5piZ2IVU0B60Sqskcb2mU -90+1r9T06ekIW/Iz1jd5RWYziu0nbmDeKeKvGAICNU+evYXW+/5kKecMLuEvDCgS -ssbt/Hb510uLHhxfhN4SbvBl2zADsLC+2arf2ATIwD8ZXDDs04ayBsejV0ZwVrTZ -ExKqAys+B3tuIHGRqL5VukdmH6g6oJziYueohPBCOuSOzDd0FhppF4uXZS8DReSg -KieO2ZYfiA1gVRiY6jPx+r7J9I5kSS1gwr/e3zHJHa79ijMB1SSIswQUmgSMkwGh -sNyDNI9ZxgJan3v7kVargMt2LiNcXvVyTzPSYSXcY7SoebfpMprVIG7vZ9TZf+Uu -FQeOfxdLFuGTnpFrYmvOD3OIKfODlY5t+TNICg7A3eTUXeJPcdBBnuVCiQU6TCB5 -H+69K5w54Q6a70sHZU1IWsGT8XtbUizPNQky+LAFsE/5oUnCqtypeEu4srcZK53x +MIIFdTCCA12gAwIBAgIUDITbsLT9hKser0ZzBZsxqgaZdWswDQYJKoZIhvcNAQEL +BQAwSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x +DjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290MB4XDTI1MDcyMjEwMTUwM1oX +DTM1MDcyMDEwMTUwM1owSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYD +VQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290MIICIjAN +BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3n +z6C9w63tD+w4S9wNAmCTNn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjk +W5C2EP86UDX9Eeq1tjbsnfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwy +LLwXNDnlMymgzflxa6+gAzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn +7jcGQbASKYjmSbuM0LWfCOIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CD +EW3Rhtf3zFb9pSqxyAFRWz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNN +D4xfMkO3Qm9fxIgk+ZsGeDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0 +kPnQm/aJfiXiB7gfUIgrX+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8 +hPCaZ+l+T1z/JY98Luy0eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuC +fREAGiNGgZ2GFjdAOKmDCZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO ++WbApbcXIwOHBgv/NfsnRzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfz +Iokr0sEBStgKMncCAwEAAaNTMFEwHQYDVR0OBBYEFKFadJx46upif1BrhYZ0iu8o +2z8rMB8GA1UdIwQYMBaAFKFadJx46upif1BrhYZ0iu8o2z8rMA8GA1UdEwEB/wQF +MAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJi4FwYJz/RotoUpfrLZFf69RoI01Fje +8ITt8SR1Dx/1GTPEuqVVfx0EYtOoH6Gg3FwgSQ9GHRDIa1vkHY5S+FUSOW3pCoZE +/kaLu9bmFxn+GntghvQEor+LzODuZKLXupaGcu1tA4fzyuI4jglVD2sGZtLk//CT +Hd4tOWXo5k1Fj0jMnJq+2Htr8yBeSAO5ZNsvtAjOUU6pfDEwL9bgRzlKKFQQMUYo +6x1FvRDRXWjpzB/H+OSqOaoNLEB9FfEl8I7nn6uTenr5WxjPAOpwjZl9ObB/95xM +p91abKbLQLev5I8npM9G3C/n01l3IzRs7DNHqGJTZO7frGhicD7/jNa+tkSioeJ2 +fIMqgDOvQE+gMxs19zw1tsI3+kqX7+ptTkU4Lan5V5ZKGfU8xtcVIlyRk5/yDUI5 +1dfQVubs6z07s6De2qa92LFz9l8sT6QuVer+c/wPPhBdMwbzcHyUJIBjFaBpxH86 +F7Mr5Zr/+qcbHglAHow1lBqdZzimqGd1koqFRat/pFUFh0iqktMmpl+ZUCjyoQEX +93j8aMU2UQjYM8NJDE2aRculo9OEoqERYFM2m3nHvrtE7iZgddryLNH7ZmC1EquX +MhZJ26GuZ2U4b9dAX858WTv0q1EF5S8KObMlxMU7IDk+cWlSD+puWliwfUKoTR/4 +JErSfjCSaRqh -----END CERTIFICATE----- diff --git a/spec/fixtures/test_certs/ca.der.sha256 b/spec/fixtures/test_certs/ca.der.sha256 index 4ce28e8a..7b37c846 100644 --- a/spec/fixtures/test_certs/ca.der.sha256 +++ b/spec/fixtures/test_certs/ca.der.sha256 @@ -1 +1 @@ -8b23238088af65cbae6ee9c23821068d896ec1dad081e2a1035ff70866943247 +d403930d5296f1515aadd3f730757e7719188b63a276687a3475128b746e4340 diff --git a/spec/fixtures/test_certs/renew.sh b/spec/fixtures/test_certs/renew.sh index 8fa81421..81de8a5d 100755 --- a/spec/fixtures/test_certs/renew.sh +++ b/spec/fixtures/test_certs/renew.sh @@ -3,8 +3,7 @@ set -e cd "$(dirname "$0")" -openssl x509 -x509toreq -in ca.crt -signkey ca.key -out ca.csr -openssl x509 -req -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr +openssl req -x509 -new -nodes -key ca.key -subj "/C=PT/ST=NA/L=Lisbon/O=MyLab/CN=root" -sha256 -days 3650 -out ca.crt openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256 openssl x509 -x509toreq -in test.crt -signkey test.key -out test.csr @@ -13,4 +12,4 @@ openssl x509 -in test.crt -outform der | sha256sum | awk '{print $1}' > test.der openssl pkcs12 -export -inkey test.key -in test.crt -passout "pass:1234567890" -out test.p12 # output ISO8601 timestamp to file -date -Iseconds > GENERATED_AT \ No newline at end of file +date -Iseconds > GENERATED_AT diff --git a/spec/fixtures/test_certs/test.crt b/spec/fixtures/test_certs/test.crt index 6f8abc44..57ffefca 100644 --- a/spec/fixtures/test_certs/test.crt +++ b/spec/fixtures/test_certs/test.crt @@ -1,30 +1,31 @@ -----BEGIN CERTIFICATE----- -MIIFEzCCAvsCAQEwDQYJKoZIhvcNAQELBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV -BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD -DAZSb290Q0EwHhcNMjQwNjI1MjA1MDU4WhcNMjUwNjI1MjA1MDU4WjBTMQswCQYD -VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF -TXlMYWIxFjAUBgNVBAMMDWVsYXN0aWNzZWFyY2gwggIiMA0GCSqGSIb3DQEBAQUA -A4ICDwAwggIKAoICAQDGIT9szzhN5HvZ2nivnCDzVfdYbbqBhgEbPppWPyFcV0r2 -rtmWfeK5EEdsTS/Ey4owTceOplPpAp4svF+a/i1/bHhqnQYYU4f7Qic4fDAszLdi -SIo0o1csNvIogm/P+uvSzE6eZRZUSmo49dY5SKSJt6Pjh6lM2MHEjsPKIKdAN57w -EN90q4IZv6AHE9rphqxcmF1k+j5xmhCUS1EJ+y7hyZ0S7Hghdgp/0cxSu/7YlVYy -JpkIlQd3RPXzEf6VSYjtr9Ajp1rhvv2611q0CB5NALg/KR3OiMPYmTg5HAKOdweN -am76nG3VxTeV3y+LW/pZAbi4qAl+4/c0eOGsL7o/YSn7qhThU1AWS9kY1WxTCrKR -h58rUGRfmvpnOR99xvR4jz942RNiY61pTmsvo+iJspTII3GZhwIGlHtxE9Rn50lW -QcDuDDHfObWhzb4rS55BERIwDUqD1LgCRd0ikRxPSvI1AM4cl35b4DTaDLcnM6EO -fy+QTYsgNoftU1PI1onDQ7ZdfgrTrIBFQQRwOqfyB4bB2zWVj62LSDvZoYYicNUe -cqyE1542WNKzmyE8Mrf3uknN2J6EH7EhmiyRBtGg3NEQCwIYM4/kWPNPOtkSjsn3 -cNbMNUZiSnQn/nTs4T8g6b2rrwsay/FGUE83AbPqqcTlp2RUVnjbC8KA5+iV1wID -AQABMA0GCSqGSIb3DQEBCwUAA4ICAQAlB7YFw7e1pzYz55Kqa91juTWP5XiCn59n -J0YKM++vdHqy224HZb9jGtJjZ+0Wod4cwiOVWm+5hLs4IrzfGuXFZEFx/VWP3SDq -4F3IJJXQkc7jSNrL6IR92xRDSB+yFZZI6FFsnaKMT2fZELndPVFnH+oclc8ZZoyz -2H/r1CT4yYx7YclAWUqq8Ci3J82qUeeM8Xj9fzGFKy6oCoRsApQb4qb4DoQ1TbZC -b8gWxHj8l4izul1MtTzSkoMb0Ot50vMoT69m1hDz5H4wF6KuAZUAgM9LQWNHJCkt -hlOXvqFTHF+y+bvK+hGs976xViq3HA45M3+5Psv0+fdoHgYQJvd23yt8CM0rGfv3 -P+34HlLCW+FdWiazmo+tl5YmtGs6pYuAEp2z5pmUO2l2CutFmv4xBOvXF+rZOzxY -Q0ackJtflnDC/Tlq2qAldY3Oa8nyI3UIaMUcqHemwm5KpDjc0XF2J1qCoSrMxD8+ -L8HdvUYlh3DIFgJIG1DlTtfQO+RwrVi9+NBBGAsforla9HJDO/POiv7O9hED71u+ -pev8flmULeisMeYqeiL55jyS/+45VaF7t36FMyiP3zXANwbHZMvzVobEsXAuzPOt -pVNo/EpszrdBe9JWt1GrFLY9c14FmWG8cAWpcwRH0ofhJPPvEB7usFVWCSduOAbA -Zytzb+8iSw== +MIIFWjCCA0KgAwIBAgIBATANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL +MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL +BgNVBAMMBHJvb3QwHhcNMjUwNzIyMTAxNTAzWhcNMjYwNzIyMTAxNTAzWjBTMQsw +CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE +CgwFTXlMYWIxFjAUBgNVBAMMDWVsYXN0aWNzZWFyY2gwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDGIT9szzhN5HvZ2nivnCDzVfdYbbqBhgEbPppWPyFc +V0r2rtmWfeK5EEdsTS/Ey4owTceOplPpAp4svF+a/i1/bHhqnQYYU4f7Qic4fDAs +zLdiSIo0o1csNvIogm/P+uvSzE6eZRZUSmo49dY5SKSJt6Pjh6lM2MHEjsPKIKdA +N57wEN90q4IZv6AHE9rphqxcmF1k+j5xmhCUS1EJ+y7hyZ0S7Hghdgp/0cxSu/7Y +lVYyJpkIlQd3RPXzEf6VSYjtr9Ajp1rhvv2611q0CB5NALg/KR3OiMPYmTg5HAKO +dweNam76nG3VxTeV3y+LW/pZAbi4qAl+4/c0eOGsL7o/YSn7qhThU1AWS9kY1WxT +CrKRh58rUGRfmvpnOR99xvR4jz942RNiY61pTmsvo+iJspTII3GZhwIGlHtxE9Rn +50lWQcDuDDHfObWhzb4rS55BERIwDUqD1LgCRd0ikRxPSvI1AM4cl35b4DTaDLcn +M6EOfy+QTYsgNoftU1PI1onDQ7ZdfgrTrIBFQQRwOqfyB4bB2zWVj62LSDvZoYYi +cNUecqyE1542WNKzmyE8Mrf3uknN2J6EH7EhmiyRBtGg3NEQCwIYM4/kWPNPOtkS +jsn3cNbMNUZiSnQn/nTs4T8g6b2rrwsay/FGUE83AbPqqcTlp2RUVnjbC8KA5+iV +1wIDAQABo0IwQDAdBgNVHQ4EFgQUb789MhsOk89lMWwSwBss1TLXDFAwHwYDVR0j +BBgwFoAUoVp0nHjq6mJ/UGuFhnSK7yjbPyswDQYJKoZIhvcNAQELBQADggIBAI+G +NKZ3s3m+/R4mH3M84gGWPE1joC2bLavYYLZjKnZv18o6fHX0IW/8v5hd5Df3SP5u +vhjC88bewiKVHldqkC6ju9rbZxQynhFZGXbN9zLvFMZGkfRH5vB2Y13ZWBdWhq5L +cRxpRk6WlwaSy0Ed4F12u9ERmhMOtSZhqAnNJBeVraOHeGlcMZXJdZkeyxkdcZ4y +YJcrI8Da0dMxILgIuc9ZCynAItRAjMw1/3wjlx0Cyxif10ct+EFiP6Zv/gzoo05v +tNeqOCrxAqAcwrS1u4q/KAKySiEIyxyU1nEI/g53nALwoQhFsRVqVXNAoy7xu37y +o+lvs98rkq/NkkbBvRBPdcF/BYNtesRxKja/QAEvslyZfyICL9oqsuPPEB2nHtXa +mWntT2NaXyr1FWCxHaXfZQOxSwco3vTk7HLuNug2wxIc/hewkLlk5NCRkAYfTlan +gLhZ3vBej4oA8cdpODMb8SrYhqKTeX8E+ulHVS0paY0kszAGK2x2kHqRGNXUlfoB +Ax0etGudHhgtTCAmUgJDyQNLkvBKHYQJ2V/Wv/xej7wXKkACNKlRORl8zcnbVErd +GM/ibfqNIPIo8dP2EDycSV6vIICqkxpCZZObNjfgKa0UN03qYi7xREhhEehXgU8H +IO9w2pG7ReiO2E+bLIs0Zh1+2IwlM1EM/eqbq+Gi -----END CERTIFICATE----- diff --git a/spec/fixtures/test_certs/test.der.sha256 b/spec/fixtures/test_certs/test.der.sha256 index 8e18e957..fb0137e9 100644 --- a/spec/fixtures/test_certs/test.der.sha256 +++ b/spec/fixtures/test_certs/test.der.sha256 @@ -1 +1 @@ -80329a197063dea8cf7905d10d221648bbdbc05b8fb1d4c2e384b831bc6590df +386ae6ef809d20ddfcc7ca68f480e82007c031b365c86cc58922cf1bd7238f89 diff --git a/spec/fixtures/test_certs/test.p12 b/spec/fixtures/test_certs/test.p12 index 0c06248e1bec460a5428121a77bd1750ccff04c1..579c68ec21febddb9faa229d0c817e7a560c3ae7 100644 GIT binary patch literal 4291 zcmai%XE+-S_r}HEq7`b@jM@^kX6>j^MTpulLX8%sw#SH~X4R(l)|%DSsI6kJDCHqG zwMUHBe*0ek|Mff{-w)?H*SYTx=Y0F!2Z9E+69Y&fXrMPam2jk1{bRk*+gmq97sq3!lXc!Uw17Wdu(b1&l(H*3&QS& zqRBdLQkBwi;*(x>(zAD;skA+RQk)tt3!PG3JP%Rba7^8XDs%R6EUlfspO9CiT{cfB zVbms7sjD3u#|`?u%_&(Y^_6LuHC8Tyeyu2`tM-+peUwn@9OgFtgI3*BM^`EE^r1P& zt<2Sj@8Xm;KL55?8z)LlagE-PX$h*cJ`oH%er8Dami0&INoy3S;ilvVgBGBQs%hcb{9h zzHE*{JvPrAM|TDDzJQ1SHscGF0%nUBiX@Jj%F8Ne*ob==nEA3jX@;zI zBDg2|7{@qUv_55Q>z4S^NJklEd#uGijE4=`@POT<1TFfH4=&eXQbJ&gF3GVH&wFy7 z>zbQJ8NXVvIwNP{1#6|xr>d`Bg#yiRs#|FQpJ^R$5_hzhrMk)EZ5VlASx3bB6t1zc zvdXHhkp5|7#xo${Cnot1Pgc6wK~!>ULNuN2sj-2c-7Qorw0!5oxxIym%yU< zNz9^Uhsf3!1_fIA5ytkZiq0V~IbxClA zOPrgfg9u$%^uah!x#gDM2bu>%okJD%K&WLQ$sJ4*MohFf+wlDsuvx>}@iIeqsTRmR zAwJ{j_jV;ylW{0>S!L=5EOd5q96Z5T&eQ>xQ?tIL&1y>sH%8RavMg0r76ffYaE^T( zmu_37IX!yXRR}e5$s&^xljXTR4-%qOKl$v+^n-R2nmwZH?S;Th~mn=^+m%^Xm2>t(kX9h04pEah^& z%=P5H33`l6-YRum6gYHBoSw4S9iH9~*%fs=su%qn*D1}dJ-d2vh3~oLvMb($qB_5t zym$RglFLtRQe@7&Yuo~Rrm$xVgF`t9llEvn2)QJ{#GK)(_rmR=md^nqoGD$!ur+mi&Y$-8{eV z?M+!b- z0Vk|ycu5IuX4A&ih%QdUdOyr11s{YrNT*de8SyX|=@LR=G*PB2N3n#>c=p2} zrzQ^dc{pe}>ilYZ73n*%^%NOrZyPMDVioPO(3@(Q>jf%PWcZbqGwZkTO+25e4672y znWQsgF5u9zpW#Q)2^mlImOms}Q(&6xICtT;AgJ*UF`q(f5CEtGSf+3p^op4(L2aRX z_N+ZQeik#rec<819T%OV?5_$zQ`7##h5v$v>WUOibp}CGo&2doa+?2V5G^%;_>bB2 zN5uRuuuR?O<}{SuX*@2o^8+Ugll@PyWS#SgIksS{Bf3qO`oh3se=aaU5@k~)pb!;< zHW;Ux+Z#gI2=QOC1~`DV4u^$8gw!Ty(g9lNz_bW5QFg93n_YaaOu=4xv@c~85 zK*)5=4x9XFDl?4OH>{9TT>oblvlIDk*WQi1fmW$Oq7NC$-gMnNX(;+y*i&+b*B+bR z0Tm`JY3oYFjr@DWV#dXTR} zP_E6Cl$0HzFVZzcBhTIXB?z6^81AYfFDbaP3axrKY^YmT?gnD%!e@6=+$@8ip7*?{ z+^D7t1e+6F0^`eFHHi7E>=tSyknvMrXe&+hoisZE{LtW1kcTI`(j>c%xW_akEU?fo zMQ3eu`XS{?fsNpvdk;>9H^re^GJA>5am85Q;=lpo>0Q%)(+AXwfdZ(`cq4eO%9pF_ zMYan>OH-hi8xQ}US{xc+ItL#zaCFmflc#m=RcFvSMv~$mFM;S_Fs+4Y`R!j zvFx7+wy+%9B59sXd=Xm+cSCNRv8u!0CA*FeVd|upcS|K~IxgDA2FMlKlLr985feMq(Q{>E<8Xal z(N({L#eSB`RxXajy@rbvdXi$p^j_Z>1A&!$Sw8nxGFHNYoriA=CBR)dPmr#q7sn|K zT7tb7b(vnL*GEzeT~WT$<=v)KbNX!5Xy-Oy(`mgyRQA%xNamF)J-sHOLIs~q&oo_P zhj(=!XlTZSY)RSW*A20JCp?I;MGU-*w#h;G{?m@fx)`EXF{79M{o>Ut1%G+xC|$F) z@+l0S!Z))=t3I+xHRCS!d6QhBw$*P6?XVjjFOzwL9`37S5FNiYKC3DN+#N@X*iWU? z4hJ(D$DA^xLXx#Yw>x}XXen7E9K_A=EQU+_XYqDSN~^#J50AA;X-vSuEQwMdG_kCs zD2m;6+B@H+f~y3rqp@mLutmv;39&N-g-2{%Peh9(oLk?&??~hk7&JT2E?7i27Mwc0 zrSkedNv-KpS5UE|X5Iy?d2L{oze;SJH2@y3CP%Xm^tmg3JCC<~5mEIw%UOIxb2-hk zg+tS=u@tG7kO2V_2R{0%k<%_XAFn{p);~mrnyC8S3-lU^YH65;!Z>K;oddsZ%3XaQ5I&I_p)gcYTmnpLT_;pI?1^Rwbm8tE~n6+nD+%Mv!okozZ`PuE{yi5sw z8~%_zbvb2wp3Y}&s>W)zz9Qe}~dC91lA_$T`@x zohZy8G;4F(w`hCg7kTS!-hNLr|gqULQLeKhF?Kxz-Em16EjipAvch2ux8 z8>KIn7t}^~nCaz#xI6eaBsE(qzAJJPQfMoA{WrE5GN`gUor`fz9%+308v2Yx6Pe_B zI`psC<*NPav&a-^H^%6Tt!-;bFRzb1iOqi;sUC+jiXGJEyR+GT#cP%Y6ns?a2WU-W zGJZO-dP%<1y`7xn}<~v1&GHBs~N~TINH=NLZqyF^yG_W zpthsvmwVk*gu$n2C_%aV{M&vPa&)>cE7Kau6pZ&@?->96AC;A`j2G+bUWg#3m^0r@5FOjd_yzN(X@5jYkY}@sK zsDcH-r)p#m7*43`3Jvs5%Zkm6Oav=zG>4zRnwxjN6Zb26y1rbbWX@oQ)S?Dp^083Z z#gC4SaK^08wY{rcjZM;|$sUb}4W#um6)$Vs>!PgcVNZ%dE^^J-!!j|cOQmZhOKMdM zs3;S4j2?+Vv-$MK8w_=%Q!*rwbxXCoAEv6xl{f4=?RB!m#vg$ZejZ|ws$60)#04i_ z1LaPwthIehUB@{XQPhGj(lisP~aa!6Ft^k_~2X zI@4rfujEVhYM1GY7PU`%L#FZHA7uK8f#hJV`yq9gwl>NXcyviH zbr|a4w|yp1#AM{(xe`J$gKrUA_G-ePr(}7wG1MPCux$lJ-}P?@2-$s`8*$=h#L9@2 zk2|&wx~ZbRebZh06Qd_LzftS->DiMwWX-`PtcIHg)Kh$;jMq5J3UCkBuak#YybDI@LI$JBqI=CS^vAGs76Pd7$I_r|~P Yf2z{sf8a?nmj;QY5d@Zyl@_F90ZHlZ6sZNI1d(ou zB?T0a>pOGr{oniH%$a$9bI!+Seqbc!6fOV=MpC8|5DP|YL|>5u2mxp$WegsYGV(7D z10(TU{u2?Pk@#hQ(Fa@r&fk^$p9BDAB_R3l4`cuWFa;hFs1yBTe1(P`7ncYp42kbq znz}@}zLP@Irxx8R)i@=MOI}kKa3;?=>uF(_*`+~qw;Hh=G5ytl!Pkj`O%226%WE~R z4F_$34FP=CamooLZvDl@BKnQV%i`(|0|j+?e*VN4sIc_vEkb<4x@nZ*w_NBQ z!YmW6d@N@Tu4jL06}|p|vCYb~khztABPD_Wvar1neh?H?=QbN>(sh=W@-~fNROkU! zZ$JW0=SRHscL5HNdp6cFP1g}J)iVv;Q#w`;u3OS4O!$T7v2G)+WUvKqx*kIg78TxZ zfOH@$>jsAtkF>k5a14Wm;^qclcP|Dus#=1Sj&fLD08tCbF~b783j}0^PlWgt6t9)o zj@mNsyobqb+qpC$Po7Z3p1)jXY$%@I?8@G{ZfepNmQ61*FnOJ%vx!!$MwdT>F%?bjXYs@!st0|AfuT`CVMRaNQZ zi9QxrG7m6=%3;QM#U$9}-vo~3QUTnaBy$FPaj~Zgsm5__C}_L-gxQ#8atN!nL{@E* zUjA2qb3uJS>CpR!Huh<-cFd2VEUdgBrBaBkj>zhF&zrvIM^dsO z@q7+BYsjBbgWp&Ql=+G(>vwInsw08{)zR+%G`8K_Y2%k*9vwxxX_&bZ416e@XIk^; z83?n*TrW@&WJ97TUeh4wix$JS&_>iL=Cl1D68)y1sEt7~^o|AR;Qh{haIdqCl+%Y6 zxd>trbh6ss)KY%;v_}9(8!f`C9hHW+|Fh!z@Tk|ZAleYsSzPtYChf@@>r+n2g~5zm zbuKZ*#xrQ)BZO%(g3a!D5u-VEZst$jkVF1aOfbPuvG^E|+8b0x*sb%iL$mpN$UU{a zX+E8;$DK{0$!C&T!==|=h9BvGSizn^l5od=Y-;kTgwcr|kk2Y_aH26ba=Ze3DPE<$ z;&i+VH_gObmNTB`uOhP@8e1&Ah0{Wv{z-lTX2=M@?L%bjepJiaeHBHhJjo|}E&^7t z@fav89TQIXUo!?td(od1nxG^F_cC|}ZQk!-(LU>Mz|N`kt?Dan3?0Oc`g7NmTJv{iJu7ZB_F8En#+=>R9hLVC~+r2w~~j3S&Y_U9)9bd z{_mbyJ0zp1Njw{>F_}B+AxHpZQW>pnhUZ2&8z?TJ%)-6FO;WuF@ODYiGqPa2kGf&F6lhBedJC^g{Y~LhS|T59?nG zl${U28_Q(xbX+(M;xKbGHsSAyyTVCMcD);Q(Y{Y9R6iK9CDp>sG z0x{0u&d2GJcjJUch&^(maX;-Ix4!FiN9XUYNsU|748b*=z)0f0|08HLlDHlZNnHIG zSNxq40@DA{H3rs~=&6zN+2k7d^XTbbPC*CC3)_r=rt&7cRQzJ<_7CWiXNIpe zv>G<9WP;I)Cu@D5>_4|-9&-diSdx}b)Dpd`y@a5l(RrEn@EKEDV zaIj=qo=aQT$B$NLi@pYjbzyvm*qL9c=g;Wg&g}26Rr+ivLd=~@Y7LwU7~Ueu|CrFT zgWP851$|HmSfm!<%>fjnhC6U&QYB_oT&92cUIi4Va8C4zEt`I23FLU*l^Ch5-RYs2 z9#Wy5x)IK!BlCy~n(*82>G#;&#;>h{89Y9Ac1eN11TQ^}BDO2w26GAy-EWkunbQ1x zufU8(X{XNltJH1&>hxLJLZvgC5e7D_^$#v!s&#ZN(khGrNR@O-j~p>6yHEf zxqr4Hf+|1L_muGQj}{@$iB&J4EJLmH!g%(xg&7<9z5P3hXbPVnM~*qa)MbX;cna!# z(PYf^5nQD&)>&kcW5hjY4?TiGFiGblkEfSGto4~UpXyF(T){ijXc>=!&wQh#qWqUR zS>S`O%(*Y>tX7*|2iW^$7~)Rk;(eH?f{!*^_^GbG!=3JvczTrEKyo(xMzkJQy5cV( z!zQ1zZ7yIe@?OoSahpe6NG*yy`GS)A4lIztlCALR9eUF|zADyhF!}J$z$UDaUvDB# z8Q2EwTeV@hwQDU#J4X^knr0rn4@e|647|^8XNaG}c@@dq8<9doVNz6QjGmaSbsxBDT?A)&gLu&!-CcZ6 zd4CPn;j4XY4SneYWfl@O9hob9zAG4bz$tShzr%bn0>^END!IURu=kxB8Rh6>FIGHG zn|FDqdq?M#_7F!;EbDx=eh|eCzG^lhNV?C^_aKYTgUX7WjEW6%n@v(FP*~%n&W}@z zP)b=+gx(*X7BNPjBbq=Bb|Iu|axBiHo@%yKC`Q3c`u>fH_Z+uQk_Q@s#Ojhnq?{(Q zZDqp@h+<7a#P*Z4jrI7~=%Z$>@zIcFm7IP|%;2*m7X~Sf@WoX|YqsCy?j!|?GJots zq$nB@+9eA^Auh}zWKva!4QuS&ZR*jXXIQ?(Z*vRgTDKU_*A6bj@gCpXDhA-9XfpjH z2JZ$OW|r#GdKuV^-WN5$w=kaHZoQ6nfjAOaUzbCxN6E5yP>QZE+WEg&hr8gO___Y2 z7#+yu1LOy*LK4(yaCr{+qAaDPeesGFY{8+)QXxw=^*r9DCVRzTg1-ho+svsO0j64>Y z3r_Okm<<~1>F?L)Iq83W&GvcUI&!pGB}tFx_;9bx-9S7eWUR2+qfvv1PqN9P_2tFT z?*mxV(xFJZ!6W4Xo$<3AyFVT&o5WPyv}H$MZ>@xTl?~q(ZEZ2CC7H-idva<;RNk!%#IOZVg|{f?_e%k!pBKc`2VQ2d zAEJzvN*n_G40)mYC|mrLxLR$J!%L?oc4|sF6wSs(Ke$i6>;bBZB8R_xvrswDogy|w zm%01~-dd_-A9&R2H#)Wxy5*`1^wa5=&L3gv#?2Gb=L*@Ym}30%!;KU%ZyB?_O^+%x z8w)r%b7B>+8=DjR7JI!_W0$I!5clN3{;Q8tEO-Sxx!;XVt4c-p@H54hR(|-Ce9I`4FyT#`r+!3A zjx{P-xtgT#3f7q?56+&#)QoSFvs{k??B31*yJiF*>Sx)aDu7GFLLv4Ng-lx38-|;O z9?_#-=q>G#d|q9Vt0245gb}TYfls@+-&#kYvW64IJ8j-+f2gIS+gO!!yg>ZU5)O|; zBrl6qEx`u+lRYt=^5hj;P7+McN|oqn9&^kYg#U7A_4+wt?2WFCijreg z-Q%k?5r;@6&gph7vsY-rcXkUqBch(R5&8)>>uG36=1ErFg6`#VOoQ(>SW2s2#NX}N zN)K&M)ScAUpgAw8)J)SunJ2!F+(g7W9L?JM?WttA*8E)E9}KT*s*&aK-d6=(GYLEG ztJXja{Szu7Su=FZ9Qiq{UqQB_D#2~Tgx4{_Y~1P15YJ+G8P-s=Whyuz=7F*h=o9(# zZQ#zk#lEyO2qZKn;L8%a>m5SD{_v!L;cL34ffv=EH@({z+Om3w>U=wiRqv zvD&Sl7W;BHE3BsHy~5*><@@QofS8MjtOhz;BypzM^f#3iIZ@{DpiMObi~7Lrt9f^! zC10BqAV)lc<(uS "gzip", "Content-Type" => "application/json", - 'x-elastic-product-origin' => 'logstash-output-elasticsearch' + 'x-elastic-product-origin' => 'logstash-output-elasticsearch', + 'X-Elastic-Event-Count' => anything, + 'X-Elastic-Uncompressed-Request-Length' => anything, } } diff --git a/spec/integration/outputs/delete_spec.rb b/spec/integration/outputs/delete_spec.rb index 9fa8afd1..5d410427 100644 --- a/spec/integration/outputs/delete_spec.rb +++ b/spec/integration/outputs/delete_spec.rb @@ -39,12 +39,12 @@ it "should ignore non-monotonic external version updates" do id = "ev2" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)]) - r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) + r = es.get(:index => 'logstash-delete', :id => id, :refresh => true) expect(r['_version']).to eq(99) expect(r['_source']['message']).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)]) - r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) + r2 = es.get(:index => 'logstash-delete', :id => id, :refresh => true) expect(r2['_version']).to eq(99) expect(r2['_source']['message']).to eq('foo') end @@ -52,12 +52,12 @@ it "should commit monotonic external version updates" do id = "ev3" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)]) - r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) + r = es.get(:index => 'logstash-delete', :id => id, :refresh => true) expect(r['_version']).to eq(99) expect(r['_source']['message']).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)]) - expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect { es.get(:index => 'logstash-delete', :id => id, :refresh => true) }.to raise_error(get_expected_error_class) end end end diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index e6c2dce1..0262ab42 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -102,7 +102,7 @@ it 'should not install the default policy' do subject.register sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) end it 'should not write the ILM settings into the template' do @@ -282,12 +282,12 @@ subject.register sleep(1) expect(@es.indices.exists_alias(name: "logstash")).to be_truthy - expect(@es.get_alias(name: "logstash")).to include("logstash-000001") + expect(@es.indices.get_alias(name: "logstash")).to include("logstash-000001") end end it 'should install it if it is not present' do - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) subject.register sleep(1) expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error @@ -298,7 +298,7 @@ subject.register sleep(1) expect(@es.indices.exists_alias(name: "logstash")).to be_truthy - expect(@es.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001") + expect(@es.indices.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001") end it 'should ingest into a single index' do @@ -340,14 +340,14 @@ let (:policy) { small_max_doc_policy } before do - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) put_policy(@es,ilm_policy_name, policy) end it 'should not install the default policy if it is not used' do subject.register sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) end end @@ -357,14 +357,14 @@ let (:policy) { max_age_policy("1d") } before do - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) put_policy(@es,ilm_policy_name, policy) end it 'should not install the default policy if it is not used' do subject.register sleep(1) - expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class) end end @@ -374,7 +374,7 @@ subject.register sleep(1) expect(@es.indices.exists_alias(name: expected_index)).to be_truthy - expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-#{todays_date}-000001") + expect(@es.indices.get_alias(name: expected_index)).to include("#{expected_index}-#{todays_date}-000001") end it 'should write the ILM settings into the template' do @@ -443,17 +443,18 @@ subject.register sleep(1) expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_truthy - expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001") + expect(@es.indices.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001") end context 'when the custom rollover alias already exists' do it 'should ignore the already exists error' do expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_falsey - put_alias(@es, "#{ilm_rollover_alias}-#{todays_date}-000001", ilm_rollover_alias) + @es.indices.create(index: "#{ilm_rollover_alias}-#{todays_date}-000001") + @es.indices.put_alias(name: ilm_rollover_alias, index: "#{ilm_rollover_alias}-#{todays_date}-000001") expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_truthy subject.register sleep(1) - expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001") + expect(@es.indices.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001") end end @@ -532,3 +533,8 @@ end end + +def get_expected_error_class + return Elastic::Transport::Transport::Errors::NotFound if elastic_ruby_v8_client_available? + Elasticsearch::Transport::Transport::Errors::NotFound +end diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb index 02f3fb76..19197329 100644 --- a/spec/integration/outputs/index_spec.rb +++ b/spec/integration/outputs/index_spec.rb @@ -215,12 +215,22 @@ def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false); it "sets the correct content-type header" do expected_manticore_opts = { - :headers => {"Content-Type" => "application/json", 'x-elastic-product-origin' => 'logstash-output-elasticsearch'}, + :headers => { + "Content-Type" => "application/json", + 'x-elastic-product-origin' => 'logstash-output-elasticsearch', + 'X-Elastic-Event-Count' => anything, + 'X-Elastic-Uncompressed-Request-Length' => anything + }, :body => anything } if secure expected_manticore_opts = { - :headers => {"Content-Type" => "application/json", 'x-elastic-product-origin' => 'logstash-output-elasticsearch'}, + :headers => { + "Content-Type" => "application/json", + 'x-elastic-product-origin' => 'logstash-output-elasticsearch', + 'X-Elastic-Event-Count' => anything, + 'X-Elastic-Uncompressed-Request-Length' => anything + }, :body => anything, :auth => { :user => user, diff --git a/spec/integration/outputs/index_version_spec.rb b/spec/integration/outputs/index_version_spec.rb index 0b1ecda9..d73872f4 100644 --- a/spec/integration/outputs/index_version_spec.rb +++ b/spec/integration/outputs/index_version_spec.rb @@ -36,11 +36,11 @@ it "should default to ES version" do subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")]) - r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true) + r = es.get(:index => 'logstash-index', :id => '123', :refresh => true) expect(r["_version"]).to eq(1) expect(r["_source"]["message"]).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")]) - r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true) + r2 = es.get(:index => 'logstash-index', :id => '123', :refresh => true) expect(r2["_version"]).to eq(2) expect(r2["_source"]["message"]).to eq('foobar') end @@ -63,7 +63,7 @@ it "should respect the external version" do id = "ev1" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")]) - r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true) + r = es.get(:index => 'logstash-index', :id => id, :refresh => true) expect(r["_version"]).to eq(99) expect(r["_source"]["message"]).to eq('foo') end @@ -71,12 +71,12 @@ it "should ignore non-monotonic external version updates" do id = "ev2" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")]) - r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true) + r = es.get(:index => 'logstash-index', :id => id, :refresh => true) expect(r["_version"]).to eq(99) expect(r["_source"]["message"]).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")]) - r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true) + r2 = es.get(:index => 'logstash-index', :id => id, :refresh => true) expect(r2["_version"]).to eq(99) expect(r2["_source"]["message"]).to eq('foo') end @@ -84,12 +84,12 @@ it "should commit monotonic external version updates" do id = "ev3" subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")]) - r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true) + r = es.get(:index => 'logstash-index', :id => id, :refresh => true) expect(r["_version"]).to eq(99) expect(r["_source"]["message"]).to eq('foo') subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")]) - r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true) + r2 = es.get(:index => 'logstash-index', :id => id, :refresh => true) expect(r2["_version"]).to eq(100) expect(r2["_source"]["message"]).to eq('foo') end diff --git a/spec/integration/outputs/painless_update_spec.rb b/spec/integration/outputs/painless_update_spec.rb index 02116fa5..0f4eabe0 100644 --- a/spec/integration/outputs/painless_update_spec.rb +++ b/spec/integration/outputs/painless_update_spec.rb @@ -22,11 +22,12 @@ def get_es_output( options={} ) # This can fail if there are no indexes, ignore failure. @es.indices.delete(:index => "*") rescue nil @es.index( - :index => 'logstash-update', - :type => doc_type, - :id => "123", - :body => { :message => 'Test', :counter => 1 } - ) + { + :index => 'logstash-update', + :id => '123', + :body => { :message => 'Test', :counter => 1 }, + :refresh => true + }) @es.indices.refresh end @@ -46,7 +47,7 @@ def get_es_output( options={} ) subject = get_es_output(plugin_parameters) subject.register subject.multi_receive([LogStash::Event.new("count" => 4 )]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => "123", :refresh => true) expect(r["_source"]["counter"]).to eq(5) end end @@ -57,7 +58,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' }) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true) expect(r["_source"]["message"]).to eq('upsert message') end @@ -65,7 +66,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true }) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true) expect(r["_source"]["message"]).to eq('sample message here') end @@ -82,7 +83,7 @@ def get_es_output( options={} ) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true) expect(r["_source"]["message"]).to eq('upsert message') end @@ -91,7 +92,7 @@ def get_es_output( options={} ) subject.register subject.multi_receive([LogStash::Event.new("counter" => 1)]) @es.indices.refresh - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true) expect(r["_source"]["counter"]).to eq(1) end end diff --git a/spec/integration/outputs/unsupported_actions_spec.rb b/spec/integration/outputs/unsupported_actions_spec.rb index 1ac4f9e3..c3b12f71 100644 --- a/spec/integration/outputs/unsupported_actions_spec.rb +++ b/spec/integration/outputs/unsupported_actions_spec.rb @@ -27,16 +27,21 @@ def get_es_output( options={} ) @es.indices.delete(:index => "*") rescue nil # index single doc for update purpose @es.index( - :index => INDEX, - :type => doc_type, - :id => "2", - :body => { :message => 'Test to doc indexing', :counter => 1 } + { + :index => INDEX, + :id => '2', + :body => { :message => 'Test to doc indexing', :counter => 1 }, + :refresh => true + } ) + @es.index( - :index => INDEX, - :type => doc_type, - :id => "3", - :body => { :message => 'Test to doc deletion', :counter => 2 } + { + :index => INDEX, + :id => '3', + :body => { :message => 'Test to doc deletion', :counter => 2 }, + :refresh => true + } ) @es.indices.refresh end @@ -63,12 +68,12 @@ def get_es_output( options={} ) rejected_events = events.select { |event| !index_or_update.call(event) } indexed_events.each do |event| - response = @es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true) + response = @es.get(:index => INDEX, :id => event.get("doc_id"), :refresh => true) expect(response['_source']['message']).to eq(event.get("message")) end rejected_events.each do |event| - expect {@es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect {@es.get(:index => INDEX, :id => event.get("doc_id"), :refresh => true)}.to raise_error(get_expected_error_class) end end end diff --git a/spec/integration/outputs/update_spec.rb b/spec/integration/outputs/update_spec.rb index 4a86dc77..1c28f666 100644 --- a/spec/integration/outputs/update_spec.rb +++ b/spec/integration/outputs/update_spec.rb @@ -22,10 +22,12 @@ def get_es_output( options={} ) # This can fail if there are no indexes, ignore failure. @es.indices.delete(:index => "*") rescue nil @es.index( - :index => 'logstash-update', - :type => doc_type, - :id => "123", - :body => { :message => 'Test', :counter => 1 } + { + :index => 'logstash-update', + :id => '123', + :body => { :message => 'Test', :counter => 1 }, + :refresh => true + } ) @es.indices.refresh end @@ -40,14 +42,14 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456" } ) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - expect {@es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + expect {@es.get(:index => 'logstash-update', :id => '456', :refresh => true)}.to raise_error(get_expected_error_class) end it "should update existing document" do subject = get_es_output({ 'document_id' => "123" }) subject.register subject.multi_receive([LogStash::Event.new("message" => "updated message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => '123', :refresh => true) expect(r["_source"]["message"]).to eq('updated message here') end @@ -57,7 +59,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "123" }) subject.register subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => '123', :refresh => true) expect(r["_source"]["data"]).to eq('updated message here') expect(r["_source"]["message"]).to eq('foo') end @@ -94,7 +96,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' }) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => '456', :refresh => true) expect(r["_source"]["message"]).to eq('upsert message') end @@ -102,7 +104,7 @@ def get_es_output( options={} ) subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true }) subject.register subject.multi_receive([LogStash::Event.new("message" => "sample message here")]) - r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true) + r = @es.get(:index => 'logstash-update', :id => '456', :refresh => true) expect(r["_source"]["message"]).to eq('sample message here') end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 207d83d5..e84fe761 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -8,3 +8,11 @@ module LogStash::Outputs::ElasticSearch::SpecHelper RSpec.configure do |config| config.include LogStash::Outputs::ElasticSearch::SpecHelper end + +# remove once plugin starts consuming elasticsearch-ruby v8 client +def elastic_ruby_v8_client_available? + Elasticsearch::Transport + false +rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available + true +end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/get_alias.rb b/spec/support/elasticsearch/api/actions/get_alias.rb deleted file mode 100644 index ef4ebbd4..00000000 --- a/spec/support/elasticsearch/api/actions/get_alias.rb +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -module Elasticsearch - module API - module Actions - - # Retrieve the list of index lifecycle management policies - def get_alias(arguments={}) - method = HTTP_GET - path = Utils.__pathify '_alias', Utils.__escape(arguments[:name]) - params = {} - perform_request(method, path, params, nil).body - end - end - end -end \ No newline at end of file diff --git a/spec/support/elasticsearch/api/actions/put_alias.rb b/spec/support/elasticsearch/api/actions/put_alias.rb deleted file mode 100644 index d0585934..00000000 --- a/spec/support/elasticsearch/api/actions/put_alias.rb +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -module Elasticsearch - module API - module Actions - - # @option arguments [String] :name The name of the alias (*Required*) - # @option arguments [Hash] :The alias definition(*Required*) - - def put_alias(arguments={}) - raise ArgumentError, "Required argument 'name' missing" unless arguments[:name] - raise ArgumentError, "Required argument 'body' missing" unless arguments[:body] - method = HTTP_PUT - path = Utils.__pathify Utils.__escape(arguments[:name]) - - params = Utils.__validate_and_extract_params arguments - body = arguments[:body] - perform_request(method, path, params, body.to_json).body - end - end - end -end diff --git a/spec/unit/outputs/elasticsearch/http_client_spec.rb b/spec/unit/outputs/elasticsearch/http_client_spec.rb index d4cee8f3..c2a93ac1 100644 --- a/spec/unit/outputs/elasticsearch/http_client_spec.rb +++ b/spec/unit/outputs/elasticsearch/http_client_spec.rb @@ -270,6 +270,83 @@ end end + context "the 'user-agent' header" do + let(:pool) { double("pool") } + let(:compression_level) { 6 } + let(:base_options) { super().merge( :client_settings => {:compression_level => compression_level}) } + let(:actions) { [ + ["index", {:_id=>nil, :_index=>"logstash"}, {"message_1"=> message_1}], + ["index", {:_id=>nil, :_index=>"logstash"}, {"message_2"=> message_2}], + ["index", {:_id=>nil, :_index=>"logstash"}, {"message_3"=> message_3}], + ]} + let(:message_1) { "hello" } + let(:message_2_size) { 1_000 } + let(:message_2) { SecureRandom.alphanumeric(message_2_size / 2 ) * 2 } + let(:message_3_size) { 1_000 } + let(:message_3) { "m" * message_3_size } + let(:messages_size) { message_1.size + message_2.size + message_3.size } + let(:action_overhead) { 42 + 16 + 2 } # header plus doc key size plus new line overhead per action + + let(:response) do + response = double("response") + allow(response).to receive(:code).and_return(response) + allow(response).to receive(:body).and_return({"errors" => false}.to_json) + response + end + + before(:each) do + subject.instance_variable_set("@pool", pool) + end + + it "carries bulk request's uncompressed size" do + expect(pool).to receive(:post) do |path, params, body| + headers = params.fetch(:headers, {}) + expect(headers["X-Elastic-Event-Count"]).to eq("3") + expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s + end.and_return(response) + + subject.send(:bulk, actions) + end + context "without compression" do + let(:compression_level) { 0 } + it "carries bulk request's uncompressed size" do + expect(pool).to receive(:post) do |path, params, body| + headers = params.fetch(:headers, {}) + expect(headers["X-Elastic-Event-Count"]).to eq("3") + expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s + end.and_return(response) + subject.send(:bulk, actions) + end + end + + context "with compressed messages over 20MB" do + let(:message_2_size) { 21_000_000 } + it "carries bulk request's uncompressed size" do + # only the first, tiny, message is sent first + expect(pool).to receive(:post) do |path, params, body| + headers = params.fetch(:headers, {}) + expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_1.size + action_overhead).to_s + expect(headers["X-Elastic-Event-Count"]).to eq("1") + end.and_return(response) + + # huge message_2 is sent afterwards alone + expect(pool).to receive(:post) do |path, params, body| + headers = params.fetch(:headers, {}) + expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_2.size + action_overhead).to_s + expect(headers["X-Elastic-Event-Count"]).to eq("1") + end.and_return(response) + + # finally medium message_3 is sent alone as well + expect(pool).to receive(:post) do |path, params, body| + headers = params.fetch(:headers, {}) + expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_3.size + action_overhead).to_s + expect(headers["X-Elastic-Event-Count"]).to eq("1") + end.and_return(response) + + subject.send(:bulk, actions) + end + end + end end describe "sniffing" do diff --git a/spec/unit/outputs/elasticsearch_spec.rb b/spec/unit/outputs/elasticsearch_spec.rb index 35c90804..1e886d30 100644 --- a/spec/unit/outputs/elasticsearch_spec.rb +++ b/spec/unit/outputs/elasticsearch_spec.rb @@ -777,7 +777,7 @@ end before(:each) do - allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions| + allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array), instance_of(Hash)) do |stream, actions, headers| expect( stream.string ).to include '"foo":"bar1"' expect( stream.string ).to include '"foo":"bar2"' end.and_return(bulk_response, {"errors"=>false}) # let's make it go away (second call) to not retry indefinitely diff --git a/spec/unit/outputs/error_whitelist_spec.rb b/spec/unit/outputs/error_whitelist_spec.rb index 9780376f..e6cbbcff 100644 --- a/spec/unit/outputs/error_whitelist_spec.rb +++ b/spec/unit/outputs/error_whitelist_spec.rb @@ -4,7 +4,6 @@ describe "whitelisting error types in expected behavior" do let(:template) { '{"template" : "not important, will be updated by :index"}' } let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") } - let(:action1) { ["index", {:_id=>1, :routing=>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1] } let(:settings) { {"manage_template" => true, "index" => "logstash-2014.11.17", "template_overwrite" => true, "hosts" => get_host_port() } } subject { LogStash::Outputs::ElasticSearch.new(settings) }