diff --git a/.ci/docker-run.sh b/.ci/docker-run.sh index 02f0c42c..72baaba9 100755 --- a/.ci/docker-run.sh +++ b/.ci/docker-run.sh @@ -6,7 +6,8 @@ set -ex cd .ci if [ "$INTEGRATION" == "true" ]; then - docker-compose up --exit-code-from logstash + # remove the `--attach logstash` if you want to see all logs including elasticsearch container logs + docker-compose up --exit-code-from logstash --attach logstash else docker-compose up --exit-code-from logstash logstash fi diff --git a/.ci/docker-setup.sh b/.ci/docker-setup.sh index 08bd3585..8e03b1c9 100755 --- a/.ci/docker-setup.sh +++ b/.ci/docker-setup.sh @@ -48,10 +48,10 @@ if [ "$ELASTIC_STACK_VERSION" ]; then if [ "$INTEGRATION" == "true" ]; then docker-compose down - docker-compose build + docker-compose build --quiet else docker-compose down - docker-compose build logstash + docker-compose build logstash --quiet fi else echo "Please set the ELASTIC_STACK_VERSION environment variable" diff --git a/CHANGELOG.md b/CHANGELOG.md index 1071291d..c193b44c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 11.4.2 + - Fixes an issue where events containing non-unicode strings could fail to serialize correctly when compression is enabled [#1169](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1169) + ## 11.4.1 - Feat: upgrade manticore (http-client) library [#1063](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1063) - the underlying changes include latest HttpClient (4.5.13) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 85be8c8f..306aa6d3 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -293,6 +293,11 @@ index level and `monitoring` permissions at cluster level. The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. +[id="plugins-{type}s-{plugin}-handling-non-utf-8"] +==== Handling non UTF-8 data + +This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8. +When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`). [id="plugins-{type}s-{plugin}-options"] ==== Elasticsearch Output Configuration Options diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index 9efaeaf1..ee2c9477 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -127,6 +127,9 @@ def bulk(actions) action.map {|line| LogStash::Json.dump(line)}.join("\n") : LogStash::Json.dump(action) as_json << "\n" + + as_json.scrub! # ensure generated JSON is valid UTF-8 + if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0 stream_writer.flush # ensure writer has sync'd buffers before reporting sizes logger.debug("Sending partial bulk request for batch with one or more actions remaining.", diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec index eb33f432..2be78537 100644 --- a/logstash-output-elasticsearch.gemspec +++ b/logstash-output-elasticsearch.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-elasticsearch' - s.version = '11.4.1' + s.version = '11.4.2' s.licenses = ['apache-2.0'] s.summary = "Stores logs in Elasticsearch" diff --git a/spec/fixtures/test_certs/ca.crt b/spec/fixtures/test_certs/ca.crt index d3a1abb6..6626206b 100644 --- a/spec/fixtures/test_certs/ca.crt +++ b/spec/fixtures/test_certs/ca.crt @@ -1,32 +1,31 @@ -----BEGIN CERTIFICATE----- -MIIFeTCCA2GgAwIBAgIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDQYJKoZIhvcNAQEL -BQAwTDELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x -DjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQDDAZSb290Q0EwHhcNMTkwNzE1MTMxMTI5 -WhcNMjQwNzE0MTMxMTI5WjBMMQswCQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzAN -BgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMMBlJvb3RDQTCC -AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMtTMqAWuH17b9XqPa5L3HNq -gnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh5er5JF2iomYOHiD5JncYr4YmRQKuYfD6 -B1WI5FuQthD/OlA1/RHqtbY27J33SaO66ro8gal7vjHrXKQkefVYRwdfO6DqqbhV -6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQqGZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju -4jPhp+43BkGwEimI5km7jNC1nwjiHtxDsY/s93AKa/vLktXKUK5nA3jjJOhAbRTV -nbOAgxFt0YbX98xW/aUqscgBUVs9J/MyTRMwVKJ7Vsmth1PdJQksUASuzESlSPl0 -9dMjTQ+MXzJDt0JvX8SIJPmbBng78MSaCUhpOZiii1l2mBfPWejx20I/SMCUNmzb -wm2w9JD50Jv2iX4l4ge4H1CIK1/orW1pdY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aL -ojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nx -UjCbgn0RABojRoGdhhY3QDipgwmSgFZxr064RFr1bt/Ml3MJmPf535mSwPdk/j/z -w4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMPLEdi73Qjzmr3ENIrir4O86wNz81YRfYk -g9ZX8yKJK9LBAUrYCjJ3AgMBAAGjUzBRMB0GA1UdDgQWBBShWnSceOrqYn9Qa4WG -dIrvKNs/KzAfBgNVHSMEGDAWgBShWnSceOrqYn9Qa4WGdIrvKNs/KzAPBgNVHRMB -Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBRQK0m3t5h2Y3CUCJYLMiCUge4 -UOzvpCoawSXH1FP2ycA+P1bP8H8htjwvV334ZADlQrDQRu0hqa1T+DxwhLxNOxgE -1XCthN3TTyd3O1mT4NmT6mcn2wYSn/JC6fPwFcloX8BcUvxl+xwmOgL/pzgf1ekK -MVS0n+r3bzdFTgGnvsmxmPHe2bUhyXXqzQIx3ObSGtuKYUu7aZEysEtJhaR+vGTd -jjTOV2S71edVlKTxRLZpHgoTZpBL/phwRQ63vdef4ftNGs0glGDc0yqXGMxMALOl -Up7+H4HI99rldZcul6oZ+ORltt047Hk7ctWb20SqxEH9tGLXKm6hDEL9HzyFXeyJ -DAue1GF+3H0KvsjSs5XH7LHMuJDCuSP64+h9gzkI+q06oBNX/9pQyQaHj0K4don8 -lWOMLI4gQibV7R1Opt2feA8MwWxouP/yni8IX6sPePVQ+fLEk1C+Kg+x6k1yQHEM -36BEP6iYOYvqG0OIjMas2U7Yhn2wWlVm9It3WMyaW8ZPI8kwc3dx715dZuNg/zjd -rJS678BNBVxInc7dzpY6el0Lr70CGwiJpX/N9P1yiTFZ7GZm3Kax8QnTtvqXzRIy -sBgt8BVZHUe1lWFYlG+jlakiXqz752nmHuwif7iBI4iWzRmW2vYPfTEmYPRLZES2 -nIg9fQPvVw+fIHACZQ== +MIIFRTCCAy2gAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL +MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN +BgNVBAMMBlJvb3RDQTAeFw0yNDAzMTkyMjM3MTdaFw0yNTAzMTkyMjM3MTdaMEwx +CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD +VQQKDAVNeUxhYjEPMA0GA1UEAwwGUm9vdENBMIICIjANBgkqhkiG9w0BAQEFAAOC +Ag8AMIICCgKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3nz6C9w63tD+w4S9wNAmCT +Nn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjkW5C2EP86UDX9Eeq1tjbs +nfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwyLLwXNDnlMymgzflxa6+g +AzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn7jcGQbASKYjmSbuM0LWf +COIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CDEW3Rhtf3zFb9pSqxyAFR +Wz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNND4xfMkO3Qm9fxIgk+ZsG +eDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0kPnQm/aJfiXiB7gfUIgr +X+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8hPCaZ+l+T1z/JY98Luy0 +eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuCfREAGiNGgZ2GFjdAOKmD +CZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO+WbApbcXIwOHBgv/Nfsn +RzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfzIokr0sEBStgKMncCAwEA +AaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUoVp0nHjq6mJ/UGuFhnSK +7yjbPyswDQYJKoZIhvcNAQELBQADggIBADeI0gRfzF5zyhCCmtlA44L1fo3BEYtm +0Deq2W6bsW9zakYT7fivCUbt0VtS0jzhAlbaLfksJk+Emg+kpsy/CdFr8nk9jlpN +XJMKwjQIVGYDDlJsP9VDw2m+HhUugEpx09QCREaCWCDtobE9bd42VloPd0u3VWZx +u5rSrZCy5wjFeq3dPEFPyDTfV6p1kXkwy9pdG3ww0rw0dqj5OqcGhLyG8b7q7LLP +aFVuwjnutrBJtDNOsvbD/dyybQVj3hF1IpVZLwoFQ3ji43/X3NJ5jdo/WrwwdwbH +NE6YxNlsIrql4sJFeHzNukTxcC5wHcOLtc6E6dhe7uK+5rRD75+odz85rPLsBTPt +zje8OWyxO1cGol7inpKUsO9ITFQdDC9iyRjvDRouwneT+XeVwW6BmAKg4DvXdLtQ +oQNLlMbbZWskvWoI4amXwSpzWmn4mRK3e4IPWFju64QRvkEExFynvC69jeih5Llh +JU3OjLobG4wKOQHB7w1UjiPJNsxGB0P1zug6ztTBLjsvJoxdJuIGkCsI7g12cKNR +PbXp8453klofFpMNGlCP8dns92Qu20pPvYJcWiC9k42nYu3FHDMbAfSrFU90RNS6 +hRWkpXZOMIY1GdudgsxanDtJgFRbkPzwNkMcBQ1538f9U4MrJ9vPsuKx4bYbZ//8 +pIe/z3b1V9zV -----END CERTIFICATE----- diff --git a/spec/fixtures/test_certs/renew.sh b/spec/fixtures/test_certs/renew.sh new file mode 100755 index 00000000..57783263 --- /dev/null +++ b/spec/fixtures/test_certs/renew.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e +cd "$(dirname "$0")" + +openssl x509 -x509toreq -copy_extensions copyall -in ca.crt -signkey ca.key -out ca.csr +openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr + +openssl x509 -x509toreq -copy_extensions copyall -in test.crt -signkey test.key -out test.csr +openssl x509 -req -copy_extensions copyall -days 365 -in test.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out test.crt && rm test.csr +openssl pkcs12 -export -inkey test.key -in test.crt -passout "pass:1234567890" -out test.p12 diff --git a/spec/fixtures/test_certs/test.crt b/spec/fixtures/test_certs/test.crt index 11d4a238..6aa035aa 100644 --- a/spec/fixtures/test_certs/test.crt +++ b/spec/fixtures/test_certs/test.crt @@ -1,7 +1,7 @@ -----BEGIN CERTIFICATE----- -MIIGQjCCBCqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL +MIIF1jCCA76gAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN -BgNVBAMMBlJvb3RDQTAeFw0xOTA3MTUxMzEzMDVaFw0yMjA0MTAxMzEzMDVaMFMx +BgNVBAMMBlJvb3RDQTAeFw0yNDAzMTkyMjM3MTdaFw0yNTAzMTkyMjM3MTdaMFMx CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD VQQKDAVNeUxhYjEWMBQGA1UEAwwNZWxhc3RpY3NlYXJjaDCCAiIwDQYJKoZIhvcN AQEBBQADggIPADCCAgoCggIBAMYhP2zPOE3ke9naeK+cIPNV91htuoGGARs+mlY/ @@ -15,22 +15,20 @@ bFMKspGHnytQZF+a+mc5H33G9HiPP3jZE2JjrWlOay+j6ImylMgjcZmHAgaUe3ET tyczoQ5/L5BNiyA2h+1TU8jWicNDtl1+CtOsgEVBBHA6p/IHhsHbNZWPrYtIO9mh hiJw1R5yrITXnjZY0rObITwyt/e6Sc3YnoQfsSGaLJEG0aDc0RALAhgzj+RY8086 2RKOyfdw1sw1RmJKdCf+dOzhPyDpvauvCxrL8UZQTzcBs+qpxOWnZFRWeNsLwoDn -6JXXAgMBAAGjggEmMIIBIjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAz -BglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmlj -YXRlMB0GA1UdDgQWBBRvvz0yGw6Tz2UxbBLAGyzVMtcMUDCBiAYDVR0jBIGAMH6A -FKFadJx46upif1BrhYZ0iu8o2z8roVCkTjBMMQswCQYDVQQGEwJQVDELMAkGA1UE -CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMM -BlJvb3RDQYIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDgYDVR0PAQH/BAQDAgWgMBMG -A1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQCaABHQxm6mtrM9 -f7kbgzuhEc47Q+bgrbjxeoIVOeO2Zshdw0SZlfkWvWe0622WSeWMsTBJ3hoaQwZe -9FUf1lnsWe6u6oOckiG9OjE0TyXJ7+eghdL1HPeXgJ+4ihwJsRtkNEljWf4HS7/n -y5LaFhcXdn2ZdbUKJ7z7zXqzh2Cp8VUBtsR+/IdiLjSN81dQou77/a2M/a/7BI2Z -HhUlUx1T7jHzNllJBRF3IaOk72yjoU4cL0qVy9874SXPwdpeFHtvS4TdQTLqnAGR -liHJcB1ZNz1sVOXndw3Wbvv6iB5y+IX/Y/kRSHS6zpZGdAb7ar/Vgl+Uvs3fKi44 -y9hq2b49bYlcSQMtmlimCBDiu82z0aYtVFLalZ2L/W7CMaeE3jpyzu/bbygRv/Bp -lKSaUtaFIVgiuRBPwIBDMyai3CJ5L+dJrJPU2JzzQvtJGFQCFCIHd9rqweubZB6V -re5cUn4dxlxA5SkZ0amFFV5DpP0YhThA/gq0t/NeWRmCEEBWNXZaqFmDhiYS5mnu -Z+NUtv8E332S46RdfneHe961SlMXEFC96I+1HOjXHdXlqKfOU8Qvy8VzsnpjuNE5 -VTrvnAM1L3LwqtYQYfUWUHYZFYdvh8layA2ImNE7yx/9wIIkw/L1j9m71Upi6WKR -FKbYFqzgpWksa+zZ2RYYplUAxq0wYw== +6JXXAgMBAAGjgbswgbgwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJ +YIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0 +ZTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHQYDVR0OBBYE +FG+/PTIbDpPPZTFsEsAbLNUy1wxQMB8GA1UdIwQYMBaAFKFadJx46upif1BrhYZ0 +iu8o2z8rMA0GCSqGSIb3DQEBCwUAA4ICAQB6Fptgev/wPHn36lamR9RCBpqvh/Xg +4nSnUfT1EVkEI7vUKkC/+/XeRTfnc14Yclo3uiYBY7QM953FhXOmlH0Rx1wwZSME +ZzyO+1TWNVR2QI7/MhpXdKpqHhNLNTYKFa602ZTA5NXa9XDJ8T3kjbuAKiNwcuu7 +RzRK0o3KOWe0uohuI6bgcpgpKqcxrbtUkrPOWAn+XK7JQVSEV1kst4uDr1S+041c +T+NHFQarDXw5g/Y0pHnU42dPyDvT/RrcBbzbe7qylFTbjKro/uFqPAayHlLMnGZp +ZthGAPJJ0pDzBBxGtUmwOnat/HKWe2UN/A3Xqmy0Za7adhrYGj0IaZ0SNblNYDKy +OjWtfOwLQrD9NEIhBXBE3KS5uyXP8kZESnHtIN0b7SMyP7lUxTQ/Rh8U7S6QD9AI +gc+AfBx8UO+fdW3Z4VK+DU0NGRP+127w0yiEz1OtfS87dWBQq+kk2Hlc9TfFeG0b +rckM4v54X8uTlnNfPo+RYRkMAXB/fnx2vVIyxTxFPo4bAnTFA6vnIz8rrRO+Vmh3 +oOclhE+ZemZ3BEC+Tgpvb4XD8pvDLnNOYr8tuDjN9jNNDofxplbsjI44wnb1v3HX +M8Xe3P1WfqpWZniRmtVi/CWjkPy2JeC6zW4dn7JdkfEFHGsHQZrEZyrVrtVFPRFo +hbgPGNWCxOhWzg== -----END CERTIFICATE----- diff --git a/spec/fixtures/test_certs/test.p12 b/spec/fixtures/test_certs/test.p12 index 777a235a..2843560c 100644 Binary files a/spec/fixtures/test_certs/test.p12 and b/spec/fixtures/test_certs/test.p12 differ diff --git a/spec/integration/outputs/compressed_indexing_spec.rb b/spec/integration/outputs/compressed_indexing_spec.rb index 4a57f323..37475031 100644 --- a/spec/integration/outputs/compressed_indexing_spec.rb +++ b/spec/integration/outputs/compressed_indexing_spec.rb @@ -10,10 +10,12 @@ describe "indexing with http_compression turned on", :integration => true do let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) } + let(:event_with_invalid_utf_8_bytes) { LogStash::Event.new("message" => "Message from spacecraft which contains \xAC invalid \xD7 byte sequences.", "type" => type) } let(:index) { 10.times.collect { rand(10).to_s }.join("") } let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" } let(:event_count) { 10000 + rand(500) } - let(:events) { event_count.times.map { event }.to_a } + # mix the events with valid and invalid UTF-8 payloads + let(:events) { event_count.times.map { |i| i%3 == 0 ? event : event_with_invalid_utf_8_bytes }.to_a } let(:config) { { "hosts" => get_host_port, diff --git a/spec/unit/outputs/elasticsearch/http_client_spec.rb b/spec/unit/outputs/elasticsearch/http_client_spec.rb index 16b84424..15774d8e 100644 --- a/spec/unit/outputs/elasticsearch/http_client_spec.rb +++ b/spec/unit/outputs/elasticsearch/http_client_spec.rb @@ -243,12 +243,14 @@ end end - context "with two messages" do - let(:message1) { "hey" } - let(:message2) { "you" } + context "with multiple messages" do + let(:message_head) { "Spacecraft message" } + let(:message_tail) { "byte sequence" } + let(:invalid_utf_8_message) { "contains invalid \xAC" } let(:actions) { [ - ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}], - ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}], + ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_head}], + ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> invalid_utf_8_message}], + ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_tail}], ]} it "executes one bulk_send operation" do allow(subject).to receive(:join_bulk_responses) @@ -258,7 +260,7 @@ context "if one exceeds TARGET_BULK_BYTES" do let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES } - let(:message1) { "a" * (target_bulk_bytes + 1) } + let(:message_head) { "a" * (target_bulk_bytes + 1) } it "executes two bulk_send operations" do allow(subject).to receive(:join_bulk_responses) expect(subject).to receive(:bulk_send).twice