From fd24b8731298a5a15be58674a51627920e6f4032 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Sat, 29 May 2021 01:37:12 +0200 Subject: [PATCH 01/10] add ECS support --- CHANGELOG.md | 3 + VERSION | 2 +- docs/index.asciidoc | 95 +++++++++++++++++++++- lib/logstash/inputs/http.rb | 34 +++++++- logstash-input-http.gemspec | 1 + spec/inputs/http_spec.rb | 155 +++++++++++++++++++----------------- 6 files changed, 211 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d472bc83..7c503e81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.4.0 + - Add ECS support, mapping Http header to ECS compatible fields + ## 3.3.7 - Feat: improved error handling/logging/unwraping [#133](https://github.com/logstash-plugins/logstash-input-http/pull/133) diff --git a/VERSION b/VERSION index 86fb6504..18091983 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.7 +3.4.0 diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 30e8dd04..de623814 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -32,6 +32,28 @@ This input can also be used to receive webhook requests to integrate with other and applications. By taking advantage of the vast plugin ecosystem available in Logstash you can trigger actionable events right from your application. +[id="plugins-{type}s-{plugin}-ecs_metadata"] +==== Event Metadata and the Elastic Common Schema (ECS) +In addition to decoding the events, this input will add HTTP headers of the connection to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which had the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. + +Here’s how ECS compatibility mode affects output. +[cols="> |<>|No | <> |<>|No +| <> | <>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -115,6 +138,72 @@ and no codec for the request's content-type is found The list of ciphers suite to use, listed by priorities. +[id="plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is <> +* Supported values are: +** `disabled`: unstructured connection metadata added at root level +** `v1`: structured header metadata added under `[@metadata][http][header]` + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. +See <> for detailed information. + +Example output: + +ECS disabled +[source,text] +----- +{ + "@version" => "1", + "headers" => { + "request_path" => "/twitter/tweet/1", + "http_accept" => "*/*", + "http_version" => "HTTP/1.1", + "request_method" => "PUT", + "http_host" => "localhost:8080", + "http_user_agent" => "curl/7.64.1", + "content_length" => "5", + "content_type" => "application/x-www-form-urlencoded" + }, + "@timestamp" => 2021-05-28T19:27:28.609Z, + "host" => "127.0.0.1", + "message" => "hello" +} +----- + +ECS enabled +[source,text] +----- +{ + "@version" => "1", + "user_agent" => { + "original" => "curl/7.64.1" + }, + "http" => { + "method" => "PUT", + "request" => { + "mime_type" => "application/x-www-form-urlencoded", + "body" => { + "bytes" => "5" + } + }, + "version" => "HTTP/1.1" + }, + "url" => { + "port" => "8080", + "domain" => "snmp1", + "path" => "/twitter/tweet/1" + }, + "@timestamp" => 2021-05-28T23:32:38.222Z, + "host" => { + "ip" => "127.0.0.1" + }, + "message" => "hello", +} +----- + [id="plugins-{type}s-{plugin}-host"] ===== `host` @@ -209,7 +298,8 @@ specify a custom set of response headers ===== `remote_host_target_field` * Value type is <> - * Default value is `"host"` + * Default value is `"host"` when ECS is disabled + * Default value is `[host][ip]` when ECS is enabled specify a target field for the client host of the http request @@ -217,7 +307,8 @@ specify a target field for the client host of the http request ===== `request_headers_target_field` * Value type is <> - * Default value is `"headers"` + * Default value is `"headers"` when ECS is disabled + * Default value is `[@metadata][http][header]` when ECS is enabled specify target field for the client host of the http request diff --git a/lib/logstash/inputs/http.rb b/lib/logstash/inputs/http.rb index d310e9a2..bbf537d3 100644 --- a/lib/logstash/inputs/http.rb +++ b/lib/logstash/inputs/http.rb @@ -3,6 +3,7 @@ require "logstash/namespace" require "stud/interval" require "logstash-input-http_jars" +require "logstash/plugin_mixins/ecs_compatibility_support" # Using this input you can receive single or multiline events over http(s). # Applications can send a HTTP POST request with a body to the endpoint started by this @@ -25,6 +26,7 @@ # format] # class LogStash::Inputs::Http < LogStash::Inputs::Base + include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1) require "logstash/inputs/http/tls" java_import "io.netty.handler.codec.http.HttpUtil" @@ -104,10 +106,10 @@ class LogStash::Inputs::Http < LogStash::Inputs::Base config :response_headers, :validate => :hash, :default => { 'Content-Type' => 'text/plain' } # target field for the client host of the http request - config :remote_host_target_field, :validate => :string, :default => "host" + config :remote_host_target_field, :validate => :string # target field for the client host of the http request - config :request_headers_target_field, :validate => :string, :default => "headers" + config :request_headers_target_field, :validate => :string config :threads, :validate => :number, :required => false, :default => ::LogStash::Config::CpuCoreStrategy.maximum @@ -130,7 +132,7 @@ def register validate_ssl_settings! - if @user && @password then + if @user && @password token = Base64.strict_encode64("#{@user}:#{@password.value}") @auth_token = "Basic #{token}" end @@ -144,6 +146,9 @@ def register require "logstash/inputs/http/message_handler" message_handler = MessageHandler.new(self, @codec, @codecs, @auth_token) @http_server = create_http_server(message_handler) + + @remote_host_target_field ||= ecs_select[disabled: "host", v1: "[host][ip]"] + @request_headers_target_field ||= ecs_select[disabled: "headers", v1: "[@metadata][http][header]"] end # def register def run(queue) @@ -177,12 +182,35 @@ def decode_body(headers, remote_address, body, default_codec, additional_codecs) end def push_decoded_event(headers, remote_address, event) + add_ecs_fields(headers, event) event.set(@request_headers_target_field, headers) event.set(@remote_host_target_field, remote_address) decorate(event) @queue << event end + def add_ecs_fields(headers, event) + return if ecs_compatibility == :disabled + + http_version = headers.get("http_version") + http_user_agent = headers.get("http_user_agent") + http_host = headers.get("http_host") + request_method = headers.get("request_method") + request_path = headers.get("request_path") + content_length = headers.get("content_length") + content_type = headers.get("content_type") + + event.set("[http][version]", http_version) + event.set("[user_agent][original]", http_user_agent) + domain, port = http_host.split(":") + event.set("[url][domain]", domain) + event.set("[url][port]", port) + event.set("[http][method]", request_method) + event.set("[url][path]", request_path) + event.set("[http][request][body][bytes]", content_length) + event.set("[http][request][mime_type]", content_type) + end + def validate_ssl_settings! if !@ssl @logger.warn("SSL Certificate will not be used") if @ssl_certificate diff --git a/logstash-input-http.gemspec b/logstash-input-http.gemspec index 677851a1..01b82b62 100644 --- a/logstash-input-http.gemspec +++ b/logstash-input-http.gemspec @@ -23,6 +23,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" s.add_runtime_dependency 'logstash-codec-plain' s.add_runtime_dependency 'jar-dependencies', '~> 0.3', '>= 0.3.4' + s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.2' s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'logstash-codec-json' diff --git a/spec/inputs/http_spec.rb b/spec/inputs/http_spec.rb index ca136c91..bdb3ec5a 100644 --- a/spec/inputs/http_spec.rb +++ b/spec/inputs/http_spec.rb @@ -6,6 +6,7 @@ require "stud/temporary" require "zlib" require "stringio" +require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper' java_import "io.netty.handler.ssl.util.SelfSignedCertificate" @@ -34,20 +35,7 @@ subject { LogStash::Inputs::Http.new("port" => port) } before :each do - subject.register - t = Thread.new { subject.run(logstash_queue) } - ok = false - until ok - begin - client.post("http://127.0.0.1:#{port}", :body => '{}').call - rescue => e - # retry - else - ok = true - end - sleep 0.01 - end - logstash_queue.pop if logstash_queue.size == 1 # pop test event + setup_client end describe "handling overflowing requests with a 429" do @@ -85,65 +73,6 @@ end end - describe "remote host" do - subject { LogStash::Inputs::Http.new(config.merge("port" => port)) } - context "by default" do - let(:config) { {} } - it "is written to the \"host\" field" do - client.post("http://localhost:#{port}/meh.json", - :headers => { "content-type" => "text/plain" }, - :body => "hello").call - event = logstash_queue.pop - expect(event.get("host")).to eq("127.0.0.1") - end - end - - context "when using remote_host_target_field" do - let(:config) { { "remote_host_target_field" => "remote_host" } } - it "is written to the value of \"remote_host_target_field\" property" do - client.post("http://localhost:#{port}/meh.json", - :headers => { "content-type" => "text/plain" }, - :body => "hello").call - event = logstash_queue.pop - expect(event.get("remote_host")).to eq("127.0.0.1") - end - end - end - - describe "request headers" do - subject { LogStash::Inputs::Http.new(config.merge("port" => port)) } - context "by default" do - let(:config) { {} } - it "are written to the \"headers\" field" do - client.post("http://localhost:#{port}/meh.json", - :headers => { "content-type" => "text/plain" }, - :body => "hello").call - event = logstash_queue.pop - expect(event.get("headers")).to be_a(Hash) - expect(event.get("headers")).to include("request_method" => "POST") - end - end - context "when using request_headers_target_field" do - let(:config) { { "request_headers_target_field" => "request_headers" } } - it "are written to the field set in \"request_headers_target_field\"" do - client.post("http://localhost:#{port}/meh.json", - :headers => { "content-type" => "text/plain" }, - :body => "hello").call - event = logstash_queue.pop - expect(event.get("request_headers")).to be_a(Hash) - expect(event.get("request_headers")).to include("request_method" => "POST") - end - end - end - - it "should include remote host in \"host\" property" do - client.post("http://127.0.0.1:#{port}/meh.json", - :headers => { "content-type" => "text/plain" }, - :body => "hello").call - event = logstash_queue.pop - expect(event.get("host")).to eq("127.0.0.1") - end - context "with default codec" do subject { LogStash::Inputs::Http.new("port" => port) } context "when receiving a text/plain request" do @@ -368,6 +297,86 @@ end end + describe "ECS support", :ecs_compatibility_support, :aggregate_failures do + ecs_compatibility_matrix(:disabled, :v1) do |ecs_select| + let(:host_field) { ecs_select[disabled: "[host]", v1: "[host][ip]"] } + let(:header_field) { ecs_select[disabled: "headers", v1: "[@metadata][http][header]"] } + + before :each do + allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) + setup_client + end + + describe "remote host" do + subject { LogStash::Inputs::Http.new(config.merge("port" => port)) } + context "by default" do + let(:config) { {} } + it "is written to the \"host\" field" do + client.post("http://localhost:#{port}/meh.json", + :headers => { "content-type" => "text/plain" }, + :body => "hello").call + event = logstash_queue.pop + expect(event.get(host_field)).to eq("127.0.0.1") + end + end + + context "when using remote_host_target_field" do + let(:config) { { "remote_host_target_field" => "remote_host" } } + it "is written to the value of \"remote_host_target_field\" property" do + client.post("http://localhost:#{port}/meh.json", + :headers => { "content-type" => "text/plain" }, + :body => "hello").call + event = logstash_queue.pop + expect(event.get("remote_host")).to eq("127.0.0.1") + end + end + end + + describe "request headers" do + subject { LogStash::Inputs::Http.new(config.merge("port" => port)) } + context "by default" do + let(:config) { {} } + it "are written to the \"headers\" field" do + client.post("http://localhost:#{port}/meh.json", + :headers => { "content-type" => "text/plain" }, + :body => "hello").call + event = logstash_queue.pop + expect(event.get(header_field)).to be_a(Hash) + expect(event.get(header_field)).to include("request_method" => "POST") + end + end + context "when using request_headers_target_field" do + let(:config) { { "request_headers_target_field" => "request_headers" } } + it "are written to the field set in \"request_headers_target_field\"" do + client.post("http://localhost:#{port}/meh.json", + :headers => { "content-type" => "text/plain" }, + :body => "hello").call + event = logstash_queue.pop + expect(event.get("request_headers")).to be_a(Hash) + expect(event.get("request_headers")).to include("request_method" => "POST") + end + end + end + end + end + + def setup_client + subject.register + t = Thread.new { subject.run(logstash_queue) } + ok = false + until ok + begin + client.post("http://127.0.0.1:#{port}", :body => '{}').call + rescue => e + # retry + else + ok = true + end + sleep 0.01 + end + logstash_queue.pop if logstash_queue.size == 1 # pop test event + end + context "with :ssl => false" do subject { LogStash::Inputs::Http.new("port" => port, "ssl" => false) } it "should not raise exception" do From e7aae676381c8e639e4776c8f93accd5bee2d9bb Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Sat, 29 May 2021 01:44:00 +0200 Subject: [PATCH 02/10] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c503e81..384d500e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 3.4.0 - - Add ECS support, mapping Http header to ECS compatible fields + - Add ECS support, mapping Http header to ECS compatible fields [#137](https://github.com/logstash-plugins/logstash-input-http/pull/137) ## 3.3.7 - Feat: improved error handling/logging/unwraping [#133](https://github.com/logstash-plugins/logstash-input-http/pull/133) From 28eb56f36fe4c0d7058177fabcf6758b70bae527 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Mon, 31 May 2021 11:48:09 +0200 Subject: [PATCH 03/10] add tests --- lib/logstash/inputs/http.rb | 18 +++++++++--------- spec/inputs/http_spec.rb | 29 ++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/lib/logstash/inputs/http.rb b/lib/logstash/inputs/http.rb index bbf537d3..292efb79 100644 --- a/lib/logstash/inputs/http.rb +++ b/lib/logstash/inputs/http.rb @@ -200,15 +200,15 @@ def add_ecs_fields(headers, event) content_length = headers.get("content_length") content_type = headers.get("content_type") - event.set("[http][version]", http_version) - event.set("[user_agent][original]", http_user_agent) - domain, port = http_host.split(":") - event.set("[url][domain]", domain) - event.set("[url][port]", port) - event.set("[http][method]", request_method) - event.set("[url][path]", request_path) - event.set("[http][request][body][bytes]", content_length) - event.set("[http][request][mime_type]", content_type) + event.set("[http][version]", http_version) if http_version + event.set("[user_agent][original]", http_user_agent) if http_user_agent + domain, colon, port = http_host.rpartition(":") + event.set("[url][domain]", domain) if domain + event.set("[url][port]", port) if port + event.set("[http][method]", request_method) if request_method + event.set("[url][path]", request_path) if request_path + event.set("[http][request][body][bytes]", content_length) if content_length + event.set("[http][request][mime_type]", content_type) if content_type end def validate_ssl_settings! diff --git a/spec/inputs/http_spec.rb b/spec/inputs/http_spec.rb index bdb3ec5a..3dfde098 100644 --- a/spec/inputs/http_spec.rb +++ b/spec/inputs/http_spec.rb @@ -301,6 +301,15 @@ ecs_compatibility_matrix(:disabled, :v1) do |ecs_select| let(:host_field) { ecs_select[disabled: "[host]", v1: "[host][ip]"] } let(:header_field) { ecs_select[disabled: "headers", v1: "[@metadata][http][header]"] } + let(:http_version_field) { ecs_select[disabled: "[headers][http_version]", v1: "[http][version]"] } + let(:user_agent_field) { ecs_select[disabled: "[headers][http_user_agent]", v1: "[user_agent][original]"] } + let(:http_host_field) { "[headers][http_host]" } + let(:domain_field) { "[url][domain]" } + let(:port_field) { "[url][port]" } + let(:request_method_field) { ecs_select[disabled: "[headers][request_method]", v1: "[http][method]"] } + let(:request_path_field) { ecs_select[disabled: "[headers][request_path]", v1: "[url][path]"] } + let(:content_length_field) { ecs_select[disabled: "[headers][content_length]", v1: "[http][request][body][bytes]"] } + let(:content_type_field) { ecs_select[disabled: "[headers][content_type]", v1: "[http][request][mime_type]"] } before :each do allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) @@ -342,7 +351,19 @@ :body => "hello").call event = logstash_queue.pop expect(event.get(header_field)).to be_a(Hash) - expect(event.get(header_field)).to include("request_method" => "POST") + expect(event.get(request_method_field)).to eq("POST") + expect(event.get(request_path_field)).to eq("/meh.json") + expect(event.get(http_version_field)).to eq("HTTP/1.1") + expect(event.get(user_agent_field)).to include("Manticore") + if ecs_compatibility == :disabled + expect(event.get(http_host_field)).to eq("localhost:#{port}") + else + expect(event.get(domain_field)).to eq("localhost") + expect(event.get(port_field)).to eq("#{port}") + end + + expect(event.get(content_length_field)).to eq("5") + expect(event.get(content_type_field)).to eq("text/plain") end end context "when using request_headers_target_field" do @@ -354,6 +375,12 @@ event = logstash_queue.pop expect(event.get("request_headers")).to be_a(Hash) expect(event.get("request_headers")).to include("request_method" => "POST") + expect(event.get("request_headers")).to include("request_path" => "/meh.json") + expect(event.get("request_headers")).to include("http_version" => "HTTP/1.1") + expect(event.get("request_headers")["http_user_agent"]).to include("Manticore") + expect(event.get("request_headers")).to include("http_host" => "localhost:#{port}") + expect(event.get("request_headers")).to include("content_length" => "5") + expect(event.get("request_headers")).to include("content_type" => "text/plain") end end end From cfab2cacea5ed072331f6b2e15e5f94628539151 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Mon, 31 May 2021 11:57:33 +0200 Subject: [PATCH 04/10] update doc --- docs/index.asciidoc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index de623814..6f5d68f6 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -44,14 +44,14 @@ Here’s how ECS compatibility mode affects output. | ECS disabled | ECS v1 | Availability | Description | [host] | [host][ip] | Always | Host IP address -| [headers] | [@metadata][http][header] | Always | Full headers metadata +| [headers] | [@metadata][http][header] | Always | Complete HTTP headers | [headers][http_version] | [http][version] | Always | HTTP version -| [headers][http_user_agent] | [user_agent][original] | Always | browser user agent +| [headers][http_user_agent] | [user_agent][original] | Always | client user agent | [headers][http_host] | [url][domain] and [url][port] | Always | host domain and port | [headers][request_method] | [http][method] | Always | HTTP method -| [headers][request_path] | [url][path] | Always | query path -| [headers][content_length] | [http][request][body][bytes] | Always | request content length -| [headers][content_type] | [http][request][mime_type] | Always | request mime type +| [headers][request_path] | [url][path] | Always | Query path +| [headers][content_length] | [http][request][body][bytes] | Always | Request content length +| [headers][content_type] | [http][request][mime_type] | Always | Request mime type |======================================================================= ==== Blocking Behavior @@ -144,7 +144,7 @@ The list of ciphers suite to use, listed by priorities. * Value type is <> * Supported values are: ** `disabled`: unstructured connection metadata added at root level -** `v1`: structured header metadata added under `[@metadata][http][header]` +** `v1`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. From 139ff12852874ef925cb7513e250107f075a107e Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 1 Jun 2021 14:12:06 +0200 Subject: [PATCH 05/10] fix parsing host --- lib/logstash/inputs/http.rb | 33 +++++++++++++++++++++++--------- spec/inputs/http_spec.rb | 38 +++++++++++++++++++++++++++++++++---- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/lib/logstash/inputs/http.rb b/lib/logstash/inputs/http.rb index 292efb79..5c7696f0 100644 --- a/lib/logstash/inputs/http.rb +++ b/lib/logstash/inputs/http.rb @@ -148,7 +148,7 @@ def register @http_server = create_http_server(message_handler) @remote_host_target_field ||= ecs_select[disabled: "host", v1: "[host][ip]"] - @request_headers_target_field ||= ecs_select[disabled: "headers", v1: "[@metadata][http][header]"] + @request_headers_target_field ||= ecs_select[disabled: "headers", v1: "[@metadata][input][http][request][headers]"] end # def register def run(queue) @@ -193,24 +193,39 @@ def add_ecs_fields(headers, event) return if ecs_compatibility == :disabled http_version = headers.get("http_version") - http_user_agent = headers.get("http_user_agent") - http_host = headers.get("http_host") - request_method = headers.get("request_method") - request_path = headers.get("request_path") - content_length = headers.get("content_length") - content_type = headers.get("content_type") - event.set("[http][version]", http_version) if http_version + + http_user_agent = headers.get("http_user_agent") event.set("[user_agent][original]", http_user_agent) if http_user_agent - domain, colon, port = http_host.rpartition(":") + + http_host = headers.get("http_host") + domain, port = self.class.parse_domain_port(http_host) event.set("[url][domain]", domain) if domain event.set("[url][port]", port) if port + + request_method = headers.get("request_method") event.set("[http][method]", request_method) if request_method + + request_path = headers.get("request_path") event.set("[url][path]", request_path) if request_path + + content_length = headers.get("content_length") event.set("[http][request][body][bytes]", content_length) if content_length + + content_type = headers.get("content_type") event.set("[http][request][mime_type]", content_type) if content_type end + # match the domain and port in either IPV4, "127.0.0.1:8080", or IPV6, "[2001:db8::8a2e:370:7334]:8080", style + # return [domain, port] + def self.parse_domain_port(http_host) + if /^(([^:]+)|\[(.*)\])\:([\d]+)$/ =~ http_host + ["#{$2 || $3}", "#{$4}"] + else + [http_host, "80"] + end + end + def validate_ssl_settings! if !@ssl @logger.warn("SSL Certificate will not be used") if @ssl_certificate diff --git a/spec/inputs/http_spec.rb b/spec/inputs/http_spec.rb index 3dfde098..d53d151a 100644 --- a/spec/inputs/http_spec.rb +++ b/spec/inputs/http_spec.rb @@ -35,7 +35,7 @@ subject { LogStash::Inputs::Http.new("port" => port) } before :each do - setup_client + setup_server_client end describe "handling overflowing requests with a 429" do @@ -300,7 +300,7 @@ describe "ECS support", :ecs_compatibility_support, :aggregate_failures do ecs_compatibility_matrix(:disabled, :v1) do |ecs_select| let(:host_field) { ecs_select[disabled: "[host]", v1: "[host][ip]"] } - let(:header_field) { ecs_select[disabled: "headers", v1: "[@metadata][http][header]"] } + let(:header_field) { ecs_select[disabled: "headers", v1: "[@metadata][input][http][request][headers]"] } let(:http_version_field) { ecs_select[disabled: "[headers][http_version]", v1: "[http][version]"] } let(:user_agent_field) { ecs_select[disabled: "[headers][http_user_agent]", v1: "[user_agent][original]"] } let(:http_host_field) { "[headers][http_host]" } @@ -313,7 +313,7 @@ before :each do allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) - setup_client + setup_server_client end describe "remote host" do @@ -387,7 +387,8 @@ end end - def setup_client + # wait until server is ready + def setup_server_client subject.register t = Thread.new { subject.run(logstash_queue) } ok = false @@ -404,6 +405,35 @@ def setup_client logstash_queue.pop if logstash_queue.size == 1 # pop test event end + describe "parse domain host" do + let(:localhost) { "localhost" } + let(:ipv6) { "2001:db8::8a2e:370:7334" } + + it "should parse in IPV4 format with port" do + domain, port = LogStash::Inputs::Http.parse_domain_port("#{localhost}:8080") + expect(domain).to eq(localhost) + expect(port).to eq("8080") + end + + it "should parse in IPV4 format without port" do + domain, port = LogStash::Inputs::Http.parse_domain_port(localhost) + expect(domain).to eq(localhost) + expect(port).to eq("80") + end + + it "should parse in IPV6 format with port" do + domain, port = LogStash::Inputs::Http.parse_domain_port("[#{ipv6}]:8080") + expect(domain).to eq(ipv6) + expect(port).to eq("8080") + end + + it "should parse in IPV6 format without port" do + domain, port = LogStash::Inputs::Http.parse_domain_port("#{ipv6}") + expect(domain).to eq(ipv6) + expect(port).to eq("80") + end + end + context "with :ssl => false" do subject { LogStash::Inputs::Http.new("port" => port, "ssl" => false) } it "should not raise exception" do From fac6369f0e5e44713e09743833adea70cedb91c5 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 1 Jun 2021 14:15:15 +0200 Subject: [PATCH 06/10] rename --- lib/logstash/inputs/http.rb | 4 ++-- spec/inputs/http_spec.rb | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/logstash/inputs/http.rb b/lib/logstash/inputs/http.rb index 5c7696f0..27efe9fc 100644 --- a/lib/logstash/inputs/http.rb +++ b/lib/logstash/inputs/http.rb @@ -199,7 +199,7 @@ def add_ecs_fields(headers, event) event.set("[user_agent][original]", http_user_agent) if http_user_agent http_host = headers.get("http_host") - domain, port = self.class.parse_domain_port(http_host) + domain, port = self.class.get_domain_port(http_host) event.set("[url][domain]", domain) if domain event.set("[url][port]", port) if port @@ -218,7 +218,7 @@ def add_ecs_fields(headers, event) # match the domain and port in either IPV4, "127.0.0.1:8080", or IPV6, "[2001:db8::8a2e:370:7334]:8080", style # return [domain, port] - def self.parse_domain_port(http_host) + def self.get_domain_port(http_host) if /^(([^:]+)|\[(.*)\])\:([\d]+)$/ =~ http_host ["#{$2 || $3}", "#{$4}"] else diff --git a/spec/inputs/http_spec.rb b/spec/inputs/http_spec.rb index d53d151a..64245ca4 100644 --- a/spec/inputs/http_spec.rb +++ b/spec/inputs/http_spec.rb @@ -410,25 +410,25 @@ def setup_server_client let(:ipv6) { "2001:db8::8a2e:370:7334" } it "should parse in IPV4 format with port" do - domain, port = LogStash::Inputs::Http.parse_domain_port("#{localhost}:8080") + domain, port = LogStash::Inputs::Http.get_domain_port("#{localhost}:8080") expect(domain).to eq(localhost) expect(port).to eq("8080") end it "should parse in IPV4 format without port" do - domain, port = LogStash::Inputs::Http.parse_domain_port(localhost) + domain, port = LogStash::Inputs::Http.get_domain_port(localhost) expect(domain).to eq(localhost) expect(port).to eq("80") end it "should parse in IPV6 format with port" do - domain, port = LogStash::Inputs::Http.parse_domain_port("[#{ipv6}]:8080") + domain, port = LogStash::Inputs::Http.get_domain_port("[#{ipv6}]:8080") expect(domain).to eq(ipv6) expect(port).to eq("8080") end it "should parse in IPV6 format without port" do - domain, port = LogStash::Inputs::Http.parse_domain_port("#{ipv6}") + domain, port = LogStash::Inputs::Http.get_domain_port("#{ipv6}") expect(domain).to eq(ipv6) expect(port).to eq("80") end From 058b7cce13552a8c80b5b8fd910079dc0e0fa13b Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 1 Jun 2021 14:45:38 +0200 Subject: [PATCH 07/10] remove default port number convert port to integer --- lib/logstash/inputs/http.rb | 4 ++-- spec/inputs/http_spec.rb | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/logstash/inputs/http.rb b/lib/logstash/inputs/http.rb index 27efe9fc..7ed5a8f7 100644 --- a/lib/logstash/inputs/http.rb +++ b/lib/logstash/inputs/http.rb @@ -220,9 +220,9 @@ def add_ecs_fields(headers, event) # return [domain, port] def self.get_domain_port(http_host) if /^(([^:]+)|\[(.*)\])\:([\d]+)$/ =~ http_host - ["#{$2 || $3}", "#{$4}"] + ["#{$2 || $3}", $4.to_i] else - [http_host, "80"] + [http_host, nil] end end diff --git a/spec/inputs/http_spec.rb b/spec/inputs/http_spec.rb index 64245ca4..34741306 100644 --- a/spec/inputs/http_spec.rb +++ b/spec/inputs/http_spec.rb @@ -359,7 +359,7 @@ expect(event.get(http_host_field)).to eq("localhost:#{port}") else expect(event.get(domain_field)).to eq("localhost") - expect(event.get(port_field)).to eq("#{port}") + expect(event.get(port_field)).to eq(port) end expect(event.get(content_length_field)).to eq("5") @@ -412,25 +412,25 @@ def setup_server_client it "should parse in IPV4 format with port" do domain, port = LogStash::Inputs::Http.get_domain_port("#{localhost}:8080") expect(domain).to eq(localhost) - expect(port).to eq("8080") + expect(port).to eq(8080) end it "should parse in IPV4 format without port" do domain, port = LogStash::Inputs::Http.get_domain_port(localhost) expect(domain).to eq(localhost) - expect(port).to eq("80") + expect(port).to be_nil end it "should parse in IPV6 format with port" do domain, port = LogStash::Inputs::Http.get_domain_port("[#{ipv6}]:8080") expect(domain).to eq(ipv6) - expect(port).to eq("8080") + expect(port).to eq(8080) end it "should parse in IPV6 format without port" do domain, port = LogStash::Inputs::Http.get_domain_port("#{ipv6}") expect(domain).to eq(ipv6) - expect(port).to eq("80") + expect(port).to be_nil end end From c176dada0ecddcceb6dc6a6a2cb6c92b133c5501 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 1 Jun 2021 17:24:19 +0200 Subject: [PATCH 08/10] update doc --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 6f5d68f6..59286d74 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -44,7 +44,7 @@ Here’s how ECS compatibility mode affects output. | ECS disabled | ECS v1 | Availability | Description | [host] | [host][ip] | Always | Host IP address -| [headers] | [@metadata][http][header] | Always | Complete HTTP headers +| [headers] | [@metadata][input][http][request][headers] | Always | Complete HTTP headers | [headers][http_version] | [http][version] | Always | HTTP version | [headers][http_user_agent] | [user_agent][original] | Always | client user agent | [headers][http_host] | [url][domain] and [url][port] | Always | host domain and port From c4705fadbae8f9a84bf90f28442315df817e44f0 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 8 Jun 2021 14:27:59 +0200 Subject: [PATCH 09/10] update doc --- docs/index.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 59286d74..28006a67 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -34,9 +34,9 @@ you can trigger actionable events right from your application. [id="plugins-{type}s-{plugin}-ecs_metadata"] ==== Event Metadata and the Elastic Common Schema (ECS) -In addition to decoding the events, this input will add HTTP headers of the connection to each event. -When ECS compatibility is disabled, the headers are stored in the `headers` field, which had the potential to create confusion and schema conflicts downstream. -When ECS is enabled, we can ensure a pipeline still has access to this metadata throughout the event's lifecycle without polluting the top-level namespace. +In addition to decoding the events, this input will add HTTP headers containing connection information to each event. +When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. +When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. Here’s how ECS compatibility mode affects output. [cols="> for detailed information. Example output: -ECS disabled +**Sample output: ECS disabled** [source,text] ----- { @@ -173,7 +173,7 @@ ECS disabled } ----- -ECS enabled +**Sample output: ECS enabled** [source,text] ----- { From a0de66ebd88a9cf93ba82264f4fb7f1ffe0c4d69 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Tue, 8 Jun 2021 14:44:59 +0200 Subject: [PATCH 10/10] Revert "add ECS support" --- CHANGELOG.md | 3 - VERSION | 2 +- docs/index.asciidoc | 95 +--------------- lib/logstash/inputs/http.rb | 49 +-------- logstash-input-http.gemspec | 1 - spec/inputs/http_spec.rb | 212 +++++++++++++----------------------- 6 files changed, 79 insertions(+), 283 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 384d500e..d472bc83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,3 @@ -## 3.4.0 - - Add ECS support, mapping Http header to ECS compatible fields [#137](https://github.com/logstash-plugins/logstash-input-http/pull/137) - ## 3.3.7 - Feat: improved error handling/logging/unwraping [#133](https://github.com/logstash-plugins/logstash-input-http/pull/133) diff --git a/VERSION b/VERSION index 18091983..86fb6504 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.4.0 +3.3.7 diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 28006a67..30e8dd04 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -32,28 +32,6 @@ This input can also be used to receive webhook requests to integrate with other and applications. By taking advantage of the vast plugin ecosystem available in Logstash you can trigger actionable events right from your application. -[id="plugins-{type}s-{plugin}-ecs_metadata"] -==== Event Metadata and the Elastic Common Schema (ECS) -In addition to decoding the events, this input will add HTTP headers containing connection information to each event. -When ECS compatibility is disabled, the headers are stored in the `headers` field, which has the potential to create confusion and schema conflicts downstream. -When ECS is enabled, we can ensure a pipeline maintains access to this metadata throughout the event's lifecycle without polluting the top-level namespace. - -Here’s how ECS compatibility mode affects output. -[cols="> |<>|No | <> |<>|No -| <> | <>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -138,72 +115,6 @@ and no codec for the request's content-type is found The list of ciphers suite to use, listed by priorities. -[id="plugins-{type}s-{plugin}-ecs_compatibility"] -===== `ecs_compatibility` - -* Value type is <> -* Supported values are: -** `disabled`: unstructured connection metadata added at root level -** `v1`: headers added under `[@metadata][http][header]`. Some are copied to structured ECS fields `http`, `url`, `user_agent` and `host` - -Controls this plugin's compatibility with the -{ecs-ref}[Elastic Common Schema (ECS)]. -See <> for detailed information. - -Example output: - -**Sample output: ECS disabled** -[source,text] ------ -{ - "@version" => "1", - "headers" => { - "request_path" => "/twitter/tweet/1", - "http_accept" => "*/*", - "http_version" => "HTTP/1.1", - "request_method" => "PUT", - "http_host" => "localhost:8080", - "http_user_agent" => "curl/7.64.1", - "content_length" => "5", - "content_type" => "application/x-www-form-urlencoded" - }, - "@timestamp" => 2021-05-28T19:27:28.609Z, - "host" => "127.0.0.1", - "message" => "hello" -} ------ - -**Sample output: ECS enabled** -[source,text] ------ -{ - "@version" => "1", - "user_agent" => { - "original" => "curl/7.64.1" - }, - "http" => { - "method" => "PUT", - "request" => { - "mime_type" => "application/x-www-form-urlencoded", - "body" => { - "bytes" => "5" - } - }, - "version" => "HTTP/1.1" - }, - "url" => { - "port" => "8080", - "domain" => "snmp1", - "path" => "/twitter/tweet/1" - }, - "@timestamp" => 2021-05-28T23:32:38.222Z, - "host" => { - "ip" => "127.0.0.1" - }, - "message" => "hello", -} ------ - [id="plugins-{type}s-{plugin}-host"] ===== `host` @@ -298,8 +209,7 @@ specify a custom set of response headers ===== `remote_host_target_field` * Value type is <> - * Default value is `"host"` when ECS is disabled - * Default value is `[host][ip]` when ECS is enabled + * Default value is `"host"` specify a target field for the client host of the http request @@ -307,8 +217,7 @@ specify a target field for the client host of the http request ===== `request_headers_target_field` * Value type is <> - * Default value is `"headers"` when ECS is disabled - * Default value is `[@metadata][http][header]` when ECS is enabled + * Default value is `"headers"` specify target field for the client host of the http request diff --git a/lib/logstash/inputs/http.rb b/lib/logstash/inputs/http.rb index 7ed5a8f7..d310e9a2 100644 --- a/lib/logstash/inputs/http.rb +++ b/lib/logstash/inputs/http.rb @@ -3,7 +3,6 @@ require "logstash/namespace" require "stud/interval" require "logstash-input-http_jars" -require "logstash/plugin_mixins/ecs_compatibility_support" # Using this input you can receive single or multiline events over http(s). # Applications can send a HTTP POST request with a body to the endpoint started by this @@ -26,7 +25,6 @@ # format] # class LogStash::Inputs::Http < LogStash::Inputs::Base - include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1) require "logstash/inputs/http/tls" java_import "io.netty.handler.codec.http.HttpUtil" @@ -106,10 +104,10 @@ class LogStash::Inputs::Http < LogStash::Inputs::Base config :response_headers, :validate => :hash, :default => { 'Content-Type' => 'text/plain' } # target field for the client host of the http request - config :remote_host_target_field, :validate => :string + config :remote_host_target_field, :validate => :string, :default => "host" # target field for the client host of the http request - config :request_headers_target_field, :validate => :string + config :request_headers_target_field, :validate => :string, :default => "headers" config :threads, :validate => :number, :required => false, :default => ::LogStash::Config::CpuCoreStrategy.maximum @@ -132,7 +130,7 @@ def register validate_ssl_settings! - if @user && @password + if @user && @password then token = Base64.strict_encode64("#{@user}:#{@password.value}") @auth_token = "Basic #{token}" end @@ -146,9 +144,6 @@ def register require "logstash/inputs/http/message_handler" message_handler = MessageHandler.new(self, @codec, @codecs, @auth_token) @http_server = create_http_server(message_handler) - - @remote_host_target_field ||= ecs_select[disabled: "host", v1: "[host][ip]"] - @request_headers_target_field ||= ecs_select[disabled: "headers", v1: "[@metadata][input][http][request][headers]"] end # def register def run(queue) @@ -182,50 +177,12 @@ def decode_body(headers, remote_address, body, default_codec, additional_codecs) end def push_decoded_event(headers, remote_address, event) - add_ecs_fields(headers, event) event.set(@request_headers_target_field, headers) event.set(@remote_host_target_field, remote_address) decorate(event) @queue << event end - def add_ecs_fields(headers, event) - return if ecs_compatibility == :disabled - - http_version = headers.get("http_version") - event.set("[http][version]", http_version) if http_version - - http_user_agent = headers.get("http_user_agent") - event.set("[user_agent][original]", http_user_agent) if http_user_agent - - http_host = headers.get("http_host") - domain, port = self.class.get_domain_port(http_host) - event.set("[url][domain]", domain) if domain - event.set("[url][port]", port) if port - - request_method = headers.get("request_method") - event.set("[http][method]", request_method) if request_method - - request_path = headers.get("request_path") - event.set("[url][path]", request_path) if request_path - - content_length = headers.get("content_length") - event.set("[http][request][body][bytes]", content_length) if content_length - - content_type = headers.get("content_type") - event.set("[http][request][mime_type]", content_type) if content_type - end - - # match the domain and port in either IPV4, "127.0.0.1:8080", or IPV6, "[2001:db8::8a2e:370:7334]:8080", style - # return [domain, port] - def self.get_domain_port(http_host) - if /^(([^:]+)|\[(.*)\])\:([\d]+)$/ =~ http_host - ["#{$2 || $3}", $4.to_i] - else - [http_host, nil] - end - end - def validate_ssl_settings! if !@ssl @logger.warn("SSL Certificate will not be used") if @ssl_certificate diff --git a/logstash-input-http.gemspec b/logstash-input-http.gemspec index 01b82b62..677851a1 100644 --- a/logstash-input-http.gemspec +++ b/logstash-input-http.gemspec @@ -23,7 +23,6 @@ Gem::Specification.new do |s| s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" s.add_runtime_dependency 'logstash-codec-plain' s.add_runtime_dependency 'jar-dependencies', '~> 0.3', '>= 0.3.4' - s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.2' s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'logstash-codec-json' diff --git a/spec/inputs/http_spec.rb b/spec/inputs/http_spec.rb index 34741306..ca136c91 100644 --- a/spec/inputs/http_spec.rb +++ b/spec/inputs/http_spec.rb @@ -6,7 +6,6 @@ require "stud/temporary" require "zlib" require "stringio" -require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper' java_import "io.netty.handler.ssl.util.SelfSignedCertificate" @@ -35,7 +34,20 @@ subject { LogStash::Inputs::Http.new("port" => port) } before :each do - setup_server_client + subject.register + t = Thread.new { subject.run(logstash_queue) } + ok = false + until ok + begin + client.post("http://127.0.0.1:#{port}", :body => '{}').call + rescue => e + # retry + else + ok = true + end + sleep 0.01 + end + logstash_queue.pop if logstash_queue.size == 1 # pop test event end describe "handling overflowing requests with a 429" do @@ -73,6 +85,65 @@ end end + describe "remote host" do + subject { LogStash::Inputs::Http.new(config.merge("port" => port)) } + context "by default" do + let(:config) { {} } + it "is written to the \"host\" field" do + client.post("http://localhost:#{port}/meh.json", + :headers => { "content-type" => "text/plain" }, + :body => "hello").call + event = logstash_queue.pop + expect(event.get("host")).to eq("127.0.0.1") + end + end + + context "when using remote_host_target_field" do + let(:config) { { "remote_host_target_field" => "remote_host" } } + it "is written to the value of \"remote_host_target_field\" property" do + client.post("http://localhost:#{port}/meh.json", + :headers => { "content-type" => "text/plain" }, + :body => "hello").call + event = logstash_queue.pop + expect(event.get("remote_host")).to eq("127.0.0.1") + end + end + end + + describe "request headers" do + subject { LogStash::Inputs::Http.new(config.merge("port" => port)) } + context "by default" do + let(:config) { {} } + it "are written to the \"headers\" field" do + client.post("http://localhost:#{port}/meh.json", + :headers => { "content-type" => "text/plain" }, + :body => "hello").call + event = logstash_queue.pop + expect(event.get("headers")).to be_a(Hash) + expect(event.get("headers")).to include("request_method" => "POST") + end + end + context "when using request_headers_target_field" do + let(:config) { { "request_headers_target_field" => "request_headers" } } + it "are written to the field set in \"request_headers_target_field\"" do + client.post("http://localhost:#{port}/meh.json", + :headers => { "content-type" => "text/plain" }, + :body => "hello").call + event = logstash_queue.pop + expect(event.get("request_headers")).to be_a(Hash) + expect(event.get("request_headers")).to include("request_method" => "POST") + end + end + end + + it "should include remote host in \"host\" property" do + client.post("http://127.0.0.1:#{port}/meh.json", + :headers => { "content-type" => "text/plain" }, + :body => "hello").call + event = logstash_queue.pop + expect(event.get("host")).to eq("127.0.0.1") + end + context "with default codec" do subject { LogStash::Inputs::Http.new("port" => port) } context "when receiving a text/plain request" do @@ -297,143 +368,6 @@ end end - describe "ECS support", :ecs_compatibility_support, :aggregate_failures do - ecs_compatibility_matrix(:disabled, :v1) do |ecs_select| - let(:host_field) { ecs_select[disabled: "[host]", v1: "[host][ip]"] } - let(:header_field) { ecs_select[disabled: "headers", v1: "[@metadata][input][http][request][headers]"] } - let(:http_version_field) { ecs_select[disabled: "[headers][http_version]", v1: "[http][version]"] } - let(:user_agent_field) { ecs_select[disabled: "[headers][http_user_agent]", v1: "[user_agent][original]"] } - let(:http_host_field) { "[headers][http_host]" } - let(:domain_field) { "[url][domain]" } - let(:port_field) { "[url][port]" } - let(:request_method_field) { ecs_select[disabled: "[headers][request_method]", v1: "[http][method]"] } - let(:request_path_field) { ecs_select[disabled: "[headers][request_path]", v1: "[url][path]"] } - let(:content_length_field) { ecs_select[disabled: "[headers][content_length]", v1: "[http][request][body][bytes]"] } - let(:content_type_field) { ecs_select[disabled: "[headers][content_type]", v1: "[http][request][mime_type]"] } - - before :each do - allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) - setup_server_client - end - - describe "remote host" do - subject { LogStash::Inputs::Http.new(config.merge("port" => port)) } - context "by default" do - let(:config) { {} } - it "is written to the \"host\" field" do - client.post("http://localhost:#{port}/meh.json", - :headers => { "content-type" => "text/plain" }, - :body => "hello").call - event = logstash_queue.pop - expect(event.get(host_field)).to eq("127.0.0.1") - end - end - - context "when using remote_host_target_field" do - let(:config) { { "remote_host_target_field" => "remote_host" } } - it "is written to the value of \"remote_host_target_field\" property" do - client.post("http://localhost:#{port}/meh.json", - :headers => { "content-type" => "text/plain" }, - :body => "hello").call - event = logstash_queue.pop - expect(event.get("remote_host")).to eq("127.0.0.1") - end - end - end - - describe "request headers" do - subject { LogStash::Inputs::Http.new(config.merge("port" => port)) } - context "by default" do - let(:config) { {} } - it "are written to the \"headers\" field" do - client.post("http://localhost:#{port}/meh.json", - :headers => { "content-type" => "text/plain" }, - :body => "hello").call - event = logstash_queue.pop - expect(event.get(header_field)).to be_a(Hash) - expect(event.get(request_method_field)).to eq("POST") - expect(event.get(request_path_field)).to eq("/meh.json") - expect(event.get(http_version_field)).to eq("HTTP/1.1") - expect(event.get(user_agent_field)).to include("Manticore") - if ecs_compatibility == :disabled - expect(event.get(http_host_field)).to eq("localhost:#{port}") - else - expect(event.get(domain_field)).to eq("localhost") - expect(event.get(port_field)).to eq(port) - end - - expect(event.get(content_length_field)).to eq("5") - expect(event.get(content_type_field)).to eq("text/plain") - end - end - context "when using request_headers_target_field" do - let(:config) { { "request_headers_target_field" => "request_headers" } } - it "are written to the field set in \"request_headers_target_field\"" do - client.post("http://localhost:#{port}/meh.json", - :headers => { "content-type" => "text/plain" }, - :body => "hello").call - event = logstash_queue.pop - expect(event.get("request_headers")).to be_a(Hash) - expect(event.get("request_headers")).to include("request_method" => "POST") - expect(event.get("request_headers")).to include("request_path" => "/meh.json") - expect(event.get("request_headers")).to include("http_version" => "HTTP/1.1") - expect(event.get("request_headers")["http_user_agent"]).to include("Manticore") - expect(event.get("request_headers")).to include("http_host" => "localhost:#{port}") - expect(event.get("request_headers")).to include("content_length" => "5") - expect(event.get("request_headers")).to include("content_type" => "text/plain") - end - end - end - end - end - - # wait until server is ready - def setup_server_client - subject.register - t = Thread.new { subject.run(logstash_queue) } - ok = false - until ok - begin - client.post("http://127.0.0.1:#{port}", :body => '{}').call - rescue => e - # retry - else - ok = true - end - sleep 0.01 - end - logstash_queue.pop if logstash_queue.size == 1 # pop test event - end - - describe "parse domain host" do - let(:localhost) { "localhost" } - let(:ipv6) { "2001:db8::8a2e:370:7334" } - - it "should parse in IPV4 format with port" do - domain, port = LogStash::Inputs::Http.get_domain_port("#{localhost}:8080") - expect(domain).to eq(localhost) - expect(port).to eq(8080) - end - - it "should parse in IPV4 format without port" do - domain, port = LogStash::Inputs::Http.get_domain_port(localhost) - expect(domain).to eq(localhost) - expect(port).to be_nil - end - - it "should parse in IPV6 format with port" do - domain, port = LogStash::Inputs::Http.get_domain_port("[#{ipv6}]:8080") - expect(domain).to eq(ipv6) - expect(port).to eq(8080) - end - - it "should parse in IPV6 format without port" do - domain, port = LogStash::Inputs::Http.get_domain_port("#{ipv6}") - expect(domain).to eq(ipv6) - expect(port).to be_nil - end - end - context "with :ssl => false" do subject { LogStash::Inputs::Http.new("port" => port, "ssl" => false) } it "should not raise exception" do