From 72d17fda12507c154e99f23e5cb2d1525dfd581c Mon Sep 17 00:00:00 2001 From: Aaron Mildenstein Date: Wed, 27 Jul 2016 15:43:44 -0600 Subject: [PATCH 01/91] Add path.data functionality But maintain reverse compatibility for older versions of Logstash I don't yet know how to test this... Fixes #133 --- CHANGELOG.md | 3 ++- CONTRIBUTORS | 1 + lib/logstash/inputs/file.rb | 11 +++++++++++ logstash-input-file.gemspec | 3 +-- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 692d0cf..06ea1e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ +## 3.1.0 + - Use native `--path.data` for Logstash 5.0 for sincedb files. ## 3.0.3 - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 - ## 3.0.2 - relax constrains of `logstash-devutils` see https://github.com/elastic/logstash-devutils/issues/48 ## 3.0.1 diff --git a/CONTRIBUTORS b/CONTRIBUTORS index ea05590..d89064c 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -18,6 +18,7 @@ Contributors: * elliot moore (em295) * yjpa7145 * Guy Boertje (guyboertje) +* Aaron Mildenstein (untergeek) Note: If you've sent us patches, bug reports, or otherwise contributed to Logstash, and you aren't on the list above and want to be, please let us know diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 04dc36c..c58534c 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -5,6 +5,7 @@ require "pathname" require "socket" # for Socket.gethostname +require "fileutils" # Stream events from files, normally by tailing them in a manner # similar to `tail -0F` but optionally reading them from the @@ -170,6 +171,7 @@ def register require "digest/md5" @logger.info("Registering file input", :path => @path) @host = Socket.gethostname.force_encoding(Encoding::UTF_8) + @settings = defined?(LogStash::SETTINGS) ? LogStash::SETTINGS : nil @tail_config = { :exclude => @exclude, @@ -189,6 +191,15 @@ def register end if @sincedb_path.nil? + if @settings + datapath = File.join(@settings.get_value("path.data"), "plugins", "inputs", "file") + # Ensure that the filepath exists before writing, since it's deeply nested. + FileUtils::mkdir_p datapath + @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) + + # This section is going to be deprecated eventually, as path.data will be + # the default, not an environment variable (SINCEDB_DIR or HOME) + if @sincedb_path.nil? # If it is _still_ nil... if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil? @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \ "to keep track of the files I'm watching. Either set " \ diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 07fa3c4..781273c 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '3.0.3' + s.version = '3.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -32,4 +32,3 @@ Gem::Specification.new do |s| s.add_development_dependency 'logstash-codec-json' s.add_development_dependency 'rspec-sequencing' end - From d7bbd7e73789b710b419cc6274ba45857490d6cd Mon Sep 17 00:00:00 2001 From: Aaron Mildenstein Date: Thu, 28 Jul 2016 11:36:45 -0600 Subject: [PATCH 02/91] End if statements I really need to stop thinking in Python... Fixes #133 --- lib/logstash/inputs/file.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index c58534c..39f838f 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -196,6 +196,8 @@ def register # Ensure that the filepath exists before writing, since it's deeply nested. FileUtils::mkdir_p datapath @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) + end + end # This section is going to be deprecated eventually, as path.data will be # the default, not an environment variable (SINCEDB_DIR or HOME) From a0f04d72f56b2dc9b5e7e1d3a3f688eb14cb449e Mon Sep 17 00:00:00 2001 From: Aaron Mildenstein Date: Thu, 28 Jul 2016 12:01:38 -0600 Subject: [PATCH 03/91] Change settings to not be an ivar And document what settings is for, and why. Fixes #133 --- lib/logstash/inputs/file.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 39f838f..cc2324f 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -171,7 +171,9 @@ def register require "digest/md5" @logger.info("Registering file input", :path => @path) @host = Socket.gethostname.force_encoding(Encoding::UTF_8) - @settings = defined?(LogStash::SETTINGS) ? LogStash::SETTINGS : nil + # This check is Logstash 5 specific. If the class does not exist, and it + # won't in older versions of Logstash, then we need to set it to nil. + settings = defined?(LogStash::SETTINGS) ? LogStash::SETTINGS : nil @tail_config = { :exclude => @exclude, @@ -191,8 +193,8 @@ def register end if @sincedb_path.nil? - if @settings - datapath = File.join(@settings.get_value("path.data"), "plugins", "inputs", "file") + if settings + datapath = File.join(settings.get_value("path.data"), "plugins", "inputs", "file") # Ensure that the filepath exists before writing, since it's deeply nested. FileUtils::mkdir_p datapath @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) From f0df01860f83c9994848f882c7a48d4b1031a6cf Mon Sep 17 00:00:00 2001 From: Jimmy Jones Date: Tue, 9 Aug 2016 21:45:20 +0100 Subject: [PATCH 04/91] Add host to @metadata like path, in case event also contains field of same name Fixes #136 --- CHANGELOG.md | 2 ++ lib/logstash/inputs/file.rb | 1 + logstash-input-file.gemspec | 2 +- spec/inputs/file_spec.rb | 2 ++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06ea1e8..e4b128a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 3.1.1 + - Add host to @metadata ## 3.1.0 - Use native `--path.data` for Logstash 5.0 for sincedb files. ## 3.0.3 diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index e1b09aa..830e5e8 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -314,6 +314,7 @@ def run(queue) end # def run def post_process_this(event) + event.set("[@metadata][host]", @host) event.set("host", @host) if !event.include?("host") decorate(event) @queue << event diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 781273c..77b9518 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '3.1.0' + s.version = '3.1.1' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/inputs/file_spec.rb b/spec/inputs/file_spec.rb index fd19609..adad5e2 100644 --- a/spec/inputs/file_spec.rb +++ b/spec/inputs/file_spec.rb @@ -122,9 +122,11 @@ insist { events[0].get("path") } == "my_path" insist { events[0].get("host") } == "my_host" + insist { events[0].get("[@metadata][host]") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" insist { events[1].get("path") } == "#{tmpfile_path}" insist { events[1].get("host") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + insist { events[1].get("[@metadata][host]") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" end context "when sincedb_path is an existing directory" do From 4c163d7b98d9ecd41cf7a1c71e38a24cd878cb84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Thu, 15 Sep 2016 22:15:32 +0100 Subject: [PATCH 05/91] reduce logging at info level (#141) --- CHANGELOG.md | 2 ++ lib/logstash/inputs/file.rb | 4 ++-- logstash-input-file.gemspec | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4b128a..46c2972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 3.1.2 + - Adjust a few log call levels ## 3.1.1 - Add host to @metadata ## 3.1.0 diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 830e5e8..51a302a 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -172,7 +172,7 @@ def register require "addressable/uri" require "filewatch/tail" require "digest/md5" - @logger.info("Registering file input", :path => @path) + @logger.trace("Registering file input", :path => @path) @host = Socket.gethostname.force_encoding(Encoding::UTF_8) # This check is Logstash 5 specific. If the class does not exist, and it # won't in older versions of Logstash, then we need to set it to nil. @@ -226,7 +226,7 @@ def register # Migrate any old .sincedb to the new file (this is for version <=1.1.1 compatibility) old_sincedb = File.join(sincedb_dir, ".sincedb") if File.exists?(old_sincedb) - @logger.info("Renaming old ~/.sincedb to new one", :old => old_sincedb, + @logger.debug("Renaming old ~/.sincedb to new one", :old => old_sincedb, :new => @sincedb_path) File.rename(old_sincedb, @sincedb_path) end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 77b9518..e15171b 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '3.1.1' + s.version = '3.1.2' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 8869c48e939bfa3daa4d9dd4e081bff6c596790b Mon Sep 17 00:00:00 2001 From: Suyog Rao Date: Tue, 11 Oct 2016 10:58:47 -0700 Subject: [PATCH 06/91] Fix changelog formatting --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46c2972..50b842a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,21 +1,30 @@ ## 3.1.2 - Adjust a few log call levels + ## 3.1.1 - Add host to @metadata + ## 3.1.0 - - Use native `--path.data` for Logstash 5.0 for sincedb files. + - Breaking: Use native `--path.data` for Logstash 5.0 for sincedb files. + ## 3.0.3 - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 + ## 3.0.2 - relax constrains of `logstash-devutils` see https://github.com/elastic/logstash-devutils/issues/48 + ## 3.0.1 - Republish all the gems under jruby. + ## 3.0.0 - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141 + # 2.2.5 - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash + # 2.2.3 - New dependency requirements for logstash-core for the 5.0 release + ## 2.2.2 - Fix for: Filewatch library complains if HOME or SINCEDB_PATH variables are unset. - [Issue #101](https://github.com/logstash-plugins/logstash-input-file/issues/101) From 9a5b31a5a9a43af9e2f41a9d4ca089979de425cd Mon Sep 17 00:00:00 2001 From: Suyog Rao Date: Mon, 17 Oct 2016 08:38:35 -0700 Subject: [PATCH 07/91] Disable ignore_older by default (#146) * Disable ignore_older by default Fixes #130 --- CHANGELOG.md | 6 ++++++ lib/logstash/inputs/file.rb | 5 +++-- logstash-input-file.gemspec | 2 +- spec/inputs/file_spec.rb | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50b842a..6a4b1b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 4.0.0 + - Breaking: `ignore_older` settings is disabled by default. Previously if the file was older than + 24 hours (the default for ignore_older), it would be ignored. This confused new users a lot, specially + when they were reading new files with Logstash (with `start_position => beginning`). This setting also + makes it consistent with Filebeat. + ## 3.1.2 - Adjust a few log call levels diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 51a302a..08a0c53 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -146,8 +146,9 @@ class LogStash::Inputs::File < LogStash::Inputs::Base # When the file input discovers a file that was last modified # before the specified timespan in seconds, the file is ignored. # After it's discovery, if an ignored file is modified it is no - # longer ignored and any new data is read. The default is 24 hours. - config :ignore_older, :validate => :number, :default => 24 * 60 * 60 + # longer ignored and any new data is read. By default, this option is + # disabled. Note this unit is in seconds. + config :ignore_older, :validate => :number # The file input closes any files that were last read the specified # timespan in seconds ago. diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index e15171b..101f43b 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '3.1.2' + s.version = '4.0.0' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/inputs/file_spec.rb b/spec/inputs/file_spec.rb index adad5e2..7e53264 100644 --- a/spec/inputs/file_spec.rb +++ b/spec/inputs/file_spec.rb @@ -129,6 +129,41 @@ insist { events[1].get("[@metadata][host]") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" end + it "should read old files" do + tmpfile_path = Stud::Temporary.pathname + + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{tmpfile_path}" + start_position => "beginning" + codec => "json" + } + } + CONFIG + + File.open(tmpfile_path, "w") do |fd| + fd.puts('{"path": "my_path", "host": "my_host"}') + fd.puts('{"my_field": "my_val"}') + fd.fsync + end + # arbitrary old file (2 days) + FileInput.make_file_older(tmpfile_path, 48 * 60 * 60) + + events = input(conf) do |pipeline, queue| + 2.times.collect { queue.pop } + end + + insist { events[0].get("path") } == "my_path" + insist { events[0].get("host") } == "my_host" + insist { events[0].get("[@metadata][host]") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + + insist { events[1].get("path") } == "#{tmpfile_path}" + insist { events[1].get("host") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + insist { events[1].get("[@metadata][host]") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + end + context "when sincedb_path is an existing directory" do let(:tmpfile_path) { Stud::Temporary.pathname } let(:sincedb_path) { Stud::Temporary.directory } From 4363470f12852a62e05c2174b7e231c036c68233 Mon Sep 17 00:00:00 2001 From: Suyog Rao Date: Tue, 18 Oct 2016 18:41:35 -0700 Subject: [PATCH 08/91] Update sincedb_path docs --- lib/logstash/inputs/file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 08a0c53..1aeb50d 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -120,7 +120,7 @@ class LogStash::Inputs::File < LogStash::Inputs::Base # Path of the sincedb database file (keeps track of the current # position of monitored log files) that will be written to disk. - # The default will write sincedb files to some path matching `$HOME/.sincedb*` + # The default will write sincedb files to `/plugins/inputs/file` # NOTE: it must be a file path and not a directory path config :sincedb_path, :validate => :string From 564ca3d18e996c5b974dd0fbc2a26818a054b609 Mon Sep 17 00:00:00 2001 From: Suyog Rao Date: Thu, 9 Feb 2017 14:06:31 -0800 Subject: [PATCH 09/91] [Docs] Add note that NFS is not supported (#153) * Add note on NFS --- lib/logstash/inputs/file.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 1aeb50d..37bf49e 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -23,6 +23,12 @@ # beginning to end and storing all of it in a single event (not even # with the multiline codec or filter). # +# ==== Reading from remote network volumes +# +# The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These +# remote filesystems typically have behaviors that are very different from local filesystems and +# are therefore unlikely to work correctly when used with the file input. +# # ==== Tracking of current position in watched files # # The plugin keeps track of the current position in each file by From c9fe9af2538e6adad32c98d2186343fa3d34a5c4 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 21 Feb 2017 16:01:05 -0500 Subject: [PATCH 10/91] Fix the description with the documentation generator Make sure we ignore the patched classes and fix an issue with the rspec suite. Fixes #154 --- CHANGELOG.md | 4 ++++ lib/logstash/inputs/file.rb | 16 +--------------- lib/logstash/inputs/file/patch.rb | 16 ++++++++++++++++ logstash-input-file.gemspec | 2 +- spec/inputs/file_spec.rb | 3 +-- 5 files changed, 23 insertions(+), 18 deletions(-) create mode 100644 lib/logstash/inputs/file/patch.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a4b1b1..0c57d8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 4.0.1 + - Docs: Fix the description with the logstash documentation generator + - Fix an issue with the rspec suite not finding log4j + ## 4.0.0 - Breaking: `ignore_older` settings is disabled by default. Previously if the file was older than 24 hours (the default for ignore_older), it would be ignored. This confused new users a lot, specially diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 37bf49e..fec3aa7 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -6,6 +6,7 @@ require "pathname" require "socket" # for Socket.gethostname require "fileutils" +require_relative "file/patch" # Stream events from files, normally by tailing them in a manner # similar to `tail -0F` but optionally reading them from the @@ -77,21 +78,6 @@ # determined by the `stat_interval` and `discover_interval` options) # will not get picked up. -class LogStash::Codecs::Base - # TODO - move this to core - if !method_defined?(:accept) - def accept(listener) - decode(listener.data) do |event| - listener.process_event(event) - end - end - end - if !method_defined?(:auto_flush) - def auto_flush(*) - end - end -end - class LogStash::Inputs::File < LogStash::Inputs::Base config_name "file" diff --git a/lib/logstash/inputs/file/patch.rb b/lib/logstash/inputs/file/patch.rb new file mode 100644 index 0000000..a54fb98 --- /dev/null +++ b/lib/logstash/inputs/file/patch.rb @@ -0,0 +1,16 @@ +# encoding: utf-8 +class LogStash::Codecs::Base + # TODO - move this to core + if !method_defined?(:accept) + def accept(listener) + decode(listener.data) do |event| + listener.process_event(event) + end + end + end + if !method_defined?(:auto_flush) + def auto_flush(*) + end + end +end + diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 101f43b..bd8867d 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.0.0' + s.version = '4.0.1' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/inputs/file_spec.rb b/spec/inputs/file_spec.rb index 7e53264..85df88d 100644 --- a/spec/inputs/file_spec.rb +++ b/spec/inputs/file_spec.rb @@ -1,7 +1,6 @@ # encoding: utf-8 - -require "logstash/inputs/file" require_relative "../spec_helper" +require "logstash/inputs/file" require "tempfile" require "stud/temporary" require "logstash/codecs/multiline" From a80bf53b52dc76946c738c38ef461a36c1fb08c4 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 28 Apr 2017 21:09:16 +0000 Subject: [PATCH 11/91] Initial doc move --- docs/index.asciidoc | 255 ++++++++++++++++++++++++++++++++++++ logstash-input-file.gemspec | 2 +- 2 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 docs/index.asciidoc diff --git a/docs/index.asciidoc b/docs/index.asciidoc new file mode 100644 index 0000000..1867214 --- /dev/null +++ b/docs/index.asciidoc @@ -0,0 +1,255 @@ +:plugin: file +:type: input + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: %VERSION% +:release_date: %RELEASE_DATE% +:changelog_url: %CHANGELOG_URL% +:include_path: ../../../logstash/docs/include +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="plugins-{type}-{plugin}"] + +=== File + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Stream events from files, normally by tailing them in a manner +similar to `tail -0F` but optionally reading them from the +beginning. + +By default, each event is assumed to be one line and a line is +taken to be the text before a newline character. +Normally, logging will add a newline to the end of each line written. +If you would like to join multiple log lines into one event, +you'll want to use the multiline codec or filter. + +The plugin aims to track changing files and emit new content as it's +appended to each file. It's not well-suited for reading a file from +beginning to end and storing all of it in a single event (not even +with the multiline codec or filter). + +==== Reading from remote network volumes + +The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These +remote filesystems typically have behaviors that are very different from local filesystems and +are therefore unlikely to work correctly when used with the file input. + +==== Tracking of current position in watched files + +The plugin keeps track of the current position in each file by +recording it in a separate file named sincedb. This makes it +possible to stop and restart Logstash and have it pick up where it +left off without missing the lines that were added to the file while +Logstash was stopped. + +By default, the sincedb file is placed in the home directory of the +user running Logstash with a filename based on the filename patterns +being watched (i.e. the `path` option). Thus, changing the filename +patterns will result in a new sincedb file being used and any +existing current position state will be lost. If you change your +patterns with any frequency it might make sense to explicitly choose +a sincedb path with the `sincedb_path` option. + +A different `sincedb_path` must be used for each input. Using the same +path will cause issues. The read checkpoints for each input must be +stored in a different path so the information does not override. + +Sincedb files are text files with four columns: + +. The inode number (or equivalent). +. The major device number of the file system (or equivalent). +. The minor device number of the file system (or equivalent). +. The current byte offset within the file. + +On non-Windows systems you can obtain the inode number of a file +with e.g. `ls -li`. + +==== File rotation + +File rotation is detected and handled by this input, regardless of +whether the file is rotated via a rename or a copy operation. To +support programs that write to the rotated file for some time after +the rotation has taken place, include both the original filename and +the rotated filename (e.g. /var/log/syslog and /var/log/syslog.1) in +the filename patterns to watch (the `path` option). Note that the +rotated filename will be treated as a new file so if +`start_position` is set to 'beginning' the rotated file will be +reprocessed. + +With the default value of `start_position` ('end') any messages +written to the end of the file between the last read operation prior +to the rotation and its reopening under the new name (an interval +determined by the `stat_interval` and `discover_interval` options) +will not get picked up. + +[id="plugins-{type}s-{plugin}-options"] +==== File Input Configuration Options + +This plugin supports the following configuration options plus the <> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|Yes +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["beginning", "end"]`|No +| <> |<>|No +|======================================================================= + +Also see <> for a list of options supported by all +input plugins. + +  + +[id="plugins-{type}s-{plugin}-close_older"] +===== `close_older` + + * Value type is <> + * Default value is `3600` + +The file input closes any files that were last read the specified +timespan in seconds ago. +This has different implications depending on if a file is being tailed or +read. If tailing, and there is a large time gap in incoming data the file +can be closed (allowing other files to be opened) but will be queued for +reopening when new data is detected. If reading, the file will be closed +after closed_older seconds from when the last bytes were read. +The default is 1 hour + +[id="plugins-{type}s-{plugin}-delimiter"] +===== `delimiter` + + * Value type is <> + * Default value is `"\n"` + +set the new line delimiter, defaults to "\n" + +[id="plugins-{type}s-{plugin}-discover_interval"] +===== `discover_interval` + + * Value type is <> + * Default value is `15` + +How often (in seconds) we expand the filename patterns in the +`path` option to discover new files to watch. + +[id="plugins-{type}s-{plugin}-exclude"] +===== `exclude` + + * Value type is <> + * There is no default value for this setting. + +Exclusions (matched against the filename, not full path). Filename +patterns are valid here, too. For example, if you have +[source,ruby] + path => "/var/log/*" + +You might want to exclude gzipped files: +[source,ruby] + exclude => "*.gz" + +[id="plugins-{type}s-{plugin}-ignore_older"] +===== `ignore_older` + + * Value type is <> + * There is no default value for this setting. + +When the file input discovers a file that was last modified +before the specified timespan in seconds, the file is ignored. +After it's discovery, if an ignored file is modified it is no +longer ignored and any new data is read. By default, this option is +disabled. Note this unit is in seconds. + +[id="plugins-{type}s-{plugin}-max_open_files"] +===== `max_open_files` + + * Value type is <> + * There is no default value for this setting. + +What is the maximum number of file_handles that this input consumes +at any one time. Use close_older to close some files if you need to +process more files than this number. This should not be set to the +maximum the OS can do because file handles are needed for other +LS plugins and OS processes. +The default of 4095 is set in filewatch. + +[id="plugins-{type}s-{plugin}-path"] +===== `path` + + * This is a required setting. + * Value type is <> + * There is no default value for this setting. + +The path(s) to the file(s) to use as an input. +You can use filename patterns here, such as `/var/log/*.log`. +If you use a pattern like `/var/log/**/*.log`, a recursive search +of `/var/log` will be done for all `*.log` files. +Paths must be absolute and cannot be relative. + +You may also configure multiple paths. See an example +on the <>. + +[id="plugins-{type}s-{plugin}-sincedb_path"] +===== `sincedb_path` + + * Value type is <> + * There is no default value for this setting. + +Path of the sincedb database file (keeps track of the current +position of monitored log files) that will be written to disk. +The default will write sincedb files to `/plugins/inputs/file` +NOTE: it must be a file path and not a directory path + +[id="plugins-{type}s-{plugin}-sincedb_write_interval"] +===== `sincedb_write_interval` + + * Value type is <> + * Default value is `15` + +How often (in seconds) to write a since database with the current position of +monitored log files. + +[id="plugins-{type}s-{plugin}-start_position"] +===== `start_position` + + * Value can be any of: `beginning`, `end` + * Default value is `"end"` + +Choose where Logstash starts initially reading files: at the beginning or +at the end. The default behavior treats files like live streams and thus +starts at the end. If you have old data you want to import, set this +to 'beginning'. + +This option only modifies "first contact" situations where a file +is new and not seen before, i.e. files that don't have a current +position recorded in a sincedb file read by Logstash. If a file +has already been seen before, this option has no effect and the +position recorded in the sincedb file will be used. + +[id="plugins-{type}s-{plugin}-stat_interval"] +===== `stat_interval` + + * Value type is <> + * Default value is `1` + +How often (in seconds) we stat files to see if they have been modified. +Increasing this interval will decrease the number of system calls we make, +but increase the time to detect new log lines. + + + +include::{include_path}/{type}.asciidoc[] diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index bd8867d..525159d 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] # Files - s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] + s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) From a4b6da733bd02dea71d994ae9372f01ccd2a2f4b Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 26 May 2017 13:01:03 -0400 Subject: [PATCH 12/91] new build system for jruby9k --- .travis.yml | 18 +++++++++++++++--- Gemfile | 8 +++++++- ci/build.sh | 21 +++++++++++++++++++++ ci/setup.sh | 26 ++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 4 deletions(-) create mode 100755 ci/build.sh create mode 100755 ci/setup.sh diff --git a/.travis.yml b/.travis.yml index 73bc767..f274087 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,20 @@ sudo: false language: ruby cache: bundler +env: rvm: -- jruby-1.7.25 + - jruby-1.7.25 +matrix: + include: + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=master + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=5.x + - rvm: jruby-9.1.9.0 + env: LOGSTASH_BRANCH=feature/9000 + allow_failures: + - rvm: jruby-9.1.9.0 + fast_finish: true +install: true +script: ci/build.sh jdk: oraclejdk8 -script: bundle exec rspec spec --order rand -before_install: [] diff --git a/Gemfile b/Gemfile index 2b03d18..93e5e5d 100644 --- a/Gemfile +++ b/Gemfile @@ -1,4 +1,10 @@ source 'https://rubygems.org' -# Specify your gem's dependencies in logstash-mass_effect.gemspec gemspec + +logstash_path = "../../logstash" + +if Dir.exist?(logstash_path) && ENV["LOGSTASH_SOURCE"] == 1 + gem 'logstash-core', :path => "#{logstash_path}/logstash-core" + gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" +end diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 0000000..076e908 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# version: 1 +######################################################## +# +# AUTOMATICALLY GENERATED! DO NOT EDIT +# +######################################################## +set -e + +echo "Starting build process in: `pwd`" +./ci/setup.sh + +if [[ -f "ci/run.sh" ]]; then + echo "Running custom build script in: `pwd`/ci/run.sh" + ./ci/run.sh +else + echo "Running default build scripts in: `pwd`/ci/build.sh" + bundle install + bundle exec rake vendor + bundle exec rspec spec +fi diff --git a/ci/setup.sh b/ci/setup.sh new file mode 100755 index 0000000..835fa43 --- /dev/null +++ b/ci/setup.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# version: 1 +######################################################## +# +# AUTOMATICALLY GENERATED! DO NOT EDIT +# +######################################################## +set -e +if [ "$LOGSTASH_BRANCH" ]; then + echo "Building plugin using Logstash source" + BASE_DIR=`pwd` + echo "Checking out branch: $LOGSTASH_BRANCH" + git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1 + printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)" + cd ../../logstash + echo "Building plugins with Logstash version:" + cat versions.yml + echo "---" + # We need to build the jars for that specific version + echo "Running gradle assemble in: `pwd`" + ./gradlew assemble + cd $BASE_DIR + export LOGSTASH_SOURCE=1 +else + echo "Building plugin using released gems on rubygems" +fi From a19ad334edb96f2166360e8330ef057cfe2b8e35 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 31 May 2017 16:44:56 -0400 Subject: [PATCH 13/91] Adjusting the build scripts to correctly load the logstash source and allow people to override it --- Gemfile | 5 +++-- ci/build.sh | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 93e5e5d..32cc6fb 100644 --- a/Gemfile +++ b/Gemfile @@ -2,9 +2,10 @@ source 'https://rubygems.org' gemspec -logstash_path = "../../logstash" +logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" +use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" -if Dir.exist?(logstash_path) && ENV["LOGSTASH_SOURCE"] == 1 +if Dir.exist?(logstash_path) && use_logstash_source gem 'logstash-core', :path => "#{logstash_path}/logstash-core" gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" end diff --git a/ci/build.sh b/ci/build.sh index 076e908..06caffd 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -8,11 +8,11 @@ set -e echo "Starting build process in: `pwd`" -./ci/setup.sh +source ./ci/setup.sh if [[ -f "ci/run.sh" ]]; then echo "Running custom build script in: `pwd`/ci/run.sh" - ./ci/run.sh + source ./ci/run.sh else echo "Running default build scripts in: `pwd`/ci/build.sh" bundle install From d33284c4fecd2036b61aea80604208dcc7034e1b Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 13 Jun 2017 09:10:36 -0400 Subject: [PATCH 14/91] update .travis.yml for jruby9k jobs --- .travis.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index f274087..59c937e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,19 +2,15 @@ sudo: false language: ruby cache: bundler -env: +env: rvm: - - jruby-1.7.25 +- jruby-1.7.25 matrix: include: - - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=master - - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=5.x - - rvm: jruby-9.1.9.0 - env: LOGSTASH_BRANCH=feature/9000 - allow_failures: - - rvm: jruby-9.1.9.0 + - rvm: jruby-9.1.10.0 + env: LOGSTASH_BRANCH=master + - rvm: jruby-1.7.25 + env: LOGSTASH_BRANCH=5.x fast_finish: true install: true script: ci/build.sh From 9cdacc3b87a89cf8a7659fda3ae4d1aff863e97f Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 15 Jun 2017 20:00:56 -0400 Subject: [PATCH 15/91] update plugin header for better search results --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 1867214..941c58c 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -14,7 +14,7 @@ END - GENERATED VARIABLES, DO NOT EDIT! [id="plugins-{type}-{plugin}"] -=== File +=== File input plugin include::{include_path}/plugin_header.asciidoc[] From ae3d84c9d2317d8c38ebe58b10fb58e54b1b09b1 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 22 Jun 2017 22:30:32 -0400 Subject: [PATCH 16/91] [skip ci] Updating the plugin doc --- docs/index.asciidoc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 941c58c..6a6f441 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -7,7 +7,7 @@ START - GENERATED VARIABLES, DO NOT EDIT! :version: %VERSION% :release_date: %RELEASE_DATE% :changelog_url: %CHANGELOG_URL% -:include_path: ../../../logstash/docs/include +:include_path: ../../../../logstash/docs/include /////////////////////////////////////////// END - GENERATED VARIABLES, DO NOT EDIT! /////////////////////////////////////////// @@ -92,7 +92,7 @@ will not get picked up. [id="plugins-{type}s-{plugin}-options"] ==== File Input Configuration Options -This plugin supports the following configuration options plus the <> described later. +This plugin supports the following configuration options plus the <> described later. [cols="<,<,<",options="header",] |======================================================================= @@ -110,7 +110,7 @@ This plugin supports the following configuration options plus the <> |<>|No |======================================================================= -Also see <> for a list of options supported by all +Also see <> for a list of options supported by all input plugins.   @@ -252,4 +252,5 @@ but increase the time to detect new log lines. -include::{include_path}/{type}.asciidoc[] +[id="plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] \ No newline at end of file From c3a309bdb89042ddd8e5c3171ae861e69cfc56fa Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 22 Jun 2017 22:54:08 -0400 Subject: [PATCH 17/91] bump patch level for doc generation --- logstash-input-file.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 525159d..56c08b2 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.0.1' + s.version = '4.0.2' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 999a620f2504b837fea907fc8ce6ea5fd48de4a8 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Mon, 26 Jun 2017 21:24:11 -0400 Subject: [PATCH 18/91] [skip ci] Updating the plugin id in the doc to match the index in the docbook --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 6a6f441..96fa191 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -12,7 +12,7 @@ START - GENERATED VARIABLES, DO NOT EDIT! END - GENERATED VARIABLES, DO NOT EDIT! /////////////////////////////////////////// -[id="plugins-{type}-{plugin}"] +[id="plugins-{type}s-{plugin}"] === File input plugin From 0ac3911044d83dc16c644267ee221420d187626d Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 26 Jul 2017 12:38:07 +0100 Subject: [PATCH 19/91] on travis test against 5.6 and 6.0 logstash-core --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 59c937e..7af01f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,11 @@ matrix: include: - rvm: jruby-9.1.10.0 env: LOGSTASH_BRANCH=master + - rvm: jruby-9.1.10.0 + env: LOGSTASH_BRANCH=6.x - rvm: jruby-1.7.25 - env: LOGSTASH_BRANCH=5.x + env: LOGSTASH_BRANCH=5.6 fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 +jdk: oraclejdk8 \ No newline at end of file From bcdb358c3d1a57d25b229252ddf6ca93e9bfeac3 Mon Sep 17 00:00:00 2001 From: Jordan Sissel Date: Tue, 15 Aug 2017 10:01:03 -0700 Subject: [PATCH 20/91] Version bump For https://github.com/elastic/logstash/issues/7993 [ci skip] --- CHANGELOG.md | 3 +++ logstash-input-file.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c57d8f..4c2b857 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.0.3 + - Fix some documentation issues + ## 4.0.1 - Docs: Fix the description with the logstash documentation generator - Fix an issue with the rspec suite not finding log4j diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 56c08b2..61bef03 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.0.2' + s.version = '4.0.3' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From f5a2363532e3f0e3431d4bdc029433cd88c08736 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Fri, 27 Oct 2017 17:24:57 -0500 Subject: [PATCH 21/91] Travis - add 6.0 build, remove default JRuby 1.7 build, bump RVM versions --- .travis.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7af01f7..1458a3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,18 +2,17 @@ sudo: false language: ruby cache: bundler -env: -rvm: -- jruby-1.7.25 matrix: include: - - rvm: jruby-9.1.10.0 + - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=master - - rvm: jruby-9.1.10.0 + - rvm: jruby-9.1.13.0 env: LOGSTASH_BRANCH=6.x - - rvm: jruby-1.7.25 + - rvm: jruby-9.1.13.0 + env: LOGSTASH_BRANCH=6.0 + - rvm: jruby-1.7.27 env: LOGSTASH_BRANCH=5.6 fast_finish: true install: true script: ci/build.sh -jdk: oraclejdk8 \ No newline at end of file +jdk: oraclejdk8 From 9c0c829a11c580e99c5d4cb10c1fea4deb10be91 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 7 Nov 2017 11:18:31 +0000 Subject: [PATCH 22/91] [skip ci] update gemspec summary --- CHANGELOG.md | 3 +++ logstash-input-file.gemspec | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c2b857..902dbec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.0.4 + - Update gemspec summary + ## 4.0.3 - Fix some documentation issues diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 61bef03..57ce29e 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,9 +1,9 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.0.3' + s.version = '4.0.4' s.licenses = ['Apache License (2.0)'] - s.summary = "Stream events from files." + s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elastic"] s.email = 'info@elastic.co' From 89f35525171124a812b289676b3f7ca69272e1f8 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 8 Jan 2018 21:30:57 +0000 Subject: [PATCH 23/91] [skip ci] update license to 2018 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 43976b7..2162c9b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012–2016 Elasticsearch +Copyright (c) 2012-2018 Elasticsearch Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From e5f0e31576729666b0c280fbc841848b79f27ee7 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 24 Jan 2018 16:01:07 +0000 Subject: [PATCH 24/91] fix docs --- docs/index.asciidoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 96fa191..36070c4 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -201,7 +201,8 @@ of `/var/log` will be done for all `*.log` files. Paths must be absolute and cannot be relative. You may also configure multiple paths. See an example -on the <>. +on the {logstash-ref}/configuration-file-structure.html#array[Logstash configuration page]. + [id="plugins-{type}s-{plugin}-sincedb_path"] ===== `sincedb_path` @@ -253,4 +254,4 @@ but increase the time to detect new log lines. [id="plugins-{type}s-{plugin}-common-options"] -include::{include_path}/{type}.asciidoc[] \ No newline at end of file +include::{include_path}/{type}.asciidoc[] From 338390c570c1dda2f003369359ef920d227a4a71 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 6 Apr 2018 23:31:42 +0100 Subject: [PATCH 25/91] set default_codec doc attribute --- CHANGELOG.md | 3 +++ docs/index.asciidoc | 3 +++ logstash-input-file.gemspec | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 902dbec..ff6c0b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.0.5 + - Docs: Set the default_codec doc attribute. + ## 4.0.4 - Update gemspec summary diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 36070c4..86044f3 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -1,5 +1,6 @@ :plugin: file :type: input +:default_codec: plain /////////////////////////////////////////// START - GENERATED VARIABLES, DO NOT EDIT! @@ -255,3 +256,5 @@ but increase the time to detect new log lines. [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] + +:default_codec!: \ No newline at end of file diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 57ce29e..4316146 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.0.4' + s.version = '4.0.5' s.licenses = ['Apache License (2.0)'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From c23d776fbf38fa8af4b66fc471683627f69333dd Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Tue, 24 Apr 2018 14:59:34 +0100 Subject: [PATCH 26/91] Move filewatch lib here, refactor and add new features. (#171) * Pull filewatch library changes into the file input. * housekeeping and one last test case. * more housekeeping * keep gradle wrapper * switch settings from OPTS constant to instance level vars * adjust spec to kick travis into gear * fix failing travis test and rename/limit state_history to recent_states * rename constant, prevent warning in travis. * attempt to fix travis failures, need feedback * make tailing use striped read parallelism * 6.X/master fix in-order event expectations when out-of-order events are dequeued * remove expired sincedb_collection values from memory as well as disk * Updates from the review process. * Some typos fixed * updates per latest review * change gemspec version to 5.0.0 * Change license text in gemspec. Handle Java close methods better. * oops, method was in the wrong place. * Change method name plus log warning. * add exception details * use "normal" exception logging :-) --- .gitignore | 1 + JAR_VERSION | 1 + Rakefile | 6 + build.gradle | 68 +++ gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 54333 bytes gradle/wrapper/gradle-wrapper.properties | 5 + gradlew | 172 +++++++ gradlew.bat | 84 ++++ lib/filewatch/bootstrap.rb | 74 +++ lib/filewatch/discoverer.rb | 94 ++++ lib/filewatch/helper.rb | 65 +++ lib/filewatch/observing_base.rb | 97 ++++ lib/filewatch/observing_read.rb | 23 + lib/filewatch/observing_tail.rb | 22 + lib/filewatch/read_mode/handlers/base.rb | 81 ++++ lib/filewatch/read_mode/handlers/read_file.rb | 47 ++ .../read_mode/handlers/read_zip_file.rb | 57 +++ lib/filewatch/read_mode/processor.rb | 117 +++++ lib/filewatch/settings.rb | 67 +++ lib/filewatch/sincedb_collection.rb | 215 +++++++++ lib/filewatch/sincedb_record_serializer.rb | 70 +++ lib/filewatch/sincedb_value.rb | 87 ++++ lib/filewatch/tail_mode/handlers/base.rb | 124 +++++ lib/filewatch/tail_mode/handlers/create.rb | 17 + .../tail_mode/handlers/create_initial.rb | 21 + lib/filewatch/tail_mode/handlers/delete.rb | 11 + lib/filewatch/tail_mode/handlers/grow.rb | 11 + lib/filewatch/tail_mode/handlers/shrink.rb | 20 + lib/filewatch/tail_mode/handlers/timeout.rb | 10 + lib/filewatch/tail_mode/handlers/unignore.rb | 37 ++ lib/filewatch/tail_mode/processor.rb | 209 +++++++++ lib/filewatch/watch.rb | 107 +++++ lib/filewatch/watched_file.rb | 226 +++++++++ lib/filewatch/watched_files_collection.rb | 84 ++++ lib/filewatch/winhelper.rb | 65 +++ .../inputs/delete_completed_file_handler.rb | 9 + lib/logstash/inputs/file.rb | 270 ++++++----- lib/logstash/inputs/file_listener.rb | 61 +++ .../inputs/log_completed_file_handler.rb | 13 + logstash-input-file.gemspec | 9 +- settings.gradle | 1 + spec/filewatch/buftok_spec.rb | 24 + spec/filewatch/reading_spec.rb | 128 +++++ .../sincedb_record_serializer_spec.rb | 71 +++ spec/filewatch/spec_helper.rb | 120 +++++ spec/filewatch/tailing_spec.rb | 440 ++++++++++++++++++ spec/filewatch/watched_file_spec.rb | 38 ++ .../watched_files_collection_spec.rb | 73 +++ spec/filewatch/winhelper_spec.rb | 22 + spec/fixtures/compressed.log.gz | Bin 0 -> 303 bytes spec/fixtures/compressed.log.gzip | Bin 0 -> 303 bytes spec/fixtures/invalid_utf8.gbk.log | 2 + spec/fixtures/no-final-newline.log | 2 + spec/fixtures/uncompressed.log | 2 + spec/{ => helpers}/spec_helper.rb | 55 +-- spec/inputs/file_read_spec.rb | 155 ++++++ .../{file_spec.rb => file_tail_spec.rb} | 107 ++--- src/main/java/JrubyFileWatchService.java | 11 + .../filewatch/JrubyFileWatchLibrary.java | 191 ++++++++ 59 files changed, 3995 insertions(+), 204 deletions(-) create mode 100644 JAR_VERSION create mode 100644 build.gradle create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100755 gradlew create mode 100644 gradlew.bat create mode 100644 lib/filewatch/bootstrap.rb create mode 100644 lib/filewatch/discoverer.rb create mode 100644 lib/filewatch/helper.rb create mode 100644 lib/filewatch/observing_base.rb create mode 100644 lib/filewatch/observing_read.rb create mode 100644 lib/filewatch/observing_tail.rb create mode 100644 lib/filewatch/read_mode/handlers/base.rb create mode 100644 lib/filewatch/read_mode/handlers/read_file.rb create mode 100644 lib/filewatch/read_mode/handlers/read_zip_file.rb create mode 100644 lib/filewatch/read_mode/processor.rb create mode 100644 lib/filewatch/settings.rb create mode 100644 lib/filewatch/sincedb_collection.rb create mode 100644 lib/filewatch/sincedb_record_serializer.rb create mode 100644 lib/filewatch/sincedb_value.rb create mode 100644 lib/filewatch/tail_mode/handlers/base.rb create mode 100644 lib/filewatch/tail_mode/handlers/create.rb create mode 100644 lib/filewatch/tail_mode/handlers/create_initial.rb create mode 100644 lib/filewatch/tail_mode/handlers/delete.rb create mode 100644 lib/filewatch/tail_mode/handlers/grow.rb create mode 100644 lib/filewatch/tail_mode/handlers/shrink.rb create mode 100644 lib/filewatch/tail_mode/handlers/timeout.rb create mode 100644 lib/filewatch/tail_mode/handlers/unignore.rb create mode 100644 lib/filewatch/tail_mode/processor.rb create mode 100644 lib/filewatch/watch.rb create mode 100644 lib/filewatch/watched_file.rb create mode 100644 lib/filewatch/watched_files_collection.rb create mode 100644 lib/filewatch/winhelper.rb create mode 100644 lib/logstash/inputs/delete_completed_file_handler.rb create mode 100644 lib/logstash/inputs/file_listener.rb create mode 100644 lib/logstash/inputs/log_completed_file_handler.rb create mode 100644 settings.gradle create mode 100644 spec/filewatch/buftok_spec.rb create mode 100644 spec/filewatch/reading_spec.rb create mode 100644 spec/filewatch/sincedb_record_serializer_spec.rb create mode 100644 spec/filewatch/spec_helper.rb create mode 100644 spec/filewatch/tailing_spec.rb create mode 100644 spec/filewatch/watched_file_spec.rb create mode 100644 spec/filewatch/watched_files_collection_spec.rb create mode 100644 spec/filewatch/winhelper_spec.rb create mode 100644 spec/fixtures/compressed.log.gz create mode 100644 spec/fixtures/compressed.log.gzip create mode 100644 spec/fixtures/invalid_utf8.gbk.log create mode 100644 spec/fixtures/no-final-newline.log create mode 100644 spec/fixtures/uncompressed.log rename spec/{ => helpers}/spec_helper.rb (53%) create mode 100644 spec/inputs/file_read_spec.rb rename spec/inputs/{file_spec.rb => file_tail_spec.rb} (79%) create mode 100644 src/main/java/JrubyFileWatchService.java create mode 100644 src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java diff --git a/.gitignore b/.gitignore index 3300a23..8e7167a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ Gemfile.lock .bundle vendor +lib/jars diff --git a/JAR_VERSION b/JAR_VERSION new file mode 100644 index 0000000..3eefcb9 --- /dev/null +++ b/JAR_VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/Rakefile b/Rakefile index 4f4b858..e1595e1 100644 --- a/Rakefile +++ b/Rakefile @@ -5,3 +5,9 @@ task :default do end require "logstash/devutils/rake" + +desc "Compile and put filewatch jar into lib/jars" +task :vendor do + exit(1) unless system './gradlew clean jar' + puts "-------------------> built filewatch jar via rake" +end diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..f2fd43f --- /dev/null +++ b/build.gradle @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: "java" +apply plugin: "distribution" +apply plugin: "idea" + +group = 'org.logstash.filewatch' +version file("JAR_VERSION").text.replaceAll("\\s","") + +repositories { + mavenCentral() +} + +project.sourceCompatibility = 1.8 + +dependencies { + compileOnly group: 'org.jruby', name: 'jruby-complete', version: "9.1.13.0" +} + +task sourcesJar(type: Jar, dependsOn: classes) { + from sourceSets.main.allSource + classifier 'sources' + extension 'jar' +} + +task javadocJar(type: Jar, dependsOn: javadoc) { + from javadoc.destinationDir + classifier 'javadoc' + extension 'jar' +} + +task copyGemjar(type: Copy, dependsOn: sourcesJar) { + from project.jar + into project.file('lib/jars/') +} + +task cleanGemjar { + delete fileTree(project.file('lib/jars/')) { + include '*.jar' + } +} + +clean.dependsOn(cleanGemjar) +jar.finalizedBy(copyGemjar) + + +// See http://www.gradle.org/docs/current/userguide/gradle_wrapper.html +task wrapper(type: Wrapper) { + description = 'Install Gradle wrapper' + gradleVersion = '4.5.1' +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..c44b679acd3f794ddbb3aa5e919244914911014a GIT binary patch literal 54333 zcmagFV|ZrKvM!pAZQHhO+qP}9lTNfnHSl14(}!ze#uNJ zOwq~Ee}g>(n5P|-=+d-fQIs8&nEo1Q%{s|E!?|<4b^Z2lL;fA*|Ct;3-)|>ZtN&|S z|6d)r|I)E?H8Hoh_#ai#{#Dh>)x_D^!u9_$x%Smfzy3S)@4vr>;Xj**Iyt$!x&O6S zFtKq|b2o8yw{T@Nvo~>bi`CTeTF^xPLZ3(@6UVgr1|-kXM%ou=mdwiYxeB+94NgzDs+mE)Ga+Ly^k_UH5C z*$Tw4Ux`)JTW`clSj;wSpTkMxf3h5LYZ1X_d)yXW39j4pj@5OViiw2LqS+g3&3DWCnmgtrSQI?dL z?736Cw-uVf{12@tn8aO-Oj#09rPV4r!sQb^CA#PVOYHVQ3o4IRb=geYI24u(TkJ_i zeIuFQjqR?9MV`{2zUTgY&5dir>e+r^4-|bz zj74-^qyKBQV;#1R!8px8%^jiw!A6YsZkWLPO;$jv-(VxTfR1_~!I*Ys2nv?I7ysM0 z7K{`Zqkb@Z6lPyZmo{6M9sqY>f5*Kxy8XUbR9<~DHaC-1vv_JhtwqML&;rnKLSx&ip0h7nfzl)zBI70rUw7GZa>0*W8ARZjPnUuaPO!C08To znN$lYRGtyx)d$qTbYC^yIq&}hvN86-JEfSOr=Yk3K+pnGXWh^}0W_iMI@ z#=E=vL~t~qMd}^8FwgE_Mh}SWQp}xh?Ptbx$dzRPv77DIaRJ6o>qaYHSfE+_iS}ln z;@I!?iQl?8_2qITV{flaG_57C@=ALS|2|j7vjAC>jO<&MGec#;zQk%z4%%092eYXS z$fem@kSEJ6vQ-mH7!LNN>6H<_FOv{e5MDoMMwlg-afq#-w|Zp`$bZd80?qenAuQDk z@eKC-BaSg(#_Mhzv-DkTBi^iqwhm+jr8Jk2l~Ov2PKb&p^66tp9fM#(X?G$bNO0Qi#d^7jA2|Yb{Dty# z%ZrTuE9^^3|C$RP+WP{0rkD?)s2l$4{Trw&a`MBWP^5|ePiRe)eh1Krh{58%6G`pp zynITQL*j8WTo+N)p9HdEIrj0Sk^2vNlH_(&Cx0|VryTNz?8rT;(%{mcd2hFfqoh+7 z%)@$#TT?X0%)UQOD6wQ@!e3UK20`qWR$96Bs_lLEKCz0CM~I;EhNQ)YC8*fhAp;-y zG9ro^VEXfQj~>oiXu^b~#H=cDFq1m~pQM-f9r{}qrS#~je-yDxh1&sV2w@HhbD%rQ zvqF(aK|1^PfDY)2QmT*?RbqHsa?*q%=?fqC^^43G)W3!c>kxCx;=d>6@4rI!pHEJ4 zCoe~PClhmWmVca=0Wk`&1I)-_+twVqbe>EhaLa(aej;ZQMt%`{F?$#pnW~;_IHaAz zA#|5>{v!dxN&ouieHdb~fuGo>qW(ax^of8<3X{&(+Br@1bJ-0D6Chg$u$TReI=h+y zn=&-aBZ`g+mci#-+(2$LD5yFHMAVg8vNINQOHN6e4|jQhIb$~sO;+G?IYshZf)V{ZewQR z?(|^o>0Xre^gj!6e}> zTHb#iYu$Pe=|&3Y8bm`B=667b-*KMXwSbr9({a6%5J<}HiX`8&@sTKOHJuGG}oFsx9y^}APB2zP0xIzxS_Hyg5{(XFBs z^>x@qc<{m0R5JuE`~*Xx7j+Mlh8yU;#jl1$rp4`hqz$;RC(C47%q!OKCIUijULB^8 z@%X9OuE)qY7Y3_p2)FZG`{jy-MTvXFVG>m?arA&;;8L#XXv_zYE+xzlG3w?7{|{(+ z2PBOSHD7x?RN0^yTs(HvAFmAfOrff>@4q|H*h<19zai;uT@_RhlZef4L?;a`f&ps% z144>YiGZ|W%_IOSwunC&S$T1Z&LDI1EpAN4{D|F_9c^cK8`g zQ4t*yzU*=>_rK=h1_qv3NR56)5-ZsGV}C?MxA2mI>g$u>i9xQqxTY3CP6SFlmqT*kJm+Vp&6|Rd&HVjVV2iE;dO7g%DBvpKxz}%|=eqatxbO9J z26Tmn5nFnvGuWhCeQ?Xl{9b3Zn?76X;Ed_yB`4Tuh{@)~0u0g-+Z&_LbVuvfXZ0hi z<)Dcp(7mi{4J2=wr$jn!SYp3yKg*nj)GwiiYeB6=Jz5 ze_>nw@IjCW&>1ztev$h~1=OFs*n#QYa*6y3!u>`NWVdsD^W6FZ)$O=LbgMzY=6aNW zplFoLX0&iKqna6%IMp|Pv~7NW-SmpI>TkgLhX&(~iQtdJ4)~YUD3|+3J-`WfB|P2T zKia5&pE5L|hjvX`9gmw7v=bVal$_n*B&#A(4ZvvYVPfl@PI(5e!i4KS_sd`yS0R*R zt|Yp((|SofnsEsS8|&NyWo{U<<66>|)Ny{8(!hRcc&anv%ru(Oac)?%qn}g3etD=i zt6c#E^r&Ee#V}}Gw*0b1*n829iQ&QWLudUqSuO3_7xb~%Y!oRTVaOEei3o>?hmsf) z;_S_U>QXOG$fT6jv$dsI*kSvnPz=lrX#`RUNgb><2ex!06DPaN9^bVm^9pB1w&da} zI*&uh$!}B4)}{XY$ZZ6Nm0DP#+Y&@Ip9K%wCd;-QFPlDRJHLtFX~{V>`?TLxj8*x9 z*jS4bpX>d!Y&MZQ6EDrOY)o3BTi4E%6^Mp#l zq~RuQGD*{Kt9jrupV_gAjFggPSviGh)%1f35fvMk zrQGJZx2EnWQBy8XP+BjYan<&eGzs{tifUr7v1YdZH&>PQ$B7|UWPCr_Dp`oC%^0Rx zRsQMQ7@_=I8}s$7eOHa7i>cw?BIWKXa(W9-?dj+%`j)E%hfDjn$ywH=Zkko}o96NuqwWpty9I2QtUU6%Hh#}_->hVJ-f711&8$r7V~O^7sth1qdm+?fD?&gIjAc zyqFI*LNCe9r)#GW?r@x@=2cx756awNnnx7U6`y?7hMG~_*tSv_iX)jBjoam}%=SnL zQ>U^OCihLy24_3n!SV-gS zOc&9qhB7Ek%eZMq6j(?A@-DKtoAhCsG+Uuq3MlDQHgk4SY)xK$_R~$fy+|1^I3G2_ z%5Ss|QBcETpy^7Fak21m_;GRNFx4lC$y8Fsv?Ai^RuL6`{ZB<{Vh#&W=x%}TG%(@; zT)NU7Dy$MnbU{*R-74J&=92U75>jfM3qQ=|sBrk_gUpJ|3@m-(S} zqrmISaynDD_ioO6)*i^7o0;!bDMmWp0YMpaG8btAu^OJ)=_<07isXtT+3lF76nBJ{ z`;coD)dJ6*+R@2)aG#M$ba<~O=E&W~Ufgk7r@zL&qQ~h_DGzk<>-6*EUF#I+(fVvF zF0q3(GM8?WRWvoMY~XEg>9%PN1tw>wLt5DP-`2`e)KL%jgPt=`R_Tf+MJBwzz@6P` zYkcqgt{25RF6%_*@D6opLzleQ)7W@Gs4H3i#4LADwy$Js;!`pfiwBoJts0Aw#g{Mb zYooE6OW7NcUMd1}sH)Ri=3(K0WmBtvK!2KaY?U&Htr#Q|+gK<+)P!19dIyUlV-~ZD zWTnl`xcUr)m5@2S1Lk4U(6nbH$;vl%qb5Vh|G5KA{_*04p!LOkPsWhxMRz}sl&mDWMOvz5;Kq0`+&T6$VoLdpvEBn-UN`Yb8ZZ0wMcv3XC z&vdicA-t=}LW3(&B6Kj(>TT!YHdrG%6Mp}$B2)7 z+;)t8QsBkfxDOo?z_{=$3mKym5Go;g$Mk=-laVV$8~3tYKU*>B?!wZzsj%|0`(rDZ zQlak~9a?7KG<`P_r`)fK5tmRtfJx2_{|%4C{wGh4l@LS$tQ$Tbg&CH~tGKZcy%EgW z`Ej2=-Hlzs6Deb(!HzY)2>45_jU5(2ZZtAeg#)2VsD^#*$8x<;w5s&*^tt+nA0nto#6hJ&M?xQ5=lhI*Tap+o@#YI~Hi-l#@sdjZ4PCVcFr zrtJF2C$N~X&6L4W47_$Flt4D!po1W~)1L9HNr#|W_L09d`a-4_H0Mx`rv5icDMbTk zjgibis*{cth+j!U;jr1ejW?${hBE1{p6EKm8=(ABt9m z73d7-{oHvvZQ4|t%Yl|k2ISat%`52J25OJ=M|CD{m|Q`~Q%t0|TS>zV%Z(g_Tfm4* zrnW_nWqsh&V(Vg+lY`u)?gp>c{g&12){~5SxL)&$i>$($pDhnsXK=$u3m0Cx-kD$+ z5Sf?E*TYQ#^KvHWJU1%*={yG9NjM(7`Q)rS7&uMenLoOe2N*xk(vN5F{sf(%CH8#I;sdqf1dw%kBI&pS`K)){>EF18AT6CAYZz0_Bc|Ws1Nh3 z%twB`i+Lm2(%hoXJP|J5lGpD^-5BDO7S(}JJ>5B*GC`HoszjIH2&%(H9^gwUpLh!i z3Qy1nE2J}h@;Ak+bcPP0N_i9XP zGP%F-_xo6mx<}RTyu}Gtjo&rvdJ)cjDjdsF2#cIzUZPQ4jw3ooBicqI*=>s6PhTHP zUbqtt70zm3RGvU{bmEBy@7>pUvN*V&xd}e^Utpe0V;b_!mCArr(MJKQnMqizhhON$ z0PU2%@B_9xKJKKe6`VjcwmWC;Y0r{P@{$)pR~JK z7W*a7V+;ltQ(0F8#ai=9MTrhuKUuc?XHbAd#{@4h9w}rzVRuq6yXejFE!8sdL8=54 zlMy{taj5+w=D#noC@!#8;au}K+eZu|Qu0-kgkp6xNYzcURuN-6Kl%)%2VR8!wVGU1 zWZEqJTSbol6_)?Gn*57aSh-rbxyjqOxm!5?6VUdE?S~B!MwhszTd>6tpLmj(o$a(h zAs07xg*#7|8#vhWTd4=LC(iu_{`BjJsuC)6y+j zVt~bjACA>0y~vnuy8LtP`50?}Sv@t*JN-yL!!hVgrCPk1MZ}gKt0uixMw>b}LVSYT zO2tkmt!7v#jQQ>8j*U6`G)hEPOU>LGS_Bb0_fM;F-V(W)wq65Rk*aya3yO z_E*B&%-+Mz#?wO5#@<52%(}O6W4o%BNVbB8s4!4(PR*gSb z$j7Eencvf9?_))K7b19T597Ql)q~!PlMm$u$j3)NoBF(=YuwSFa=2J3EM=@!qJ=bK z2UY^`gcpl_0a{Nbh&mL-S}|dXDc@FYTzkR9u>DlO|r9zMbY9 zcvi~*Sn!-XdibS9>V|VmH54$J!N;-k>U|!e$!EePWpr0wZn4~|?w4vo%-Ffcx{+}N z74+Dx>^&$SsYtq~oLkztY&j;cG5S5NN)rYFS~F@`)MVA%911fMO^vLB+%;E2kGcx|C?bj%K*Y#Btv7K6inqIt~eN9{d@I&&(VF z1}bT14cQy!1jpa|7DiCJuBh_{+56)f_l3}qLWwox4&D>1NwX@~lG&(9Cp!ZS@vbCbV>$9jV0PWrUoc zGQm`Y5){E1K~q2RUK#=U*e^6&?8-y!fP9=6o+W+4nm+mSQeDNJD5!E8CaU;I#+HM)Gt`;3%$yq7H_kqm0#(U8c<8HUpZ5@8zRzEG5L^AX4{< zwDEN(lUW!^k%H!t&T_;T6To1i4r0S|tu+lWr|`3wjbo+~>MjOj62{&D3H$OiWs=Dw z`m6MW^8|~J3*ER5G^h~UbH*UPW$7ZHfg&@9%r2u(d@8YN94k?}pzw`3tuCNVl%MV&<#4ESfo@VX7dX=)C-e#!(E` z#+;b>rvW^#ug1(yr&cS%w96I($;2(O*FuVoTK-KiA2Qgwkhs0^Xt=eXkh&mx)iBSK z+r|&Xi($%(!3BO6G7f)2qliGTP)G50)i_iAAQYn_^v$7h=>j<98G2H|p1$BA(xe5i z0+-b-VX6A*!r*B>W<`WMPAsKiypzr_G25*NMBd*U0dSwuCz+0CPmX1%rGDw|L|sg- zFo|-kDGXpl#GVVhHIe#KRr^fX8dd>odTlP=D0<~ke(zU1xB8^1);p2#8t_>~o&?jKIG49W)EmhTo5fZ|aP=E2~}6=bv=O`0e4FpgaP@U~KHt>V*oR z{wKtxe`uCFdgYHlbLL2`H>|$?L@G&exvem8R^wQppk+Gu8BI;LR4v=pU`U4vlmwFw zxYbNZXbzdqO{7#b`Eo2>XlNcQEFC-Gk2v__^hqHG{bb%6gvMRe9ikQ>94zOK3o85` z)Ew{!is}|b0%g#qa2H+$A1i=5;*y)hv$5m)&;Z~CTv zpdZz#9k)yhrLH%G>|ly;%|Fe`K{}d{6vyNO^Gk$ZYOIL$3&5XuJTqse&XvY7TH(_z zb3L0aT`$6i&c(dBQVcLsV?yM^@BTj>C_2=Ih6Yxsk zP5r-Yg34bu;lJUUrT!1Gt>I?jD(&Q8A@Ag5=i&TcT(g><60QjPmt>;B(xYk(bt}+T z4_t3m_flhFXrd}o9hw+M$vh0Ej(*GdO21EJaL-eD*b$UHHZnUN|OJ z0Jp^;Ep{EvhbQw6K_&t~eB7m4_csSE=CWXyWY4sLL-`>gdwbXUqW8FqVwQ((K>Hes z6?QDu2SZjI&_Oqc`A&D$)~oa&r%dn2G?-*9nvEt&L!4PeU(lyXCgK1^guGj|F$M$j z(GuZXkiyMXV}lhNuz5oi;9>+0nCgNO|gp>9FS%CFa9W(t_WRn1h zi*Vk4IQG@3-{J`U=9`Ky!DmF2O%ld1w#`8Drc@C6KGz2^NhY^gQZo9SG}}BF9G0<> zUIO))F&%dt6uAb`cN%_jf&q5I)?_7J^9T09fb~#ll%%T{?}PznT^_22(*OROJ`X;tg`78+=eW z{nLQs1%;?R)4yhs=QXy;Ww3ta7dfE~<&UNFZ#6bKVY=m1@p+4G(=Yx{7vDsa`}d$v2%*jQt+wTN!@Q4~!T4`0#GI8YfG!RD zA-RJ))sAlYej5x5RQ-^2I`1%|`iFfD*JoRd`hJ1Hjq_1EjBZ7V)S;?@^TS;{^==d= z)f-C;4#XD*THtvXh>{A80hZC?O(tJ)M}tK1Z4n%Y}= z7G#ciWgC-qm?9fE0?893;j3|Em(+qaH${U|Z^A^QleR%Z7 z1tb3_8mwUDjv6g+M+PH*#OmXvrsOq;C|~Oa;`LR+=Ou;zBgy?^)d&PxR|BoHj6&sQLvauxiJO7V_3Dc#Yum zGB>eK>>aZ64e9dY{FHaG&8nfRUW*u+r;2EK&_#d;m#{&#@xVG;SRy=AUe9+PcYYs7 zj96WKYn5YVi{SKZ^0v}b<>~7D3U^W@eJTVKCDk#O!fc5%`1KJ%473-~Ep)z$w6SC^ zTLzy~^~c+8J4q^gv9G_h((u6+#9K|Hwyv?kkbEpaO6^U013F*&bbnuxwtH~v%F9#0 zmtLmWALa{|zD`KnzKOv=DK^Qdb+qyOnd??*IXEprOa{&tVKg3pExuAFe~YQ4t|)j) zij8hA%U)XCd1Xs~{O?y^$^Ay>@J#8GF%+8%LcH*p@gmDRZXB5qIXD z8>)QYQpTPLtK)oS#azTHeBGCqsnlj9NCIGNEpJb;iSSJPZ2?lGVE8nj#y*wRnoLNP zUDvlQvp`STbAjrwgsMtnowuaK;8{D_vB36%w zJv*S667QTThf?Cmh=Z!={xFo+ID2<-Vy`H~ArX{AKl+?KW=|8LZO0Np%7v|KE(}&? zkm-iqK;uMF5)cH3KYs+zl0BM%jvE+hMDx-L*xqRy;-OS_rAK2sX;%0n1!Ma{5Lmy9 z^imumWb?xIHBgd8Q<3ZITO&oZe53WDFt~k-gkZB#xr?4x**{ecHCK=){(+%{U)emp7C}WTX-ec@8h(}WY4jqVq71BVnXwP*x&;{_d zN*3_vi&qrs&)e8zxt-odRm_T)R;UhvD$t{UlTf!SlB8E1GF4cNqHtgHu}%8Q8%zI^ zpO2!5*(g*etB5GgYL`Ac=M!b)Xq2bNT3ITjN-o2|WjTohM*|Zlubs@v$LuHc` zZ9L$4X`?POL_=tgyId{qVRj|31h_W~uwSBS8Ah`MRZtYNw3)JW;zH~Pv)aMi=uCgq z#Os}gx^be(^r#pj-M0If8r_YMPZT)4&1&7mrz) zh!z$uE9c|~q;;`W8Ai3H!KF-#GtuGf98}gBI3*2zD4rHswCwmtL-<*{PH$;(Ich%i zT*e+^HTbEiukgv7AMqKZ_!%!^91tMZXJ&a+eBiBB>)uZd6=!3wJGNOlZBqfyTo_(Jq z52h7Y#wYwKScBP<{-&F}%`x@JiQDol9`9Y82JRmh8^6_R_^6I7I(oY45vsM)2Mg0! zNA^4MWmRnm?JM)uuzN;;ogInuA5}Qk;oaQ$cs9Ai)!zvU7TmWOs>`bxrdCQ#mnxk} z5Qpoyg#i0duj8%&Cc)XL_UW9Y?IgF{#`HuraxSoAO7mma*cOEu@T)wAF;<^bOp|dR zADP}}$WhfJnAd^kp5&R5b(nQw_sNEB!jZ-p!ty@M!(=`!YrVm5qzwmXy!+l^Qp||H zv)&M{iBPo$VxFKnW{T}^(SSQhrcO8bGeIkBJ=JR;#?sW8mMt~^yS(gY`@?F17Z%jH zb{eMek^AG53t{vvM+t+R{@qK?fCZn7^EkTA!lZMl?}J59=&K`ZSgNCVJpfBBkb%)0eYGJXVS%p1UU)y*F6#Od-P`RT#1*&Ua*G-rTNAwiZ_43phR z$Tt_#Lfj(r=Zu@nx5yBV zF=8b~y8XrjculznaTL$d_A?<3CJzV%`@=R?nu3qGhpnniU7b64jQx=U%#3e_@5n7P z9CZn~<+hnXIoahha&pWlKH!M&^LRKwKLg-_J)&7>fN$!Zhh*IevmsWNm%}J!& zx5esSGz=)HgFY>*tW#_Bh8hH?clu~3dMZr!u|cf<&P_Ks1R4orwjF4Qmy<{9I7j2^-P1Qe-E$ZHv^Y2|8)>4abo8@^ExNA7B+Oy;0NIqz z!#d;E2rU+kkB0P#KYyn7N;Nuo2k!qQugm($Hr+YiqO^0y2CRX2m^!SZq@xDICbo~5 z6K1##iSi zz-lajV(rBC^a}AEt3AqMcJSKZsorc=(iiiCwip4!9->vgGF5(@L;ix&mq$LxsQ;yn zCD@C_!;8(Kv^6$mb||Lfhhf5I6~WBlJ&cje30%f>NXFsAPq<6#QkQbOXF|Tn)4360 z9ZbI~k=SJ5#>G^Tk#7(x7#q*dL8Sx?4!s4*FGxDT3=jA- zd3uD7(hY0)XnNaS4GSis{9xF|$|=it<}R2GMf5Wql`jRfCIlWupKy@#xLkR# zzy28n_OG7iR%5>`{zXeUk^Xy69o^hb?Ct;Aua~R!?uV|06R7mWI$`-8S=U+5dQNhM z9s#aU873GO#z8Dy7*7=3%%h3V9+Hyn{DMBc>JiWew5`@Gwe3-l_Nq*xKzBH=U3-iE z^S$p)>!sqFt2ukqJ`MWF=P8G0+duu;f17Wc$LD>!z8BIM?+Xa8che3}l(H+vip?rN zmY_r$9RkS~39e{MO_?Yzg1K;KPT?$jv_RTuk&)P+*soxUT1qYm&lKDw?VqTQ%1uUT zmCPM}PwG>IM$|7Qv1``k--JdqO2vCC<1Y(PqH-1)%9q(|e$hwGPd83}5d~GExM|@R zBpbvU{*sds{b~YOaqyS#(!m;7!FP>%-U9*#Xa%fS%Lbx0X!c_gTQ_QIyy)Dc6#Hr4 z2h++MI(zSGDx;h_rrWJ%@OaAd34-iHC9B05u6e0yO^4aUl?u6zeTVJm*kFN~0_QlT zNv9T613ncxsZW(l%w`Lcf8uh@QgOnrm@^!>hcB=(a!3*OzFIV{R;wE73{p_aFYtg2 zzCY5;Ui~l_OVU;KGeSM9-wd66)uL6N3DqJHJ0L6rET&y2=f)>fP6;^5N)R`BXeL+& zo6QZ-BrVcmm1m{!!%^&u^*L!e>>{Tg?Du<%-A6<{O8xZCvmdNv?|;Xmm;55oj300) zByD!GlJZaPau!g@XX#!j!>VHPl5bWf^qk=Z+M%N_!myUu=dg$C;S{|)(pcrOI5b6g zcV*=qSI|KVEI(o_(QiDzss>!+>B>W5IhxlS^Eop*rIB0e3~F_Ry*d7(0zb2SYv%Kb z_K~7;{#bI4uy<>P8(6oG^->yVwA%#Ga{s{Xn{$C^=B;Y4GEp4m=&suBjN6XN-ws|h z6tG__V^Wl+rCfTPUf8trHW>GCue? z58?dkGg|8!;YQ(dl}+2_Im{K0{l$)Ec5rW*Y2Z!w?tGQ@ZkO%A?&@KMXBFF9EHi`i zOwT#+Fz~do?#nt1Hz3;_?3rEQU^K$J2BgxOX2AT>!bmMv8&0nQSVYKW83j(9ZEV#w zjN&G|L)`7uiV;>?**_x)mP$&Zg}sh;>8W-$u!qozJS8IH9zQ1|+90mWT-zni7m2b0$Anx2<6 zpgF=^bxuc|t#XClG*jIl^LA3hx?Z^%49PiWfiUKeVVv(xH_AIRe8-Pl=_1S?FaEF$ zZ!IPxsXgx_Sl%jaPlB<1tvQ^!2ii2R`W@xr@#^kRW!y^B-x4+3`V!9)HHE^F%>IqO zh;0Ul3|&UwF?&L-&5@Spcs2w(uSgY{aIB{MbAqjDb%)nrZUw`=7S+4d)K9AS5NS1B ztX^Dm+m$5hO#;9xtxqoNB6(|gHUyBn4`2C_<%a8abEB~01nwRf!?+T#Big__!bMbF zt|-LS;8LPy3a$3$gAD6^;xulrXsZXjKW-1pFu829!mWo?yqwx&THb1Th-c*q*u2^k zeefe7T+G~7CiS=Z5~B?}bW-J>-WuqL13Xx~@Q^)QhHxDgk+x*nyVFjnX8tR1^Sdl-R(PR#|j?hx!oryI`_wmmB4z4{7wrEBF>sclHoe z2JB6c#_$aL%lp4!UAb@_!sLIi3O&()fDr#T(f=PY@t^ItF#Z^atwL1KN7GYN4G^O3 zHDst`gr4lwxJkr~B*Z2x#CzmkNiiD~)46h}=bA*Cx|c;BZ5Un^r5fs}?6g3Svj=j;fV|OR^i@=cCh)VMW_5+L*;k;r!;9t>|w{@)`;;)E->kUinNJ?X8kN! z8`}GhsA>#DPeGkd8dg4r`L zyS19T8YH@ihS=4~WrkUhg$=sYId}&g^9vO>KCnTIzZ66a=?JDsc*B=vngxfB?;*qV zL|Xu(P(H={Trz4ndsE#KyKv}^sWN(EEpcsO6`4%x-hL6fp-yZ@=m!LME{*J|u;(PU zhn!*SVlA=jA^0#&C;}}4DRC|Tk)2eG1v`?uIH(hb7|mL7IBeI~W6fP_36}|0t9q!} z@!h`tf|zFCFY8G0K$!&iwF*jOb@C9E-u5s?^Rlaad%bCX{YDpPTBm z829R2aPrE$*^pP7-pjT|pATPS5NnI|WwT++-L34$e1-}4%*dsYYnu}Hm#92MgFE{o~NjJ{EMM1=Mai)NW%TmhhCo7lUYkk_3rXFLXs;*u? zgRA~x>&_K>WvT0`Pd9_t44Z?otM8lH}ukI$yM3RtOb}S@I`i-+*_MWx=B>k@KtGEN8>e7{~g_4w!LHb-T8%?i{F01C+zU_~n>ZWyA#$r92il-{03qE7w z=Cpz1(vmmZVhNpscjG0M0K4$Tenmdqi6Sa_1=KMJKbaxz-TB2#j| z6%G1&3`Cs*FXeBf5(kCLyAWQvCo0ZsL(P{pXxPqF2l6D7M->xL%)qCYEkc|mAi<}j zM!2f7X2*gpVHIkatPI>>9cVyXLNiS%vFL9?smnYBm z(8k{xAaDSFG3*O+n{p-<+h z7l32L?Kv`Udr$(2lSmFBW$yYNd>T2?L+3N;I5dSOJ3s}q5#UX0X^z@DgEB$HV&10A zh$rhWVb)Pj!doaXx0#;$Bcn=|-z~XKopH&SA^!)ZkvcurJVErdUW4&BwdCV8j+VY$ zciQn&1L7%B8%%^|UFw={uTc`symy1L3LMfFY3N*^yU?cSJQCgLc%}394vUB-)Itp( z))pWllOb*Nj8O0}RkoI!FBX!U4yC?kPD@vFu|>qeg`S&VXlPQMy2}GEa<|}5e#^L&lXX^D1U!rce9c0+G>TC7~L+bTW5AF8gv#eYG z_;WNQQpE>x&kqA*?^}TS2B(=Mr5>Ase_e4xngO--eRT4DtMq`h?QLjn;YW)HTixlc zpnP+~DkXWgh7H1Lu2wUeE>u&y<%4N*+>;F)+x=UWvKjon(XuB@r$%7Jb7cQh^@qdO zM9XJ}Xo(M1KWX8xU^Y0d(B!s?4bx`v-M6p0@$DZP?GrT3lb%%H>>?4TX%etz)cC`dOmZ__G2X+AGcJoGFy@wtQ zeakz$cBhhehjg_(SuL#qVk-xYE(aUTzIG8AK3XD0mZM0EJ13YVzUS$oZg^^hO{b+^ zWy#6}LqU}|3q#lZqO#g=>*2Az7iHbW68sdBHa@f4CwB*}eQsFu7Tt1TJhp;6vXBue z4Z&aWG#~BbN)h`=E<(Vw-4-1?9pAqoG$@yitG#M$ z{V)~zAZdJ9n{7$_oi$!R(XyIv*uawdn?iLi0_|*UpE{z}H(+r#IfP9?u^% z!kKxcc+??s1pNs5YaXS!5+zbthP-;O;!^z!rLXWNUgHa3&8% zFnn7A;Y{bf;(_n0W1vs@RX}8v>GhLDF1~V3{R_i?vJdlO68|#BgDk4eW|fA=Px|8~ zxE(@omgp2MOi2Be%RhF!?{Ga)FTRJW;ECWYF+u9F?c_jdOf1i1BmIzVaa^@Hjh%Dc z?F+^by1;e_#f|(klA^TO3A`*eE5&0ZPj%0yYALQ9XCW@RI&St+OHRvu1>@Onb5fQeP=E$YVLhC zMpkEIz*}74t>;PK?7p#~Z%%f?7~v`0DRg{|bgVzLd*4!|S_D~Bs^i}}-~bm7W%PuM#$_t2fExWw_|WAamWxY6S=i?9Vv z%r%BcXG@HRZ58<(=pqR3&TX^GGZa(U>rmsz|48$YB!5Mbd}P5~h{T9z78BD2Hc~3x zKc=D%SQ$%P6OieeGg?oR7gqz4+_JkSUx-yl&y1FKX^s)nU<6PVuXc@ z5Q^F76 z{SeBk&t7-TvH9etn33qag}(s;Y#{$}DuS}%Dsh-D+#S{21Xu}Sk&DG)xHL^Qw|H>V zxET9a!QifM%L2`JPex5!_AtdT_*%k`VeIDQ?HT<-M)oaKV}&lR%R{pCedOz43WD^xnWfcqCkBF@ z9VL7YK`@>c7LO}V=2TqML`PYb>%P~dvj3iOGBECvD{|;Qxf^$-ay$lo8O#nsR?je@BD*SU*98?E={03WiP!k{}RCQ9m z$}#Jzcn)I25#^-Qz>JN^??=RtAucr-Jg~DzhqOS$;j`Nvn04M4em6Ki1o7#9mexRO za1Xpdyz4D?3QY~9CFGp2%?f=2jo6e$v!*L(L}2VrIGXj$Qo`z2<~wn>{lP=(&WO_z z%zI*bMxNYxqS^^Q%LdYtVK#tB?aiXO4M+CB82bvCy5B5q+}+)^xE3hx?(XjHPO%Hc zp}4!dLve~*ad&rj`|j+_?#}#o_RA)akU$`p-?{HO?{gm6pZ01@yeN33rIEH6_h#S& zAtyDiJrVMTQI^fsYm9y9uY^o2bTA1eX3xK4_JcOpgRO?X!s>CM^h@c2{%VH*gzC+X zm|DU@rf9<$tml$Jms2>4!=KJ6d8-32{Whg&RZ)|_&kVZ0FTt!Gs9OJ(PnX+!>5)Qh zUlC8RiylPF@@L#Kl%)qKKc6ZzJ_2|rcY##{ID-2IQXd(&W*dO0U`Xf^_O3hzv+xkb zyWZ`jB(PC_st2sEDep$CoUQ^V_XIDXDA&I?s}bkBW^0jQ{7$(3#>|Pt&`$Eg+Gz5E z;1W~$+#bKU41|KrdzjU-}M$(v|Z_GtP$3uCNzu7r6tT zbL<-Yzs4_hl6Ar@TVoqX`_{xb0v&U6)YpWp#kj60veHC!+z-J61{@B5su999=xpMx-gS$e@eFvqMEK%gabP9K}#r0IvW%eC!?X4N_8L|4?qdX5#mx^1+!K`l5>-B!e?Zi&>J~yXe z^EiDXWNlAa=vKuV@D7qCAc#+)(rDN_h$lAQQr1NEM1~of6g0s&*Wa7$zfuqBC5F}q zIq_;)KITrRf4ja2p8@)7#`a)Uf-R*tDDuh~r5&3r|B*a)_||C;726hD33bKC@ZHC# z?zQfi_d71~w6Ulk;z5n@cnfKt56Ynic~^~u?4{Um-f)^FWFF-Hjo6)cC(RcWV-pld zUNDj_5A{hC~NfI(fVO2HkQ=y;Tzvm zhzHk*XBGZ<414*^20jeoP6fycxbX_4ZS-C0#Q+>;R*@QA_E_mUo$Lovdi=e6WBOgM zO$r}XbX2^Ad<4XtiE?#6K{o?sk1)A-V?YF^rd4z8@D$1MWZh^By(-wVH{ANZNZ60f z`VxgC22Jem%k!#k8&%#{WvT_rZ6&fo>ti-xff|7Cr6BIfkKPk5o&VJAoeS+3ZoU3Q zL%3tr>%#lX%>{;tPj-YL-?vb2jzl<>z-(*JU z#NgY(Xne)TUG*ZAJQ~DTMCGtEk1WReb_%|XglxGE-9F|)dF+enZ>5s#WpS}MuE!-@ ziZ2T!lpxm^3#caGuE!u+G$4Kc$I<|Ba8vj-l~>D5_%~He?)uB4i9Xj9SE#HO$E#r> z%SJ-{)O`xKRWCpsauH)Y634V#LG!Q&%L|cQ$cB+6KQfQH;8??vi0OE&;IYY{7e2}( zPBTv-c$2rgimyl;^vpeKO)1 zC>_sX@V&--z}6m#@s^0ExO@gZZ00=}D9*iM!~N(*W$uoP@(KSg!J}Dzov788kl!IyaRHISj`d0HO8AS*(KzxG4!kYWX6Be=3xjN< zV%-thv=OdVJ8<&z&!_kFH8GbI&!(@bU42xP_wdQ*z53EX9#7aJ7_5DVSbVFZ`SET9PA)Q2Zam@YoV458Nf#{uQ=< z*0n=~x)Z7MRDC<29^87p{+*hVetwUQGQXeloWGij(}&7UV7_rhwUrEpP-{6 z89MJ56vT+HDYZ9OyOa!|aM)$#DV}GS5vvZUGUy$*#TXqk#4F<6jEK&6BG4hJ=6u%z z2MikfzN)%;`||E559&09Mq+2T(8yCPP?-RXH3>x65|@udly}iJ+A$ zo8$4>0ZgZ|dGG{Se=jM2*dmF_;^7h$#|vu~>g%)#8*9+)-wK|3kY=^6^>_YV6f_jnm&w=h6F^A2G_%6x=JIK*F2`2&_J#h>IR zsS<`$vYK4_hShk9N*a}W>ZapIGBmH8qE*(CFsWe|LaNsDH?o}gH-M!dV2QOA0@iG% zhVgrYi(|5UGoK^sH_#_Fkjdw*MC6$6ly3Swx{xk;(pUJSHG-^uOzDe)F;MLSMw7eA z*P|%G6b}ncolp%}eR9e5;4%Ltf^6h1;nkuIvg~FF?Kv4whK`gOgc)m|&>0SzLfjdd zP#(f97vZEs-ga$#{7>Y&gOCy^=D&M}0 z_){+OQ@U62Do>z?SdEtrFjI=+yOieg%ILB*){Pwi(lJoMJ#JV9gRCHTH%>6+*Kwyr z^<>8}9IKkcym=InL#D3PQG@pEzgA8scXeaJQF?~LiI;Zqn~-7UM^u2-^rZ}80P6Gg zh9Qa1gsAnP7qM#jO>9W#$=$Wo^oZ?k+}1*UGX*`n>K6e-AGxw_SSYkU@ddPzyg#FR zyZJUzXjpbNlMhYSNG?f5AzLJJMb(r+MP8;Jzp|CxZVxUZc!zX2 zaH$O%^6W=WDKb%(Ia@)*cwtZs`FaSx4W#0%FewwWUN?eh7U1RiA_or`9lf z!_HZGo3ni_pdx6=>xh9TB3Nchzk=j|hWwm)c=nB;)t5;^hg|UvU;fTJMEK4e;xXzJ z35z}~O=*12Yz~>8ROkntnYjr))^l)lRI&+qfqf&9ky$0?t(@dyxFi>RNBlG<98cJwCS3?L< zwfHWqfkm?qag5EV9UT^5{7uwDCW-5Hnl5T;1NCb^OaVnl+xEt4Y-+iorirEqn`C-O z?S*;-pZwBqG21j;ZeISj&feB;Rz}wT_oKGoXIvRO>J!c&WIt^vhA^V*$@1CV&>h$a6Jih&0ef@ghZ?jshYO&hn z1PN!tTQ_tvx6rPH^z?%(8=h)`lT+qvbQ!~9EkW!-+Y?E6RXvZZQ(B-&^&d{IQF{V)}sp8;a@Ff3w$ zr)od6lhObk9u;uUy?E6KC}FN3jkMC=>rCc&gYjVJh0fAw#~tt-pg%y=>5mmVq<*5s z9kF~$s}#R>LF`63PH8RJdiz%6Sa(f_*}cFVthI5nwnzTOzhJxNDJx>r<_Y|xbX(!6 zA&3!qiE6@Za6)*&IXWo!C6Xp;rzXf!qW2mrP5sa8QdW&-b(_`MbAv~|D(wNf`iPuu zEi-ztT6HUIH@o=nhl;4wzRfESL=T`vOu4A9#+n=FS3yLMHItj*$-zhsBR2ezjOK^{ zOHVyC<_NuoY|{_pprRz^EYSh)jW6qDslRoUBy*w-%@^%)PCHPMyC=p*`bT;Xta&%) z<_A0RPNkbGPt5nZYZAzJMn~yz{B=BdXlRcW?X5^#gDo=f?BPYmKC+BrZ&;wfO6-vSrP6UXzH3F#y-XVoW@84{!B^gdOcUL3TqNoPPR;XJ`$F_QW8jxE4=puGt2L z=SPF&tssz>hvkS;)dIB^Sv#?Qan6Z8wvhzHyCD@bdJnSE76@`;)mW#cFHRPbdQbx!K`kJr}j1`2ZH@+vcv z;73k-7__tN5+9qW1K%&MPBgOo4ZIf~=yFd->Xyjg(r*ZC^Pd2VX9SgxYQME;Cjtp* zlMB;&pd^{z55DV>B`o$z6#6-B2&^u%s3V+`DLtO&1(n|CXmyVgIgVe(j<%)R z_01L&JobJ=h^zCb{bkk8I->rLKDz>|%4}mM`EEn@XGlQvMIJoyJ#XopX0KY!@bfXs zQ+*kOyZ7*rNE@kCZ%+|F55WrV2|S<1KtEzEH7+iWOsbP*RN>F1-Nub!X@zwgFOrrzV52|(o%AJ8e2`QP_S6)&Ke*bXQy20CrJTA8^>8rcJFI{(WoQ%6Nd4da7T zii?zBw3A&@r?4qRN0~{IvhfQB1tu6JOp*QxX(m+|z-4Dd3e@5LMcaVD;w0DsX_9Ml zE`@nG%I{I4Y*U_WZ(-E5{$a(&&*!|UyJ=DW4;g!#DNO_nb8 zx|clK;W^h(U7k$&SKgK#qzl}EpJiVmwh}j^WF5_b9I-0BlxHRCm}dzpoo3Qb^4eZ8 zwhjN<;4kG4>Va3Z7a{VCEfL7{Ah*EgC2dwKqhvyJ++l71mKYV8>;luinuhg-KsWE)oR|7{or&9mR%(J&>yyjbg7mJj1}~D zm19gUVwyr5%{*N4qA+N<*-Dc_;alzW(+Jq|!)?=6TSr1&v2J~fyb=OgDZOzTOT_h#9L9xJ?gm>~7dz%=_p8`qzqgwWIB3>(C z(PFj%jv%zP=M57VLvk17+TJZG+ztS;&p7`j7?M&n1sRH>?d&mX=vLo2PZhmDO;5*M;4-=0lOB>pJ$Gp7$b&~* zWsN1k<{yo7M^z~}bOV{1R~xSMhrXnGegm5qB!jXsRW#O;Us-5A%kcfUKl@0%7~W0U z@J!$9*EEl-k*hmijx@VU7|N|$`I1Y~B&)h<1k;j6JgOq#ZKnMN-9q5ntT}7Ee4FAK zFi)1!RH1NeE)1qQ3iHbIQ*R1m(F2N%L(7?R?+4>M@~cD|M^Y!0?xYQgW6|IZI^^$L zt|?;H?HyFe;0~D#OY&J z(xvYT&XC+{5t*wx@8|fM8vH8Z2_Pcw6A^iTBTeKGe-ICoaJJl9Y=L%LW5Dcw9U<~A z2vb}{nijn)Yd#>*#>wXhYmWD86u_O#+Xcx2n~n$1#PSR|Rc(hDT=(}tvRHZJb`|Km zn%-+8@E+vzM{dgb!@c*or)P1@*Tapi{`kR-Oe}@ zxRKu#4Rept=nlmrZAHWteObcWt|KDlij{WWF_=!`n6jxc#_4XyLbun3K9qRVWszBi zS&3f0*CT1A$rse1q{g^d9j%yVwGM4L5 z;vQtP%ub!$%GKXr*&5hxbKcK&Utg!D3_uR9Xu@PtM+`Y538D}#oCJm@c)vcjdG$;P z<3(EWn*MpP6Sz84|5~dTW>o8B>CcKd1Q%5`abJQEy73ZmtbHQ?Je{b>4Mh4ar4H)3aYnb{VV7&MMNw%0C~<#U*|vScop8mbF-HllyNf z$EXs^3rI{}@`)x{ww8vA%$|GuEWl@6`l~i=X?@@!Vj@iI8`v|}aGdX!4r
K7|BUm`^7>V&Zk%^_d-%A~k@lFe zJ29@)d6R=}098x)iL_mZLWI0K!FqBf3ZpOzvy+Jct8hK3BkXB|;{d;X&YC^=&6Ir$ z7dO(0F~nn3Gr|Rt;+c_XW1`>ZY0JmUlh|dGco5o?f9f0Y-h5b}XYwKP?NvN;_U?Fa}eW-)d@m zG(?{8rVK0|*ho7_Opp&!{iFuJUdcgq((l3@m?b)KL^()Va<63&5uKdl;a(6D;1J`U z;42^^7JCB#5|pAZ^5rG-lbPu`C$c)l**QEUMp7;DOxo5PJjDmn=^+bWzE_JJ6Cn$8 zu(?@2m4>yoN2Kw4Tlx-N@a-PQ`@>cYdaLXnZ};Y9Yl|Y6K*=+viVLwZ=+Q}QT4m_h z-|1S6u2bLQ(SKvVIDwGu(ezr)jS5pX;6-V$ z69nqiOAC@Y@k%a3swx&M%ck9gofsP2yXq=0h`^4o8Llly(mCHXN z_$=78d#||+)1kiO`H(mp6tWZ;8C)v zw57vIxFga4uE_TD%gVGst)f!7dE(gSY)5}W8SyFns3>ErCf;*(=u)gdI|nDFSIjM8 zAG5*H68om6K~IYM8gN5e2)jA*1HBHtB{`m0nJGn$@o?;v6(RCW1^)euPhonpc?3RO z=>f*`@?Jr3)E_%ZSUV488l!;_1?;w$b&LA6?1_X;PSw==cO zl}tiKT(g>~wqIhS)<3OjJsKp=f6*1P7?jqQWqnbSvM3`Mq<~OZjhjfE0$AOj4v>wg zWhTv%d7UTdD5=2c;2QM3eCo081+|D%{OgNFV~$963&5P8R6e#XN-r}+ly?+?+x`aE z6?s|Lcd4@4Hg=+Ph1a3pi`t>xt919pGj)P+AT@}1E3Ax=7B#21RIh@Ttd}ZN;V~JzPXAQu>+Kf+;v2mA zTLP{ezh6Sol3k*+7AlRs{4^Us3r93A>TDH3nE@@1g#pk>q`TJv^DRcB8=7)+##Zfh zysozdV|-_B!q>^W$ncNJ@dT;DstI3!;+4c3ZHNHf6FjvTmI>*bTJPr7Bg#kKR?bsO zhzPj2DuwS|l)an;@wEB*7!y`w6n~k`a%uLX+p&4NqJHHyUUK$?&WVzJLd&vVqLkmS4BiD*$uoMxW|#zjBghEf zY->VN$QZ=^kVjRrBuRBO*WSJ83fY8tAsg0l4|WlN_+nr@QSG@h*@8frYlEN-HPD1+ z`FI;aELzQa!+P+#7Fls+gknx*QCm{g5+etHEy7SQ-sm`bL zwSRn%Ds>`0Jvt3wc^|bBgeU3=7VV5E<*_Ayi3`&gb4>};7jbO~>k2#SC-UZ-<|FbZ zCtJ(4BHSioFh5ygXChtqJE9%|&2LvypvyG_ojC$K5#Nm$GlRfFAz&!ziu#lJ9lvlI zYb^vLI>Ha82K^5rjx#8+u;f+3wO2^a&)NI6*69k5C21dTc} z|1>T$_9>GhO>y;W_Sku|#_@vr4IPuqrXQV64;y?B8=V-bN4yKm8K>tHh{Cn&8>^O= zc4$5sO!;ntp4|fv{Jk3R{JpN$NHuA`e*io@_d4j68wf-i^V=#Q6X~%&DSu77!sv8bj+L-tmN`f&~!4M zn zNlj=wAdNpZP58T$EAVUF#aA@U+-K6A*kA3l#>ix~@x#qtw%wrIM9b=fF}v_f++UJ^ zjV|eBP`wwrg2)xtCs3Ud6k)2d24r)UXXm=u-mE~L;ZkZ`o+?lr)}?$r>V@$3xInMV z6Pme_r%TnQ`C7TpH!CB4@4=&Kk1nJVMzt+&i}p1_&+n^jvM;X2j4!U1ek?N%QnXJ` z$_wzG%1U1rV#6nHzO@Ljo8UWhVm{-d5$Z2=>6+yx-n(rIE8z_bzSyRf{l+p9KP}WX zURd?s^C2jaA6osgRg~^2AY3p+guC8LBb-c>||BvcYtTmjhlS=k&c39kJgP}vh<5m z#DK|O@2;kt))IjF$7dpS%y~7#-#%g(I(VYl$YQEOo^rz%D)BopnuLe$N>WIu>DPRy?#93>CyCkM<1{ADA#8~Vq92si`*Ew}%}xc={9A`JgX2x0h- zWDiH+{)f@=zkm!nn$am~IY!!MIVNe@5vh5($&tM;Unb~A#^stI|ALbMf9ro`ngEq{ z|B-3(_dmg8Vr%t30!ZS9?~-|e*A5lne)KP%ZGZc5A>+SAkC?cMIM~?%(G*!Ldo$qm z!ySmP{3ouGr1}qkdH6`W=5V{J%|FQd1+J_7X~L2))0V>Js58HZ%y1X&3{wz93Ih5z z^O@MEe-m%TvTkU_DJD1G869qL`&_oU9Bix$1O$9QIfj#i!=4>2aiH|ZfD%q6Jqmkq z6M7Ls5{dyl2kv#X%)$?DN)WWyFC78%fYa-rMl};+W7Zz9QeS;nPqMZ9)LvmrN2V^m z=gnP(n(*|UxVBk&=rt@5Ng6HJUp#szFDjY3ZGJlxc2+W9Y8}6C`pmgJq7qF~uh6CB zTqhz&7-}0#bF)v=8*>?N!N}JfV_W+5fZJlmO$?BXq$HTBZw?QtmYT6)oadt-j(%id z*$OhU(eD}W-GpYr=sZeH!mXqYJ>?E;rm-?**7vLPGHCDm`loKlvErB~n=&k@`pnRZ zGk+A?mH125Zf%4$PP?#dDUg3n442XEu14ITac^fZFV)v$2N-u-OcI5Cl}hE3+#y23 zjrf|10+{Qd0-RHdhK`Mk&WEs_IVs3z2qWg9zU}b{iMYEgPJMrwG435_?$G6GeD+Ep zXc>j8rl$#u90d8 zR8uVCY+Xh&oxWhQN+~=4Ra~9?*E4*4EOvM{hBUclsIpVY(gw`+ zsVdH){1;k>tc}{9UkVB#`6`~@!xAed<6*ftsSk061kwiuil3x!c z>V_?U-HUE}4Km9D5xzs9`OCNeS-JmNivNx8{qIFtrLLoa4+Q(GF{6_x!M7ahWFY`Eia6a#=vSjmD34{Uan&@^(KaL~Sjp7T}ZlmY8!PGYq_P z=a7Gka6k=*Pwy(7JtMU zTx*@E3Ye}euE4*y7UCeL359bC(kdubZN^mDb&aH5dQBg21p0~Xi!Q55V{#}}TK;hD zt(PmZbVw7IqqzuvIPLpJt3%GF@I&aE`}u z=0|I<1WxVh$pm{ca;v%}S3rkL> zo0ZEdY@*Z4w3Fd!m*_J1?Xp?djlPILD%l1@lXC{wd5i9f4Ux>Rs2yM*vbRUBV;`2f zJ9|}oL>6~216K(b4pmC388BkJ#U}@i_0>!EZULU>z7NNo-tx7NuTXo|_E<=B`B_ok zS_nm-C-wTBNj%v4Ux9o%d#rgMyc(s-Zh8H^X48%zQh>Tycc76iE^b3A>UDIKM?Cg* zRTMQzH1|j0_xy0Qfc%K1pGt#WFmi*S*%76~rNSvjx#Avg%~6+va&!pA(Y!b6)GJe_-2G1@o=K0G zrw~{iXTF6@{p5x794aZ~pXj0r0?dUkb?4JIKCLS`6mm%3cCEV!Hz-lA&7SHFo@3Fj zE;vw43#o-|3q^le_=EKsCsao_0V}oZk7pv@E+>rB@6|Rf?WI6`sjh7ZNrA?Mjm zxf}P|`jJ}>P|4FhXBr!pFmmU62q5cx>ZA7))CK!Q@AX`qeZf+KT`BvDs`&(Y#!cv( zn(x+Q24F_qXsHHa+=U~7@nvs)wYACF{Wj7O{G2?EC-rL8jR*gRv{@a{8z|61_lIha z0AgVm32I?iGy)0AL*E-wIM*%WyZr1WYu{cxd8(DR4Vj~Y(TfGeS7~$_;gu+4 zTXFbJ7#LE}PhlDoUZ*SZ(`kY3!JK&L?#LIoB8;2X1{bQFK@UN#{_06K!dJc<$F3CS!f+xY8?03k& z2DA*$?9oY4X9rW(58Fw@*FC|@a>4L@D`-|8yOqi4N}k8C|MfcB{jX5Q5jom;QTlDIRR~(-v%F1?P)AptH3e=Z|MM?&fAxLX&FMI8E9sTCx`UPqWVFC?qiPdOT zY+Wq4hx;(7gfHkNFF=8~49F(*ephuub&mx=gvxN6L#XAzyJrlL7el#XSQQLo7|IGxw|yk_`!be_nV0k;E*cX( zHiQaRi}fR1ug+iRlh+t+IkkN2jSfc84fT-YS^eW>5r{TUv+j%hf0?PMAtVuSfltK( z_*8&W%D)ah|MXP;GQC7A$;tE!qWH}&49?Y*Q%{kx!-?0((Ml>|fWg6Tv>dnFN`0+g zPyFCS{s0L`Y?aG{_$iE?oaNPU3CsdJd_2YP;hQ9MCCo(2q)>scM$FrUFR|@?OQhZI z#;IQB+82WLAyn`(2CIQX<%t~&3BXG$YYS!z!k5ZR9pRu}n}ffwk!co3d@%8&-F-S~Fzqd@`dZac6XMtZNmTjU zl=x5oUxj}v^(=KA4|HG`rb0|($6Z0QoOQ;AD}=S1(-zbgqG_>alC+@{3$bD?4xW`w zm2C}=csym=8u+?D0PP4{IjYT=<9lWCBrV8hH^$QsRs;yzID_qcp$&DBWvg zB{NpqD0N`(E~5NQqKPmb!Vr-{SPX5U1k@wwh>Hc;CflylCsVr0>#I1FE=N@1FKbN@ zCH>*Az>X-_t7C`tIrSJSR}o>rs&8m6!iFyxI?5|m&#TYJJa1d2uC zUL9Q&YQbBR4pVgmMakovWd~u;<#i z4VhX{@xQ|4f6j;)zNBb9YQ=|X3N=_Pgf!4{pu|mf4K`sJ?T%SLhg9Igl9zoqgj)ES zLJlfGTJF~NP_p1Adwso^^v&~A#lP2H>z6~PDS5JbHBN_?f#IX6*w>qMAYrIUbtdAO zwn|qWzEYcW{^rVx`kFHlRMHILO;H1*aaHdu(fdFp2-yHPlBrymL$NxJqDArL!Si^+H z)VFdA-FI|mK9~BQb>OEhDKzA3twArhZ!t+Q#!v6EhipA{M<@$Sf>Qgr4S9Rt7$-=B zEt&1tq@bGXXrP$!XnjgrmGC;P$VPk8{Wo*B`08@%S2uNDUXSZHt7Mv|YRT}E3;1E) z#iWf#R;r*1RW3Kas&(Tz$LZ%e5B;PB%W@vbxPo-*q6^ilN|YPJ*#pboi;UuJukPBfA zD2pP(`WqcN0jfbJ4Qp>yAvYcG?4PWY-q?#s#&Nf#ll~I;eQ#aK{$RB47*dh~cKE3+F-?Q%V{b>dz(36dJ*lD1p;Wv;FZ zqRF#EE-xXNE^RL&>`@Hr#eJ&`c6p%X(Y%|KGOsyBrop`i=D)#P8BwBT-+AhG@r_H1ajPoqlC0pc1&p%uBN0#b) z^pDjnws|zUV=#q+j1SXqB~k|sfkCH`4~NKU(6=^`(}1`>nK=ZYEpP+%2b$pJrIFF;P~hEhPn5D!-QzJ#Rd4{)Y8QP&0= z_BelO1Byn@ zKoi;jH1Y|J68c;4p4g{llQz8jetWo$$dn=mgjg^7Z}(CLD=?{hM@HW7VQ4D4?T-An z0>tJUr|+I%!zf`eBBCKjw)V|ic2%jh!*Z+AdKWem)K-M6ZseB2bWUl-`fsqV0V0!cR%56K-%{izCQQ zuqaDQxRtYutBRZP zKfe8U!sdYbsXV$8%Ex4LZ7qW$%9jmPx}yP4 zkWFxO#4kUtbAH6`h~ONaVbNo?hsHe}j%TKEZ>FVXrSSoAl6NSQKr`5?xD2ZwGM2&g z@wUTZMr-ISWIOzeQBo)@j5~qhu(15H(s5UkzfDkS0ph1k>TmWhu%EB@JQ` z>TSi$t~Y}*bY&GnSdqxQL;8WndSE*15m_pH z$9^fcKRcmL6nwP$B2c}}<6#?by?7rKsryCsqwLJ ze=T;$RN*6lBjB0F+8uT0C1Rq}BB<$lc;$=FJ<0JfQHm30EqA&sg-NSW3wP<|Gz8PM>Jxd$)RlO5u27E$yScHz zA14qe4&n4-=2eN?4bVb0dk>IJYYJ(yfHTGAdXGJ6XlT<&OAB1rI(lK-Wq0Z`UDrK% zxRz-dd&dhTCoo7t2^f!USjWVV`baIf=p2mm)aA`o{AVLh6;MW^z(^btE^`;7Z`PAy zC`}D`4J=Sjp+^{Ixk>uE>lAHLcgY&U#7Yq9N1|W_TMAVW35AcSelQ=BGKQmchJltV zbnkze^F3crR|@|&<3sk|?^scj8e`dkqOQ9k@aEW4^;R zmw>}epDDY5kCz8pc(ld;$YKU^?M+ zems4sBF0ReVAXfD6QHKYeWztCxn37~zG;S&6XlWfg^faE?MtuAOl`ByW^;#y?<(n- z;YgKZ$vB_RNgm7b3`OWN2194mWa#V|)BYzGfV1x%a0D;A8QPMy8 z=WFK!*GScUQSEHoKJ8Nj1~F}_pH$=yY7mmY&0`TW;Ykg+K`~bn?WXRI4CG=ac5**| zVT~fRfDLZGxbVh2&129pX`Qf8$4V1}(t2)>7h___ghz<1yFJm zb)t(DTQg7PRzhZ#%`tt&Jy6&nbPeA1NHWSl7yXr`K{^?`EmETYiHwMDHxMA#!oaw0 zs9(jubjzoIFj+mnPp&8)*p+HE{6L(@C#H;yv20;_On#1P1s9E*MJPBO%_MpDvphFv z<6ZL4=;4u3#-AlDXH$IpcJf#iK@utYfO#hk|{z)s`~j2Yqm|6XqY z(TRl3%pIJ8i6j5E71^nvYhd`>*E>2jSV|%$HCq-6kuZgTe34RwpKC$;VVB5RYWLMh zPUEMZMMD`dUO40f{@W~)_F(fS&n(kB@jGf(_Ah)9=0L<4ws&WPNxuv3DZhuchQ}IU zQ$iHP1Cok<&#+jtvi52243EUs(vwHZfa(rn#wh$Y4K-2g;ZGvn{W8=mNQ!h!c2Nw6-y=xAlkgMQp;n`IhsDNLrcjfqr526Ym5fA z9bsGTJkQE%(Y3+|J7Ygt0cyY4$Z|nj&W@cuh`}o%>cLf%8d3Ejm+$v6KYV|!6^7k> zJ-mYLIy+aFA&%3KJ-v40$l`+QNBm1?dU=^Rhgu`Udg(zs1KY;jFJE-%ZfmtrSG|v; z)ik7RQD^82Fgf_w;xd2m7Q$FpNj1v>F8T~z*_eW15WvtSMN)@WNtWv^Uk19IHv28Y zwEqLkuvmkY8jYMNQjEKidFUFPype1#&BkGCe;jW@l<}<|WX4m%E*&JLEsJOeg{mX+ zBQ9%p`~_Yt;%(V9Ij#a>W8oG(6-0#t&JHxRW?lJ2yZMqvj#}eFiNLBeu2qp(y?ASQ zhD&_e$lx5kh$E8#{JwJxU_^bmrcvvWSK&Q468nme&{NTi<9G!xi z%&NjsZs>D?fn&SI#<92MPAduEzAHkpJ4ITZ4zp@HoN;1$U;Aj6f2y@Ey;)yoT{$Ow zr)^3ww6c5|;gH9wJ?+NZp~NayNSrzKEUXs``WSbq8KI&yo3r#;!H`HZ7&nKn*4vju)9<*BOh7mmu#(tK#|C4A_ zN%tZ&`!69EfqQBC4|v}?Ph;qh9LtOTusI@Z8(UCtTU1bYBI0{-Qrl$C&boZzDVK5FX4ouZ+T!b>!Sso#I`O9deKCT+uHEPPCCB$vqh7b}m1?EaDwv?70Hw5fgiox3mc zO0iogzg@f#cUUq982UoXK6P)lLGKM@ZUX)lw(M?(E$0I^&IRCpMg0GAhKLxsm`T~Y znAy8nxdP*hRDjwudkf%H>u3bz9sXywbdk!c{j4Ag->L2zR2ZNUQBhS}I=4;ftDg{! z5`?I51O}*bd6z>%^zvvO-D=qr<_9TL2gVQR-)sRPt&=P2C~_o{G^3MePvdFayVoU` zmjWQAyENd00|@GK@qK)5Ym0R?eUyZlgldEw09O?rR!bHN>3wv7=_(-{psCvR_w7h4 zQ-{e$3vI$>JGgz0qe8h4fh<%_;Z*JHLDvyim!mK4u*)<&@3E$xhwmUCQ7cjKv=hO0 zlikH@5L&jo-V`fCEV7*ulC2e*`*>Df`AdRN*HwfJ4L-sPNrw{tYtaR*z+v$O;aF5$ z^s{7}2=|2+iC#(d-8iUuY^>z6VvIOKrOS_Zu}@Wmph4flwdw2cprrm~?cO4YIzE2G zif`EL{niTFNXS&u4z~)3a$r^&-GI5w#U-+G*{Li~@N3y}4b4(8$7%_VXn1pG)0mNSMNtbXqfydnD`XI+KT7laJ>1yP296NHJ{ zUs2h`d9xB?T6bxbd1c(w6S)~u$($f%qu(qYMyBJ6*s6lg*s2p8L_sP^k(=n)`?$PB zk0_RXo7@9MZC(+TS5@|@OW2A#glm~38)}AY9hjG5F1?!Ny-?wmIF8 zyuf~uejq&v`(Q8jWpm&;rIp)mV`=TF`~O7>=b+2oy$J;ZQi}?t`2SxDRK^~d?*8}5 z?(c0+#ns5w?C&$)y5{lUfXB~H&hrr09yA(F#i*GX&UN@87|`JpgIftcfdI>sMCs$C>8fy!80c8 zkg}s^mFea|M$8lU7iC9ZevP!JT;C~J{j`k@V8bdSohapsN{KV7;7`5WqFMt-o@TN& z>|6`Jc?ZA!m%0#bVmZtEDshF_{Gk;Nz4g-6Wb5SU6az}dBW;w{1G4;T1Sf2

Qox z0`xkkAPQweAlfOtBr;PCpCyY@I(B}_q2#9zd3W%J|3eWKpVLA(TO z5%Zf>!cM)^YQ?&n@bvEeMq7qf)_Rqe86vho+bO6^&4TNMJrCK9V`zKRuXfd8M5%~s`9IYm95q_DwQl# zw{#U3?nojDov=wtw2sQ^BnoussoxlxR&D21ZG+h=hHHPRxddwfoNLfm=2*#>S;;QV z!b3X2P@Y~tG@ zEsv?a$avqb z!A;+xKmVyOCP2?u_M?6ro!|6p3hE1XWYaW#CmFc3%s^$13Jd-mV|FHKD;5_gD8=oL zv9{Lt);bu_WV&2XT749?b+HvE@zDP45=p1BaTTD|Ujs_}Pptcu-!Z)p9f!fEsGcW0 zNI*A-;X6d73JsXdwnqOVLo}*B?BqJxV>?b(wQd&e?en)d{)G}U1e&OCD|aImZ`3H6ub*NDlQpCW z7Fvb22s61l4U30fGmyZE_9%KpbX?j2jtpKREvCcg;qd6)+bMk%rMajuBY7%4@T_MqDUPcc-On;3{h}TDaHHiD8llM)Y zenv30d7+wIdgsx!>bknt{ArjL-`i3>%>zm7b1aEWPdW0}Dn`+tNiz|#nDU#_Mw2GC zF??~VSmm`iB5JmNJnfW{;S|zFTxex&mW5Oa^r*W%uJM>*pmo=TO24r~ap-AG@Z^z& z@ag%!NpczPaLM}v-G7twO{k8Y@*^M&%;gdP$@biw`0`qQ$SNmi*8mkopTL?V(*&}c zBLjqsFZ6T@g5&L+aa)+Qr61|;9SRLU@j)Cb*v4VnqP&h-Cqz$)nB3x)s@C4u!g%pM zEyb*^R3|r3{4MKBUPH?(D8W81Y2Wi>?d83MZ{MQ=!DaVyWJQG-->ZYzQh6mm-2RAr zwJeG0GKJdfJyLuoeXc_f?Ancb`$9pUO=9Ebr%&VtFna#h@=(gm!2vLt`(x|`>{9<} z;LQAwbHwG{$}BQEX-KrBUk$h+Oe|hb=vXisNt!NgrwZ!qNZKii4fNz~AIrU&Cthe& z52`m1Pr}7=!w75=OcL=4TjSp2n8D(|{FJg?rBNVX+2cqF#nR*srLf3GN^A4tb~jU^ zw^00dk6n`pHdS@eyf=nvnjNK@PwmDHX|tg8hQda*<{Z&cN~6kAkK*PmYn!Yzdc&qo zZRN_;yI>xRqWF|ahf0Yk&#(p9mfqqvcEXjhG7XuCqJKPLZjihSvsrMYmv?GtZtpBC zygaAfZLcR?ncPb{QqRN2JsWmcosmDIY;l(-I{^F9WE4l-zK$g{sJwQ;rCrzj0d1cdA`jz{$1?pXrG=acA{?JbGvy(oh&ivO9cX;@g)xX}$b5Kq948PdDBiJbiYt zR0vER&T`jt{Dj;JtKbTgsy#L^0Zs{7FHT^NL1-580djJX)=Wk;e1aj-1UzILng@P` zgo%F__Zz9(sqT9~vJ}FxsRdQtC%d@`Y#?J>qrJisrL;3PxBXf$=g6%%F_Kn$wT!uy>CK@uaU z0F>zhy{(7o7W{}c*oBRdoE}3X9G68iyzT}{29wew58xymHl3&f zuKG?e$hb&uX*2Ki=|a54*X&bX`B`dyny*-oDJu~g-4!B*9?~JIa+lH+$w8>&CeB|M zHvac;C8+@GF9lftZ_OM3ZT2pD_C|l3H&!SuSWnBsak1EK_1KA#TB#1nPbCna#xZ|L zpr$O$`yj6vKXAO9!cL#;+Jqw2C99vUJ7z+5)ek$x)ON(BhmLXEvqt zE!l_#8jiyN2{>H4nZuoy$hkMW7~ZA(&|1LI{Yc%}K>^G0u+8Mhn>+&O@;9PmZ+CBO zd<`V`uQ_1;u#fK2XLP6rV;~bO>TAn7O zQMZ>EM(ELT)0mClcC7IkY##L4t!cV?uT^+Uv(ezz;AQS!p56^|2ln2^-NffhZ58{8k5t*V zK`^yH?32h(0seh<&w7XO%$Z1y)w53NfD`s^S{ugGPuHN8_N`V=MyaLW6}=7_9keUc zvywH`bHX{CBFadUFYkPsYx=p;Pq^#j9gMo|hCtf!oZMZ6X~|VEMT>W)6bPXLuT2Ap zJ%ZZk@$w9(`$o7^Iy-RnM@|Xu={|tY$Y&YlR*My=zA-==mW?tW$O31Vktg8KK&8c| zt&F3QqchlLNVw7JK-*T|@o?4G%0i>wMA$*6Ho#wB=#~XnqUXjFR}?T@Q0ZC4cK~uy zai|eukdf#KcZjRHEmS(8y5K?=Gy&|vDh_o+kTdxq`%T@zMMso0AuN*p|hGHue ztCRZL7%~=DgK+i8FgEJPi?!01K5?H;fX!C`Y@X$J)=Gca{L9sQqSC)S;ohgSlXA>x zl|!Cx$o0kf70i=VQyK_; z&K^)rtR@yP*;m_RzF|SzbaP7PBWHUc?&b|#+I6n2Hfgbm;0k9HKrS{`Z4Dakb4dY*Nn57C#) z=ECn}*Y1u~%pvL}>{5-!9ou<#23Q+=AWl%|Fh%D`@94AW$~9{*_^6gdOv_vO&i4#0 zi>d7wf0OY^@!GR6z5U_yf%%@H zb_*}SllSF=(a5w$dA9WgP&+VDPtU-lb%--Yg=2F}3b)WP0VEyFbgc;K0!u_p1{4rl zuT+SIC>2yD51g9c>`p3T&p2+oQL(5e|2W(B$-NV`5TnJLPXMj)X95zlFc(T zV;*6TyX^>C`K+kBi4bGJ>i#^BW(A^ z2R?pZE|5he!8_?UlcB|w%_0M@^j3}-P=KiErPlGVW3{%4&fPv#IAO4uW)`Fs%HdX0 z4uXay5=!}E#1_g(zlx6i4*S=UAd|qct{89ztmyBuO26J4`s1zm+aQoAuk}+_iK|wv z)>%rbE^X5#f=rmq8cBx`-;@{04=R@PmRT(5WWZS2n1skDm#0`Jkoy++K0nNb`4v30 znKSlSX6s(oFqg~Iu@@rhE)gMy+y%s!B#=XC5lrSbcUrKR$z_rHy{EXWQk4a zmmK_S-=qaodySWOuo0Yn0BnhzJa^IL{EV%fVr%SpfN3d4*xzu`(i-(9^dQMw_P_=J3AAf)c! zAse)jx9GXO<_2en3`Uh-2z8`DF&5mVd9kgOIN~Y#PHsnmFyg$b8z^Yy(D02 zoKEp6SSnKeg4dW0^j?V;Nn5Msgfom9_Ra|-8Eq(DM2}Po zznRFri~2Y@(7*&=g{uWLz>v=P+NbkQ%-4S*!O-i6?^~ojVUXKfh^9Jb%7Ug488T`; zw%)u^R7wXUN^k!Ch~9-yz2O91qMVV+)k#Se#gDM&Z-nT)& z`UYdx9f?)jAU1d0MkwkmwszZ9x^9G4YoBv2mCTx!u*`eK7){fT)5EE;*$DjXHpwDf z+B>rK9jC1zCQ1Bc10wytMU7r7OkgF~_?uGdw*u+T705iMs*&&Kw3bSnqm-`FrA}vr z!W%guPH=rNWM0$5a=0G^P$m1Q?MNLmXp%Z3rbRtARBplpqpfO+n%Hn7vqA5C%b-Qp z+eQD1+DQj-rcg*QeYitDz0(!Y!KC7r^cItL6*ZnfuNh6R}}T(~1u5O?VNB zazm$B2ZzJRrqkk@@!TD{k*wqsa-1eO`MW5waLvX58*vi*Apt}OUQ@w(Q1@!D(UW>e zcO0zH`fRacvP`=RNHEB@r>%OdxQEbG=|2&qN@3-lQ4o9cuW<6K2YgR3sl()d2)fvc z^ksPGL6UJVNL3_`?cQoV;vZTJcT;DI>_PSo?%u7+8!E%x9~O@p)qhSD8#35D$v7(K zI6H7FIw1XofP_Jo4t<=rHzC9K+?pUdAhr){`9xQE^SUL8+nAY5f+8iU;k}(35!A}5 zm!^M^MqQWaj~5xVnv+C0ya7h81TgadkGbxzefOD);{eG3q$gwNrNF|#Fj-_Od}ULz z8YDP=@sNU0v3OxgT0-}CLj^Eu&V#2(x0Rm<)4@G1UWXF*)%qk{j5g%S*Y$OeJ? zrF-59F#A3AL1aYzc$qfI_b6}LRCM2~8=I9THdQ0E{)ZU}7FdO>e;(H)(3iSoVHkG|S#aj2Tq z13192TLHUM^uIHq{rjM;u=Z28^GTWv3EBa)vBW`cSytEb%bhW8nkXY3-V(wH_O-Kb zkP}(sZUe(T&)sG?G50O_tqA(K)qYg?c>VH6H#`}x6q z^DW3M^$!}RaP~A_2mO^0sqR|=y3Sp>BC03%Qygt*H(XbIm%!HvtsA@`B>Z=aS*)YC zBhe6n2D$h$SNia^wYS>hGET4Ig|KlNT5>U(35bGx_ujl-I|9FIiUn z%A!qX4=Gi_*^Yx@ek2!es9RP$&WoWkyKoO_s3fM*-ZWPXC|6kr#%W@9iJ6;+K=B8_ zgLBgb&2+wc=YH{yfsSfL79Qm*NZAv+`Eg?!%5~Vh$RK}sRimWG^2(=ISXblie3Gsm zkK2$-;pwf)lq+C2v?v$rk~-@{_#m}iJ}PhSt9AF`&k?MvcWSmHaa$jN`&g7=<{wAR zNZ3fLv?YO6KfWer;3IoQUMtDBm|b|oLr4eVAU1OGL+}d=m5|f}Yjo!b6}I*bgVH1ubk21&MUkV)QN7<&uymkUFE>r< zRJC!XLc#MB*=_8uo-W;Fba(JOkRc)8K>If?}tg%gm)QkX(fIQa|paNyJ8fcJnWvT2Uz|@W^8=TE8K%hO4V={C$dIW zk<_T%6h2)427`Bs0W+9r@(4Pvw#;mAk!7(6hSdultQxeDKf*0j9hHq63p&l*E(FHq zl~K*c=h162i{3RX9UFFpLROYIRdmX|o1R3iy^YjVKc=N{?5{iTVIC(6EOWfq@NLSw zX(u)6dvXRcHYKWnVf9zj!?PJ-8WU%! zdEZM6*bp}($=xSOM%u!x2^BAKOZfSc!}MT;t8+GqQSzI5X>Z1-J85T-mVmxY<0e^& z7~XF%qlW1*u9!0frNO=uAfZ7yv-Y6Y*;5X@{vO#^|7xb1f=&>p>&?AtPz(}mu9AG+ zz|9w;ukfOIUX0b>>nJ9vB|CHsz+>vFxdQ5rvAY&;vA40ZJ@E0nI_}!cuNc>j zSfe|EQlVpN8lnf%3D(b?beq9Cc!v}_9kvVOKl6CnmZr&i#72Zag{PpMy*G}v??HyN zO8&AaWQrqa{}nGEUv*xlXQ8qs4naxzP?UxmT=QK4?m>78a}pL0&=Q;c3^)#t!f1&S za(5yxVC4v$X(0N*9uQ{#cWj(`#rCG-Fy;-80sV-kOj z2GWhcO2{(!nHJH6m|ycyyR3e(1*Lpu%Di-DmI<$Ds$;f-TjN3dA?wU(@|vonx3EIX zvO;F{Y?*^0Rg9YWI(pgRlx^)M)8_linWXm9eri4t%5Z%1yno}DEvqY6k$yKOSQ2ZhtlABUwteQ;g#Dy+(+fYbu;gkjV3cE;=xrY2}c4kOd}3t7r&sENjgXy znUD)|0haHPGcN6??4{G-@)Q3IDSjGyXcsp%y_+6S;$Vc0b1NIKkL6@vL;TH&G9EN7 z!BoD~ATT2@UmJydh+b;QsXQ08fM3Lau_Rtxs?@Q(n71U!?Nv#xN`dkTB@}L{v|2f~ zgd>}hv_frR+Ls-@{0!_EqclpDX?LgXu=nMP?v+pj=2soU@eGc2WSy|LF$`+MaHO@1 zhDpSL?PBePnGXhy870Ohpxc%^nZ#OSu?|iPxTCMka)~2?Ex#DWTfP}^Gp|*Or+N($ zQ6$-*5s=d@(4Fi4GY2wjvX^gYIPH`g;WZpM7$N}#q!p%7H-OJ%`!2m`J3J?&cy|* z5T_-Ly24xvz21zOCgLSfhT}vAfoj*h`pQiA69$4zq^jA&u)cD-qqJjDjvT#D=(ROt zD`W%1>hrz84DCcI9d^@6MUhmk8W?HsTx`teYYH#gQ21=SvA-eIHqgLB&GnUAAMu_5 zhMo$13J`_-s2Yn01^OamS(fznfc$a!R1(H;*&bty{za2&E=b0lC_ z%Vjwk`jnU}N?NVHPDWvp(0-JcnKYG6Qh#}3(WtM1l$&EKP}dD(!(@PWm8E$}?9QLS z`NQCgQ-+k0SGzeeYrAE?tH*G^c+~!3-FUc{y4k0MjiyZnpTtjL z381SjY6g#q`z-qOVTxHSg;*tz&@|R@ zbd<#4L`k4$XfR3evmym5l>K0ejVsGDFsJt0>nQEKmyeC%{8MAi_D_t0IFy7QY4g-n z*$FU?>hw$S?UfVN+v&=N-w2r(;tEv2<~B`zshv9{vDDNLdT{+P9!98t*glCKUPD*c zqphqt*%2Vls{*U$`>20h>&v0hlUialwQWKswd1Mh?w@ax?Z#WBTMn)@-DnuW*N>;M zVH~ss-kIoe(1U}Z!hM!y8iL+XL+S6M#faI!ejL(TSO=|o7xF|tkSf|x?e#X0bh(yg z>p(Vw%Re_n;~=SfZFO#@P@mpona|<`%Ski&e!|2jR0Q;6xol8{U8AU#^wb9#&B+7# zFQZX!D6nbNT1;be>MZr)NcW1__de&zjTwb~`!Z-7WkDm4pF{!gn`r3Jap-PQM>E@r zEtY#WVi#wgfC=2Vi2}^BNerB=P)oDU%s;gcZ<2n2jh#PeEkKPh&SCM{xw7IxXc4{r<4&%*uV_Gv8Q+3Qhh%eVQI1h(0MS(iKGBXp@ z6JVyswUL`@^?^OSq*zJitjTufqqxBRw!Q#$?Drtd7;gdU#Nm*4Mi!epVqr>5$U&Oa zDx`Tb==O!0LY8$mGYyNqdv?$sY1`^oAJd?WeZb5M-Rt{QDKQwf%?mHfFM8pjTuNKu z7o8$CEe4$I+wroMqnh}r8MYbh^YK^)m4ZA`8qw`*J*DF{V49W0-o5*5CuTLUw*!4# zr>QGXH0V%>g7BeW@*(i+snwxfE1t_hCK*TkJoJ(gf>UXGAraOGZ{L=Z)JR8}tY#%UPMNjFrCF~oCZ!m7FJr`mg`l^aM7h@ij z`rIV83S-NA9C9XNDn-Ar-F~HH!LY(76AzC39mvBsLOCR7 z)+%U0;re8Yg>L1nrq@oAMq3p_M-?*+HGLz+$oU%8<*UZKYIchR6de_7?}31DT)og`sIzEIud*k%-vx2vN1K0@Qi6W~ z;UFffX2pQKL3I%%fMh_*&1>f}4%qGC$Lhu6icketpd5QtG+F3A4P?SeuaZ7zx=X@~ zCKHk-Uuxd{n%SPr6hL+phIOEJb*hED6U0d^Gf{%Li{Nq2Kunl+&fV_G58vOaEOL3k50-xR_JxGz3#Y-H5vu<;srb1&&Y@gH4W^p5(6H zYqP+udfjjY@l`EIZ?#>cWi#mhN(45K5!Y}hT)iK^XQYGtXo??=q#HAZ5cqwZ{YJyvsQjT;hwxjKG~P+9F4rG?~i9wQJmdgjF*-( zOV#UgMn!x|viNZH7UgcRJ0boAhZ;p{Q=4=5sWK2hbM}=J-}O`hG4d9%%e3P=!DD-b zawq6f5-tv!JEhR=BN=H*?t z_If)wCJljVi(fKcWW$QUpZy|b)mI5IbrJgh@AU!gcp?`)tZ4}QT4zrM1D zE^&Zn$mLu4uCz*((eyPQogGX~UWdVBe7qZ@Ya`khCn;Roe~M+_OpWRE5g|4^@_m%R zoW@0zD(O|NN@dG1jl;ztVf*%)#nsa3AkK;U9}=gw4u*gIDpO$LEZ>?(An6fYs<8;*w~0zLKZkzj`%#s4Dw@oz-@WA&41ie9!O%NmtJ!8VqLle z{mt9ct`*G6U7`ovlEgM8Ob6CoWkqaX=8(?@W_;f1C6g$$(|F=gvb6$D!4Eo{%flDi zPZzsm`D9-lP)A4d(as?3mxOZ~l{f=4^tK^`bYb+wzd?LmA}=+BP|zR`miv6<$Fh&r z$Joi|CNv5Ky4HK?uH!Vp5`qrCGnrFaWeUgeHcuC%b`k05IO$b$@^B|#hAkXP4E;XA zMW{b($tup}Tm3hX)Fhpn={dyv6sk-iZcg68H6cj7Vam|vd>w8yHEuG*(`trkHVm1T z)9zkk@?o&|k7g}yGP<33NU<#eUxH&;{N#hS63$`*1+Tn~oF{l90@*HaB#DNzIVWe| z@JJ1PoU;_C5_5C9f*2zG&{m}nml)P$52s|#S;7qm1Cw`;3+3;d(5wi`QnHhVqN8Ok z_t9SMM2|9G$y31@dG2Td|EfTgi>jt*r$rN;^?Dg-Ru*+ok)@gE{Z#0sykHAfjSv+u z4pk|3&n9`I3^qr07B6ykI$e5T6;OrgXOs;8Z+FX3h)Y$ds5v-RO$bYBZ#Yt1I4*#k zH^?+YK6P6^qM>e}7I*@mxZ+^321%#BmN3qh*v-)hnXoyI&rBxJASagLZ9XcZpD)C$~!S=cnRMT(r0mO1)9 zVyyKv?tkl-542I>%2KL$v(MRi7k^m^OeN8rN3LCV&J8QmOA5E|e6hw)WIf7@NL3PG zJEIg3foR7ew7h}8Y0fD{vxMIxG0ODuM6ro3fM_(4YDVO!EsI?zwsOEDg-C5%L;kE% zd}g+U4Xw|NZQeOE`tHGfhBgUGy%dYKv;2@S=?hsv2}aKWaQ|vK+UVfjCG&nVkQaUO zZGDIVmO)i2-D+Qol?hB@2M2m(^9V2rIXi<}$n759e9{KQL0d|YeBT}|)v{!m9%pyG zQi?(Uh=GKt-kx;C{5-nuuFt#iDTWeJHVP3d67OK~CF~2!0?xdWM_Z8LMe^XPjB_;^ zRjo;3Bu%yeC8`-SPpm%k7JU$l{T7D9_L&Bj!%#gjpSC<>vEW-QI#}@$^|0#L801gX zM21{}j5Re(BI4GxEM!JyX+(JHD!B4T?Kt23U$I1>_oX5+zjw=D6548v=0bx(%5nlR z`G!Su*&opq)w)5Qx>|rd^P9p0B!#I!d)O0^bsXy4MT-h^B&an zT&hJ+4N@_Uy1qvoTuBrSrAubJG<|(Fy+hzB|R5B8)Q{XHddbNgL0yaQ%e3oTLY#+!pzjN}(n7xHrUFzGr0dTGZJVThU%RY3H|s z;hhqPbHCB*&=#2U@o0BexSg$qAXx9Tk^13HJ$?fgy+@(P_ZI17liCVmndH!8+I?#b zI}ST+ZGJd45Pn~gyai!7Rq=1umAa~vlei?>l~POc*dp`u_jn4f9!3009cE>kb_ZC~ zk}edI3O{;BN?r4O+7#uo9fMdz8#x(Wok^tW(s3ON3e!6tu#}Wdvy?paa(IK+80Nd$ zTp{jt>|By+a`m}-4s8Kiq_a>sk*XfzTrrbmcZ;d3XB+~Xhh*Z>kM|q<*!rF&RlR9X z<%wx|5wntIqjvYFi3Z#~v5CFnuR4R!9@h@{%ALLH;&((;6J&c%_>N%vOP4mbjyX>% zAPcXuHr`vl<;pMTR$tI`a;z^N)7Z{*Kzk?)Ym+$iVy?N5YZtWzX5GSkBD@^_m%|l?>l8;#$nbby= z70Hd{fj~Bjk>1*e^F+WldSI)>1)sXdZdfiyZ5CwPf~g;|lO4`59z(I+vlFjPW`F3Y za^V!@dV#rHn%>B*DlymX*?I@Uo?zeK$-i4{-_F$Et4|)a7Q2$+pK>@8`Y|q96rD>#oIDVK*+lpFDe%FLJ{&`C*WK`Dwpi&zd~f zGP*()xIf$tKFlt{L9>&tvpRZy`brL)(|KE&8Zr2QQR<3Rds1t;FT=Jy+!Z zGB)k4(aw6zN`miKm^@M~k+%feU-zDP{<>kR;cA_d0Pu_U13Wyx@b3J}!EX4cAm@MY zk*X~Cyi-Ab5?&gZ60BD0k6IyCnr2NhVhbXia4iYnB9_8jBC`{-Rfj^fz?X?JNthf6 z)ex7+od_%}1WilwVhHywV1y**Nn*LZ7<*^acCG@~!NGtbG228(1K2rbyW!aLG-;mV zd3xyQ0luYOmB~R2f?@E5i$K|yOR^*L{m@#~laJpmozuHgLR=j%ET-96NT@5rt#miKK(YEQbW#J;BlX9pFw&ERcRz=`p~tW>_eq1$YOWBx#9 zN%&zLyK4Q_)OwvdcI*Uw87l0|NAOjTLq}`!a$2-^3JBrAerFf{UA} zSV~|uhRq0VI^@^CF%hqX*l&N=z}y)Xc^G6JEw(>0OO{c^B*CRKC44_78X}njD&;zU zZd@9^$8-dFA;s@Ll%XPFq9}oCVN=_?lR+H7A1?)o=T#YS&3=Yy&|r3vF7JHn2%H$R zLcS8weK~f#;7TmYp-;1)2(&`%c{pSod3}u=MCiykRi*h+&W@GW=koM4v@Qa~$UwqG zsBg1DjpFv&Qb!gcK|%?jofFwrPy(IjACKcmuY3_>r1Amcw9L8LTw>px-L{}K87fV* zqFg2FKsiu-iY;~_=lnH=qvLRk?^6TiheUO*lL2On%gOXv(3!I4Y3t%xT%mg5aUdGdG4GpU1!wY>+`;RSnI86o zn&Uny$$U3ln5%0R16umR-^s_BpH#X?d|9iRFL8QZ zY!)PEdakEjt$w%OpvCk&ium?>ml|dx9vGao6TEN)&O9H zQ?(!L)@p|}xT>8Z=W^&O$Zh^EMxH92H|JiUJfGhZ8J_O4Ff=eJBRxX!BwZjf_XwXF zJt}sNpF2Q;x7)F19F%M`M54yF&bexYwu60E*rTb5K^|F8@I!v|QyC{#@OKg7&R7QGaU(D2C`GEb(UO4cZ*AXwIW7Z(dm!` z%bC5Z{ryOc26$!#7F~wW7OhJtp`c&p(Rfw^n84|Hgca;NSPNyMNY?2G+XnPDHnS%aNeG)n3MPjio~E`y@mAG3_QpCxUm0pk2@F4T>kS0 zkMNE=0&l4MJ>z>?!_&67R!}mRce%|P5No^v`U(2SVB1b~($#_bn_zL@Eo2EL_vWgk zx|A{sV&cwKEVG*c3U^oZZIq!^p zQa){S$s7xy>)Lxo>gmj|jrCZu##a*0GwMRW?Lim%KpU=ARsD`;)4MGIMw?hHpVKbm z)q?1TUIboH@npEjIo(r#3ehHO6r4@mEuw3JGj>;hW+fiqtEf%1bep)SZFsI}9v z0+~%Q@eEAQIDSt*?pOyI{Aydc2;H6`-Y9X!Xn%!D^ype2xdR~GH?f?)yNIn24Thn(7Y?{F`=H!D(JNs=g!Wd zR0k-q7s!&r9NtR(8?)@eY6k4KFjGS(z(eRR^M+y<&HGSnEngi>cwAW7eyN%=249!{ z2GT#zh{1d1`tI~{L!tcUz6F^h``YX~43W08_9Jp$Kwy-q0DCKo;GTN%H=ph?fLBxE zESz;_nFi__#r;Q|PUT`9qMol*kz5Ba2`VND1GR3Z05uO;L%C@?+|IX@n-mPk6yUr( zG3Exrl8;1r*g5Znd}ShqS4gq1YCb@~3C9{;X|Bb5rk8k-*Lsb-#s@0Y&qoWiBEZ-N za%#P9B%GkNnriCBNLO`gGX$wFnL%DW6_-tgf9vebF)eCe0NKBXq>d%tvk-n;Zt?&- zel6=`<$Qk?Ur~nBLSGf68L~ts*Qj|J+ynA`&wbyTj;kB*j&j8o>xPUVvlz-o~`P;bu$04{sM)ybs zjei{pX=tQ6!7tQA;v+@Pr5XxDZIdknp~ExlDFE}g5#Ue@vUEvbp@R2;8Yk|!%?TBc z5%jtSY@{Dk7b1yyre?A|WS)7hu`zu5;rZj0E<6R9p{%T&B%UAt+k4vVyq%!1bTP_; znD<$IRFuSa8s29gnkYWqY}XWQc7%aLA$W{f+Ntmr)eK*!tbPqBQ3*JrqS!Bi>ekmD z-heW0@lN)u9i$YfbdRcv*r6{Z6z@XNR^wyTnOB6 zXEL^?7a4Fsi*V!cOW3ApFxU_3J|v#AD4Nir@87vnYMsWKZ z*`{%!koSx#jm_z`mNbps*RD}&4{Sf^DF^r!$g#~`LE<{cK%z5dbX7Gz}Lkv#VyIC(58=rn|46|`NWI5CJqqK-HiEk8*PI{qzwF_3*TLEM{Bh?mN9h+_K{Bp;)37iNA|PO;G)#!-5QEhjvSUQ0#}s3`f@CaW>Zkz z=8*Zn545hcPzWM0uy>MFy}C6q^_-sL4+@AhuekawJaVnJzkpRCxSzT|7QIh2CwaR} zlz=!37KLyT6&Qs{9;_4kVW*wvYBq$O6hD~LcQHWUM|>vo8WI)jW5s-!<5%M&ZE}g5 zrWq`#wfZ7hRi)K)4CQvLi2P+UT5LL>0Snl!PMsyvPCfp;4AbSxnv@ihOcxQZV%&gWnR5;M3Gz8 z3PeJg682V6)pam0?CMj3u^^~o4v^660+Afd9@%~sB;T!9;#MC`y=yA^a2VP6PRv~^ z>L;sUE2bT~O|M5_O}?b&S;MhD_A`|%Y2{E0`yzdb`{Yms&UUpfH~czuEN`<0Bb6L6 z(cyuHH%rL`Qk;C(p!$swGKGWx5CvTa)C|Zep>0veW!-z`Pr0cyj#QwdlzAK_rhtE` z^VFeAxh;;>e}MdT5&HH=w+a&AQ6d1YpUIj3Erkv^vHQ<5=sPdP&mkZn0M+>b*Khi*7y%G;Dp?tN(SKG#@^&_oIn5MKQ#cVF@Gwb0rx*^{96+KpQwKJ z!E-qR-2SQJzvb%x#(d_ko_q#qSt)255SuNTm;X!fIC${Aj~hI1px{HmNt5Z|B<142{>fK?(i1SO}v2iGX4dS z6W|&7CqTf+;p)cc;Fjizl8S=p@r85bmb1fWPTo^e=bXa zm+2R+1xOoPIynI3L4?gLjra@<01Q%k)_VV!P5mVSNQwK3CZNOR03H5U;|K%1{=Xm) zvDX7+a8v?F35wcS8A;mMSUB1Kx@(Tg&Etyz!tsoNmXbd=9B@a6{}0grPCNC}_I{1K zcdY7A3P4!`TmYay6&%1X_(hY&{$q8&#>*^2yZr*_V`e~ZhQH!LQvVvy+QCuJ((=FA za3*v!FCpMfy#{k8dyTa*D02neux`1wJ8E{|4=U%3tyl@F&1eEBje~ z<|uhCwgA9Cy;MJAcV1S0nX%-#a`xXV|0ik0f1eG$gnyZZ;u)UY^lxqZ5B%?}BwiA| zRFZ!t8n^r#(VyD?Uv%YP!oQS6e}*@*{wMesljWDNFO|TbVS8=<3HHx^hL@NxWo4f+ zN1Xl%^N%Qi|Mq2kDd75y+T{EjsQ+Eg^=0#4ic&rkNxALt}nsf=eTz3|_l`Ul~R zmrO5Z37(mBqJD$v4|CxArAWa`s+ZB=&r}MrzfA@BzS#a*+3h9C%i!8)60?NgCi&xi z{gd3tOO}^WoX;%ANx#kV=a|ly1TQ1#o(UvU|33--SC74nX?mt21gzR#jB$VZy#M>7 z_CNdTWpK+gzDL$?;Qw=|%gcUy84K`C)(lvW{I4JL>q*wj9q4DwzPOF^WS)0PCNf(M*m|Nf9ZL7 zrors`zbV~+^TYh7&HwSb{Ml*p)9dnFtN>vD%?BeZ0SZ_L{S5e{2hstYLKp!2EfCQE E17(ftEC2ui literal 0 HcmV?d00001 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..568c50b --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-4.5.1-bin.zip diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..cccdd3d --- /dev/null +++ b/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..e95643d --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/lib/filewatch/bootstrap.rb b/lib/filewatch/bootstrap.rb new file mode 100644 index 0000000..5eac4fa --- /dev/null +++ b/lib/filewatch/bootstrap.rb @@ -0,0 +1,74 @@ +# encoding: utf-8 +require "rbconfig" +require "pathname" +# require "logstash/environment" + +## Common setup +# all the required constants and files +# defined in one place +module FileWatch + # the number of bytes read from a file during the read phase + FILE_READ_SIZE = 32768 + # the largest fixnum in ruby + # this is used in the read loop e.g. + # @opts[:file_chunk_count].times do + # where file_chunk_count defaults to this constant + FIXNUM_MAX = (2**(0.size * 8 - 2) - 1) + + require_relative "helper" + + module WindowsInode + def prepare_inode(path, stat) + fileId = Winhelper.GetWindowsUniqueFileIdentifier(path) + [fileId, 0, 0] # dev_* doesn't make sense on Windows + end + end + + module UnixInode + def prepare_inode(path, stat) + [stat.ino.to_s, stat.dev_major, stat.dev_minor] + end + end + + jar_version = IO.read("JAR_VERSION").strip + + require "java" + require_relative "../../lib/jars/filewatch-#{jar_version}.jar" + require "jruby_file_watch" + + if LogStash::Environment.windows? + require "winhelper" + FileOpener = FileExt + InodeMixin = WindowsInode + else + FileOpener = ::File + InodeMixin = UnixInode + end + + # Structs can be used as hash keys because they compare by value + # this is used as the key for values in the sincedb hash + InodeStruct = Struct.new(:inode, :maj, :min) do + def to_s + to_a.join(" ") + end + end + + class NoSinceDBPathGiven < StandardError; end + + # how often (in seconds) we logger.warn a failed file open, per path. + OPEN_WARN_INTERVAL = ENV.fetch("FILEWATCH_OPEN_WARN_INTERVAL", 300).to_i + MAX_FILES_WARN_INTERVAL = ENV.fetch("FILEWATCH_MAX_FILES_WARN_INTERVAL", 20).to_i + + require "logstash/util/buftok" + require_relative "settings" + require_relative "sincedb_value" + require_relative "sincedb_record_serializer" + require_relative "watched_files_collection" + require_relative "sincedb_collection" + require_relative "watch" + require_relative "watched_file" + require_relative "discoverer" + require_relative "observing_base" + require_relative "observing_tail" + require_relative "observing_read" +end diff --git a/lib/filewatch/discoverer.rb b/lib/filewatch/discoverer.rb new file mode 100644 index 0000000..b98f77d --- /dev/null +++ b/lib/filewatch/discoverer.rb @@ -0,0 +1,94 @@ +# encoding: utf-8 +require "logstash/util/loggable" + +module FileWatch + class Discoverer + # given a path or glob will prepare for and discover files to watch + # if they are not excluded or ignorable + # they are added to the watched_files collection and + # associated with a sincedb entry if one can be found + include LogStash::Util::Loggable + + def initialize(watched_files_collection, sincedb_collection, settings) + @watching = [] + @exclude = [] + @watched_files_collection = watched_files_collection + @sincedb_collection = sincedb_collection + @settings = settings + @settings.exclude.each { |p| @exclude << p } + end + + def add_path(path) + return if @watching.member?(path) + @watching << path + discover_files(path) + self + end + + def discover + @watching.each do |path| + discover_files(path) + end + end + + private + + def can_exclude?(watched_file, new_discovery) + @exclude.each do |pattern| + if watched_file.pathname.fnmatch?(pattern) + if new_discovery + logger.debug("Discoverer can_exclude?: #{watched_file.path}: skipping " + + "because it matches exclude #{pattern}") + end + watched_file.unwatch + return true + end + end + false + end + + def discover_files(path) + globbed = Dir.glob(path) + globbed = [path] if globbed.empty? + logger.debug("Discoverer found files, count: #{globbed.size}") + globbed.each do |file| + logger.debug("Discoverer found file, path: #{file}") + pathname = Pathname.new(file) + next unless pathname.file? + next if pathname.symlink? + new_discovery = false + watched_file = @watched_files_collection.watched_file_by_path(file) + if watched_file.nil? + logger.debug("Discoverer discover_files: #{path}: new: #{file} (exclude is #{@exclude.inspect})") + new_discovery = true + watched_file = WatchedFile.new(pathname, pathname.stat, @settings) + end + # if it already unwatched or its excluded then we can skip + next if watched_file.unwatched? || can_exclude?(watched_file, new_discovery) + + if new_discovery + if watched_file.file_ignorable? + logger.debug("Discoverer discover_files: #{file}: skipping because it was last modified more than #{@settings.ignore_older} seconds ago") + # on discovery ignorable watched_files are put into the ignored state and that + # updates the size from the internal stat + # so the existing contents are not read. + # because, normally, a newly discovered file will + # have a watched_file size of zero + # they are still added to the collection so we know they are there for the next periodic discovery + watched_file.ignore + end + # now add the discovered file to the watched_files collection and adjust the sincedb collections + @watched_files_collection.add(watched_file) + # initially when the sincedb collection is filled with records from the persistence file + # each value is not associated with a watched file + # a sincedb_value can be: + # unassociated + # associated with this watched_file + # associated with a different watched_file + @sincedb_collection.associate(watched_file) + end + # at this point the watched file is created, is in the db but not yet opened or being processed + end + end + end +end diff --git a/lib/filewatch/helper.rb b/lib/filewatch/helper.rb new file mode 100644 index 0000000..5da8c0f --- /dev/null +++ b/lib/filewatch/helper.rb @@ -0,0 +1,65 @@ +# encoding: utf-8 +# code downloaded from Ruby on Rails 4.2.1 +# https://raw.githubusercontent.com/rails/rails/v4.2.1/activesupport/lib/active_support/core_ext/file/atomic.rb +# change method name to avoid borking active_support and vice versa +require 'fileutils' + +module FileHelper + extend self + # Write to a file atomically. Useful for situations where you don't + # want other processes or threads to see half-written files. + # + # File.write_atomically('important.file') do |file| + # file.write('hello') + # end + def write_atomically(file_name) + + if File.exist?(file_name) + # Get original file permissions + old_stat = File.stat(file_name) + else + # If not possible, probe which are the default permissions in the + # destination directory. + old_stat = probe_stat_in(File.dirname(file_name)) + end + + mode = old_stat ? old_stat.mode : nil + + # Create temporary file with identical permissions + temp_file = File.new(rand_filename(file_name), "w", mode) + temp_file.binmode + return_val = yield temp_file + temp_file.close + + # Overwrite original file with temp file + File.rename(temp_file.path, file_name) + + # Unable to get permissions of the original file => return + return return_val if old_stat.nil? + + # Set correct uid/gid on new file + File.chown(old_stat.uid, old_stat.gid, file_name) if old_stat + + return_val + end + + def device?(file_name) + File.chardev?(file_name) || File.blockdev?(file_name) + end + + # Private utility method. + def probe_stat_in(dir) #:nodoc: + basename = rand_filename(".permissions_check") + file_name = File.join(dir, basename) + FileUtils.touch(file_name) + File.stat(file_name) + rescue + # ... + ensure + FileUtils.rm_f(file_name) if File.exist?(file_name) + end + + def rand_filename(prefix) + [ prefix, Thread.current.object_id, Process.pid, rand(1000000) ].join('.') + end +end diff --git a/lib/filewatch/observing_base.rb b/lib/filewatch/observing_base.rb new file mode 100644 index 0000000..387ba77 --- /dev/null +++ b/lib/filewatch/observing_base.rb @@ -0,0 +1,97 @@ +# encoding: utf-8 + +## Interface API topology +# ObservingBase module (this file) +# is a module mixin proving common constructor and external API for File Input Plugin interaction +# calls build_specific_processor on ObservingRead or ObservingTail +# ObservingRead and ObservingTail +# provides the External API method subscribe(observer = NullObserver.new) +# build_specific_processor(settings) - provide a Tail or Read specific Processor. +# TailMode::Processor or ReadMode::Processor +# initialize_handlers(sincedb_collection, observer) - called when the observer subscribes to changes in a Mode, +# builds mode specific handler instances with references to the observer +# process_closed(watched_files) - provide specific processing of watched_files in the closed state +# process_ignored(watched_files) - provide specific processing of watched_files in the ignored state +# process_watched(watched_files) - provide specific processing of watched_files in the watched state +# process_active(watched_files) - provide specific processing of watched_files in the active state +# These methods can call "handler" methods that delegate to the specific Handler classes. +# TailMode::Handlers module namespace +# contains the Handler classes that deals with Tail mode file lifecycle "events". +# The TailMode::Handlers::Base +# handle(watched_file) - this method calls handle_specifically defined in a subclass +# handle_specifically(watched_file) - this is a noop method +# update_existing_specifically(watched_file, sincedb_value) - this is a noop method +# Each handler extends the Base class to provide specific implementations of these two methods: +# handle_specifically(watched_file) +# update_existing_specifically(watched_file, sincedb_value) +# ReadMode::Handlers module namespace +# contains the Handler classes that deals with Read mode file lifecycle "events". +# The ReadMode::Handlers::Base +# handle(watched_file) - this method calls handle_specifically defined in a subclass +# handle_specifically(watched_file) - this is a noop method +# Each handler extends the Base class to provide specific implementations of this method: +# handle_specifically(watched_file) + +module FileWatch + module ObservingBase + attr_reader :watch, :sincedb_collection, :settings + + def initialize(opts={}) + options = { + :sincedb_write_interval => 10, + :stat_interval => 1, + :discover_interval => 5, + :exclude => [], + :start_new_files_at => :end, + :delimiter => "\n", + :file_chunk_count => FIXNUM_MAX, + :file_sort_by => "last_modified", + :file_sort_direction => "asc", + }.merge(opts) + unless options.include?(:sincedb_path) + raise NoSinceDBPathGiven.new("No sincedb_path set in options. This should have been added in the main LogStash::Inputs::File class") + end + @settings = Settings.from_options(options) + build_watch_and_dependencies + end + + def build_watch_and_dependencies + logger.info("START, creating Discoverer, Watch with file and sincedb collections") + watched_files_collection = WatchedFilesCollection.new(@settings) + @sincedb_collection = SincedbCollection.new(@settings) + @sincedb_collection.open + discoverer = Discoverer.new(watched_files_collection, @sincedb_collection, @settings) + @watch = Watch.new(discoverer, watched_files_collection, @settings) + @watch.add_processor build_specific_processor(@settings) + end + + def watch_this(path) + @watch.watch(path) + end + + def sincedb_write(reason=nil) + # can be invoked from the file input + @sincedb_collection.write(reason) + end + + # quit is a sort-of finalizer, + # it should be called for clean up + # before the instance is disposed of. + def quit + logger.info("QUIT - closing all files and shutting down.") + @watch.quit # <-- should close all the files + # sincedb_write("shutting down") + end + + # close_file(path) is to be used by external code + # when it knows that it is completely done with a file. + # Other files or folders may still be being watched. + # Caution, once unwatched, a file can't be watched again + # unless a new instance of this class begins watching again. + # The sysadmin should rename, move or delete the file. + def close_file(path) + @watch.unwatch(path) + sincedb_write + end + end +end diff --git a/lib/filewatch/observing_read.rb b/lib/filewatch/observing_read.rb new file mode 100644 index 0000000..272fee9 --- /dev/null +++ b/lib/filewatch/observing_read.rb @@ -0,0 +1,23 @@ +# encoding: utf-8 +require "logstash/util/loggable" +require_relative "read_mode/processor" + +module FileWatch + class ObservingRead + include LogStash::Util::Loggable + include ObservingBase + + def subscribe(observer) + # observer here is the file input + watch.subscribe(observer, sincedb_collection) + sincedb_collection.write("read mode subscribe complete - shutting down") + end + + private + + def build_specific_processor(settings) + ReadMode::Processor.new(settings) + end + + end +end diff --git a/lib/filewatch/observing_tail.rb b/lib/filewatch/observing_tail.rb new file mode 100644 index 0000000..e2f51b6 --- /dev/null +++ b/lib/filewatch/observing_tail.rb @@ -0,0 +1,22 @@ +# encoding: utf-8 +require "logstash/util/loggable" +require_relative 'tail_mode/processor' + +module FileWatch + class ObservingTail + include LogStash::Util::Loggable + include ObservingBase + + def subscribe(observer) + # observer here is the file input + watch.subscribe(observer, sincedb_collection) + sincedb_collection.write("tail mode subscribe complete - shutting down") + end + + private + + def build_specific_processor(settings) + TailMode::Processor.new(settings) + end + end +end diff --git a/lib/filewatch/read_mode/handlers/base.rb b/lib/filewatch/read_mode/handlers/base.rb new file mode 100644 index 0000000..3a8680e --- /dev/null +++ b/lib/filewatch/read_mode/handlers/base.rb @@ -0,0 +1,81 @@ +# encoding: utf-8 +require "logstash/util/loggable" + +module FileWatch module ReadMode module Handlers + class Base + include LogStash::Util::Loggable + + attr_reader :sincedb_collection + + def initialize(sincedb_collection, observer, settings) + @settings = settings + @sincedb_collection = sincedb_collection + @observer = observer + end + + def handle(watched_file) + logger.debug("handling: #{watched_file.path}") + unless watched_file.has_listener? + watched_file.set_listener(@observer) + end + handle_specifically(watched_file) + end + + def handle_specifically(watched_file) + # some handlers don't need to define this method + end + + private + + def open_file(watched_file) + return true if watched_file.file_open? + logger.debug("opening #{watched_file.path}") + begin + watched_file.open + rescue + # don't emit this message too often. if a file that we can't + # read is changing a lot, we'll try to open it more often, and spam the logs. + now = Time.now.to_i + logger.warn("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") + if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL + logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}") + watched_file.last_open_warning_at = now + else + logger.debug("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}") + end + watched_file.watch # set it back to watch so we can try it again + end + if watched_file.file_open? + watched_file.listener.opened + true + else + false + end + end + + def add_or_update_sincedb_collection(watched_file) + sincedb_value = @sincedb_collection.find(watched_file) + if sincedb_value.nil? + add_new_value_sincedb_collection(watched_file) + elsif sincedb_value.watched_file == watched_file + update_existing_sincedb_collection_value(watched_file, sincedb_value) + else + logger.warn? && logger.warn("mismatch on sincedb_value.watched_file, this should have been handled by Discoverer") + end + watched_file.initial_completed + end + + def update_existing_sincedb_collection_value(watched_file, sincedb_value) + logger.debug("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}") + # sincedb_value is the source of truth + watched_file.update_bytes_read(sincedb_value.position) + end + + def add_new_value_sincedb_collection(watched_file) + sincedb_value = SincedbValue.new(0) + sincedb_value.set_watched_file(watched_file) + logger.debug("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position) + sincedb_collection.set(watched_file.sincedb_key, sincedb_value) + end + end +end end end diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb new file mode 100644 index 0000000..cff1a76 --- /dev/null +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -0,0 +1,47 @@ +# encoding: utf-8 + +module FileWatch module ReadMode module Handlers + class ReadFile < Base + def handle_specifically(watched_file) + if open_file(watched_file) + add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) + # if the `file_chunk_count` * `file_chunk_size` is less than the file size + # then this method will be executed multiple times + # and the seek is moved to just after a line boundary as recorded in the sincedb + # for each run - so we reset the buffer + watched_file.reset_buffer + watched_file.file_seek(watched_file.bytes_read) + changed = false + @settings.file_chunk_count.times do + begin + lines = watched_file.buffer_extract(watched_file.file_read(@settings.file_chunk_size)) + logger.warn("read_to_eof: no delimiter found in current chunk") if lines.empty? + changed = true + lines.each do |line| + watched_file.listener.accept(line) + sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size) + end + rescue EOFError + # flush the buffer now in case there is no final delimiter + line = watched_file.buffer.flush + watched_file.listener.accept(line) unless line.empty? + watched_file.listener.eof + watched_file.file_close + sincedb_collection.unset_watched_file(watched_file) + watched_file.listener.deleted + watched_file.unwatch + break + rescue Errno::EWOULDBLOCK, Errno::EINTR + watched_file.listener.error + break + rescue => e + logger.error("read_to_eof: general error reading #{watched_file.path} - error: #{e.inspect}") + watched_file.listener.error + break + end + end + sincedb_collection.request_disk_flush if changed + end + end + end +end end end diff --git a/lib/filewatch/read_mode/handlers/read_zip_file.rb b/lib/filewatch/read_mode/handlers/read_zip_file.rb new file mode 100644 index 0000000..75689fc --- /dev/null +++ b/lib/filewatch/read_mode/handlers/read_zip_file.rb @@ -0,0 +1,57 @@ +# encoding: utf-8 +require 'java' +java_import java.io.InputStream +java_import java.io.InputStreamReader +java_import java.io.FileInputStream +java_import java.io.BufferedReader +java_import java.util.zip.GZIPInputStream +java_import java.util.zip.ZipException + +module FileWatch module ReadMode module Handlers + class ReadZipFile < Base + def handle_specifically(watched_file) + add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) + # can't really stripe read a zip file, its all or nothing. + watched_file.listener.opened + # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading) + # should we track lines read in the sincedb and + # fast forward through the lines until we reach unseen content? + # meaning that we can quit in the middle of a zip file + begin + file_stream = FileInputStream.new(watched_file.path) + gzip_stream = GZIPInputStream.new(file_stream) + decoder = InputStreamReader.new(gzip_stream, "UTF-8") + buffered = BufferedReader.new(decoder) + while (line = buffered.readLine(false)) + watched_file.listener.accept(line) + end + watched_file.listener.eof + rescue ZipException => e + logger.error("Cannot decompress the gzip file at path: #{watched_file.path}") + watched_file.listener.error + else + sincedb_collection.store_last_read(watched_file.sincedb_key, watched_file.last_stat_size) + sincedb_collection.request_disk_flush + watched_file.listener.deleted + watched_file.unwatch + ensure + # rescue each close individually so all close attempts are tried + close_and_ignore_ioexception(buffered) unless buffered.nil? + close_and_ignore_ioexception(decoder) unless decoder.nil? + close_and_ignore_ioexception(gzip_stream) unless gzip_stream.nil? + close_and_ignore_ioexception(file_stream) unless file_stream.nil? + end + sincedb_collection.unset_watched_file(watched_file) + end + + private + + def close_and_ignore_ioexception(closeable) + begin + closeable.close + rescue Exception => e # IOException can be thrown by any of the Java classes that implement the Closable interface. + logger.warn("Ignoring an IOException when closing an instance of #{closeable.class.name}", "exception" => e) + end + end + end +end end end diff --git a/lib/filewatch/read_mode/processor.rb b/lib/filewatch/read_mode/processor.rb new file mode 100644 index 0000000..5a8c615 --- /dev/null +++ b/lib/filewatch/read_mode/processor.rb @@ -0,0 +1,117 @@ +# encoding: utf-8 +require "logstash/util/loggable" + +require_relative "handlers/base" +require_relative "handlers/read_file" +require_relative "handlers/read_zip_file" + +module FileWatch module ReadMode + # Must handle + # :read_file + # :read_zip_file + class Processor + include LogStash::Util::Loggable + + attr_reader :watch, :deletable_filepaths + + def initialize(settings) + @settings = settings + @deletable_filepaths = [] + end + + def add_watch(watch) + @watch = watch + self + end + + def initialize_handlers(sincedb_collection, observer) + @read_file = Handlers::ReadFile.new(sincedb_collection, observer, @settings) + @read_zip_file = Handlers::ReadZipFile.new(sincedb_collection, observer, @settings) + end + + def read_file(watched_file) + @read_file.handle(watched_file) + end + + def read_zip_file(watched_file) + @read_zip_file.handle(watched_file) + end + + def process_closed(watched_files) + # do not process watched_files in the closed state. + end + + def process_ignored(watched_files) + # do not process watched_files in the ignored state. + end + + def process_watched(watched_files) + logger.debug("Watched processing") + # Handles watched_files in the watched state. + # for a slice of them: + # move to the active state + # should never have been active before + # how much of the max active window is available + to_take = @settings.max_active - watched_files.count{|wf| wf.active?} + if to_take > 0 + watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file| + path = watched_file.path + begin + watched_file.restat + watched_file.activate + rescue Errno::ENOENT + common_deleted_reaction(watched_file, "Watched") + next + rescue => e + common_error_reaction(path, e, "Watched") + next + end + break if watch.quit? + end + else + now = Time.now.to_i + if (now - watch.lastwarn_max_files) > MAX_FILES_WARN_INTERVAL + waiting = watched_files.size - @settings.max_active + logger.warn(@settings.max_warn_msg + ", files yet to open: #{waiting}") + watch.lastwarn_max_files = now + end + end + end + + def process_active(watched_files) + logger.debug("Active processing") + # Handles watched_files in the active state. + watched_files.select {|wf| wf.active? }.each do |watched_file| + path = watched_file.path + begin + watched_file.restat + rescue Errno::ENOENT + common_deleted_reaction(watched_file, "Active") + next + rescue => e + common_error_reaction(path, e, "Active") + next + end + break if watch.quit? + + if watched_file.compressed? + read_zip_file(watched_file) + else + read_file(watched_file) + end + # handlers take care of closing and unwatching + end + end + + def common_deleted_reaction(watched_file, action) + # file has gone away or we can't read it anymore. + watched_file.unwatch + deletable_filepaths << watched_file.path + logger.debug("#{action} - stat failed: #{watched_file.path}, removing from collection") + end + + def common_error_reaction(path, error, action) + logger.error("#{action} - other error #{path}: (#{error.message}, #{error.backtrace.take(8).inspect})") + end + end +end end diff --git a/lib/filewatch/settings.rb b/lib/filewatch/settings.rb new file mode 100644 index 0000000..5d7b61f --- /dev/null +++ b/lib/filewatch/settings.rb @@ -0,0 +1,67 @@ +# encoding: utf-8 + +module FileWatch + class Settings + attr_reader :delimiter, :close_older, :ignore_older, :delimiter_byte_size + attr_reader :max_active, :max_warn_msg, :lastwarn_max_files + attr_reader :sincedb_write_interval, :stat_interval, :discover_interval + attr_reader :exclude, :start_new_files_at, :file_chunk_count, :file_chunk_size + attr_reader :sincedb_path, :sincedb_write_interval, :sincedb_expiry_duration + attr_reader :file_sort_by, :file_sort_direction + + def self.from_options(opts) + new.add_options(opts) + end + + def self.days_to_seconds(days) + (24 * 3600) * days.to_f + end + + def initialize + defaults = { + :delimiter => "\n", + :file_chunk_size => FILE_READ_SIZE, + :max_active => 4095, + :file_chunk_count => FIXNUM_MAX, + :sincedb_clean_after => 14, + :exclude => [], + :stat_interval => 1, + :discover_interval => 5, + :file_sort_by => "last_modified", + :file_sort_direction => "asc", + } + @opts = {} + @lastwarn_max_files = 0 + add_options(defaults) + end + + def add_options(opts) + @opts.update(opts) + self.max_open_files = @opts[:max_active] + @delimiter = @opts[:delimiter] + @delimiter_byte_size = @delimiter.bytesize + @file_chunk_size = @opts[:file_chunk_size] + @close_older = @opts[:close_older] + @ignore_older = @opts[:ignore_older] + @sincedb_write_interval = @opts[:sincedb_write_interval] + @stat_interval = @opts[:stat_interval] + @discover_interval = @opts[:discover_interval] + @exclude = Array(@opts[:exclude]) + @start_new_files_at = @opts[:start_new_files_at] + @file_chunk_count = @opts[:file_chunk_count] + @sincedb_path = @opts[:sincedb_path] + @sincedb_write_interval = @opts[:sincedb_write_interval] + @sincedb_expiry_duration = self.class.days_to_seconds(@opts.fetch(:sincedb_clean_after, 14)) + @file_sort_by = @opts[:file_sort_by] + @file_sort_direction = @opts[:file_sort_direction] + self + end + + def max_open_files=(value) + val = value.to_i + val = 4095 if value.nil? || val <= 0 + @max_warn_msg = "Reached open files limit: #{val}, set by the 'max_open_files' option or default" + @max_active = val + end + end +end diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb new file mode 100644 index 0000000..6d5dba2 --- /dev/null +++ b/lib/filewatch/sincedb_collection.rb @@ -0,0 +1,215 @@ +# encoding: utf-8 +require "logstash/util/loggable" + +module FileWatch + # this KV collection has a watched_file storage_key (an InodeStruct) as the key + # and a SincedbValue as the value. + # the SincedbValues are built by reading the sincedb file. + class SincedbCollection + include LogStash::Util::Loggable + + attr_reader :path + attr_writer :serializer + + def initialize(settings) + @settings = settings + @sincedb_last_write = 0 + @sincedb = {} + @serializer = SincedbRecordSerializer.new(@settings.sincedb_expiry_duration) + @path = Pathname.new(@settings.sincedb_path) + @write_method = LogStash::Environment.windows? || @path.chardev? || @path.blockdev? ? method(:non_atomic_write) : method(:atomic_write) + @full_path = @path.to_path + FileUtils.touch(@full_path) + end + + def request_disk_flush + now = Time.now.to_i + delta = now - @sincedb_last_write + if delta >= @settings.sincedb_write_interval + logger.debug("writing sincedb (delta since last write = #{delta})") + sincedb_write(now) + end + end + + def write(reason=nil) + logger.debug("caller requested sincedb write (#{reason})") + sincedb_write + end + + def open + @time_sdb_opened = Time.now.to_f + begin + path.open do |file| + logger.debug("open: reading from #{path}") + @serializer.deserialize(file) do |key, value| + logger.debug("open: importing ... '#{key}' => '#{value}'") + set_key_value(key, value) + end + end + logger.debug("open: count of keys read: #{@sincedb.keys.size}") + rescue => e + #No existing sincedb to load + logger.debug("open: error: #{path}: #{e.inspect}") + end + + end + + def associate(watched_file) + logger.debug("associate: finding: #{watched_file.path}") + sincedb_value = find(watched_file) + if sincedb_value.nil? + # sincedb has no record of this inode + # and due to the window handling of many files + # this file may not be opened in this session. + # a new value will be added when the file is opened + return + end + if sincedb_value.watched_file.nil? + # not associated + if sincedb_value.path_in_sincedb.nil? + # old v1 record, assume its the same file + handle_association(sincedb_value, watched_file) + return + end + if sincedb_value.path_in_sincedb == watched_file.path + # the path on disk is the same as discovered path + # and the inode is the same. + handle_association(sincedb_value, watched_file) + return + end + # the path on disk is different from discovered unassociated path + # but they have the same key (inode) + # treat as a new file, a new value will be added when the file is opened + logger.debug("associate: matched but allocated to another - #{sincedb_value}") + sincedb_value.clear_watched_file + delete(watched_file.sincedb_key) + return + end + if sincedb_value.watched_file.equal?(watched_file) # pointer equals + logger.debug("associate: already associated - #{sincedb_value}, for path: #{watched_file.path}") + return + end + # sincedb_value.watched_file is not the discovered watched_file but they have the same key (inode) + # this means that the filename was changed during this session. + # logout the history of the old sincedb_value and remove it + # a new value will be added when the file is opened + # TODO notify about done-ness of old sincedb_value and watched_file + old_watched_file = sincedb_value.watched_file + sincedb_value.clear_watched_file + if logger.debug? + logger.debug("associate: matched but allocated to another - #{sincedb_value}") + logger.debug("associate: matched but allocated to another - old watched_file history - #{old_watched_file.recent_state_history.join(', ')}") + logger.debug("associate: matched but allocated to another - DELETING value at key `#{old_watched_file.sincedb_key}`") + end + delete(old_watched_file.sincedb_key) + end + + def find(watched_file) + get(watched_file.sincedb_key).tap do |obj| + logger.debug("find for path: #{watched_file.path}, found: '#{!obj.nil?}'") + end + end + + def member?(key) + @sincedb.member?(key) + end + + def get(key) + @sincedb[key] + end + + def delete(key) + @sincedb.delete(key) + end + + def last_read(key) + @sincedb[key].position + end + + def rewind(key) + @sincedb[key].update_position(0) + end + + def store_last_read(key, last_read) + @sincedb[key].update_position(last_read) + end + + def increment(key, amount) + @sincedb[key].increment_position(amount) + end + + def set_watched_file(key, watched_file) + @sincedb[key].set_watched_file(watched_file) + end + + def unset_watched_file(watched_file) + return unless member?(watched_file.sincedb_key) + get(watched_file.sincedb_key).unset_watched_file + end + + def clear + @sincedb.clear + end + + def keys + @sincedb.keys + end + + def set(key, value) + @sincedb[key] = value + value + end + + def watched_file_unset?(key) + return false unless member?(key) + get(key).watched_file.nil? + end + + private + + def handle_association(sincedb_value, watched_file) + watched_file.update_bytes_read(sincedb_value.position) + sincedb_value.set_watched_file(watched_file) + watched_file.initial_completed + watched_file.ignore if watched_file.all_read? + end + + def set_key_value(key, value) + if @time_sdb_opened < value.last_changed_at_expires(@settings.sincedb_expiry_duration) + logger.debug("open: setting #{key.inspect} to #{value.inspect}") + set(key, value) + else + logger.debug("open: record has expired, skipping: #{key.inspect} #{value.inspect}") + end + end + + def sincedb_write(time = Time.now.to_i) + logger.debug("sincedb_write: to: #{path}") + begin + @write_method.call + @serializer.expired_keys.each do |key| + @sincedb[key].unset_watched_file + delete(key) + logger.debug("sincedb_write: cleaned", "key" => "'#{key}'") + end + @sincedb_last_write = time + rescue Errno::EACCES + # no file handles free perhaps + # maybe it will work next time + logger.debug("sincedb_write: error: #{path}: #{$!}") + end + end + + def atomic_write + FileHelper.write_atomically(@full_path) do |io| + @serializer.serialize(@sincedb, io) + end + end + + def non_atomic_write + IO.open(@full_path, 0) do |io| + @serializer.serialize(@sincedb, io) + end + end + end +end diff --git a/lib/filewatch/sincedb_record_serializer.rb b/lib/filewatch/sincedb_record_serializer.rb new file mode 100644 index 0000000..81e8a34 --- /dev/null +++ b/lib/filewatch/sincedb_record_serializer.rb @@ -0,0 +1,70 @@ +# encoding: utf-8 + +module FileWatch + class SincedbRecordSerializer + + attr_reader :expired_keys + + def initialize(sincedb_value_expiry) + @sincedb_value_expiry = sincedb_value_expiry + @expired_keys = [] + end + + def update_sincedb_value_expiry_from_days(days) + @sincedb_value_expiry = Settings.days_to_seconds(days) + end + + def serialize(db, io, as_of = Time.now.to_f) + @expired_keys.clear + db.each do |key, value| + if as_of > value.last_changed_at_expires(@sincedb_value_expiry) + @expired_keys << key + next + end + io.write(serialize_record(key, value)) + end + end + + def deserialize(io) + io.each do |record| + yield deserialize_record(record) #.tap{|val| STDERR.puts val} + end + end + + def serialize_record(k, v) + # effectively InodeStruct#to_s SincedbValue#to_s + "#{k} #{v}\n" + end + + def deserialize_record(record) + return [] if record.nil? || record.empty? + parts = record.split(" ") + parse_line_v2(parts) || parse_line_v1(parts) + end + + private + + def parse_line_v2(parts) + # new format e.g. 2977152 1 4 94 1519319662.852678 'path/to/file' + # do we want to store the last known state of the watched file too? + return false if parts.size < 5 + inode_struct = prepare_inode_struct(parts) + pos = parts.shift.to_i + expires_at = Float(parts.shift) # this is like Time.now.to_f + path_in_sincedb = parts.shift + value = SincedbValue.new(pos, expires_at).add_path_in_sincedb(path_in_sincedb) + [inode_struct, value] + end + + def parse_line_v1(parts) + # old inode based e.g. 2977152 1 4 94 + inode_struct = prepare_inode_struct(parts) + pos = parts.shift.to_i + [inode_struct, SincedbValue.new(pos)] + end + + def prepare_inode_struct(parts) + InodeStruct.new(parts.shift, *parts.shift(2).map(&:to_i)) + end + end +end diff --git a/lib/filewatch/sincedb_value.rb b/lib/filewatch/sincedb_value.rb new file mode 100644 index 0000000..d5fa921 --- /dev/null +++ b/lib/filewatch/sincedb_value.rb @@ -0,0 +1,87 @@ +# encoding: utf-8 + +module FileWatch + # Tracks the position and expiry of the offset of a file-of-interest + class SincedbValue + attr_reader :last_changed_at, :watched_file, :path_in_sincedb + + def initialize(position, last_changed_at = nil, watched_file = nil) + @position = position # this is the value read from disk + @last_changed_at = last_changed_at + @watched_file = watched_file + touch if @last_changed_at.nil? || @last_changed_at.zero? + end + + def add_path_in_sincedb(path) + @path_in_sincedb = path # can be nil + self + end + + def last_changed_at_expires(duration) + @last_changed_at + duration + end + + def position + # either the value from disk or the current wf position + @watched_file.nil? ? @position : @watched_file.bytes_read + end + + def update_position(pos) + touch + if @watched_file.nil? + @position = pos + else + @watched_file.update_bytes_read(pos) + end + end + + def increment_position(pos) + touch + if watched_file.nil? + @position += pos + else + @watched_file.increment_bytes_read(pos) + end + end + + def set_watched_file(watched_file) + touch + @watched_file = watched_file + end + + def touch + @last_changed_at = Time.now.to_f + end + + def to_s + # consider serializing the watched_file state as well + "#{position} #{last_changed_at}".tap do |s| + if @watched_file.nil? + s.concat(" ").concat(@path_in_sincedb) unless @path_in_sincedb.nil? + else + s.concat(" ").concat(@watched_file.path) + end + end + end + + def clear_watched_file + @watched_file = nil + end + + def unset_watched_file + # cache the position + # we don't cache the path here because we know we are done with this file. + # either due via the `delete` handling + # or when read mode is done with a file. + # in the case of `delete` if the file was renamed then @watched_file is the + # watched_file of the previous path and the new path will be discovered and + # it should have the same inode as before. + # The key from the new watched_file should then locate this entry and we + # can resume from the cached position + return if @watched_file.nil? + wf = @watched_file + @watched_file = nil + @position = wf.bytes_read + end + end +end diff --git a/lib/filewatch/tail_mode/handlers/base.rb b/lib/filewatch/tail_mode/handlers/base.rb new file mode 100644 index 0000000..c8bc7a0 --- /dev/null +++ b/lib/filewatch/tail_mode/handlers/base.rb @@ -0,0 +1,124 @@ +# encoding: utf-8 +require "logstash/util/loggable" + +module FileWatch module TailMode module Handlers + class Base + include LogStash::Util::Loggable + attr_reader :sincedb_collection + + def initialize(sincedb_collection, observer, settings) + @settings = settings + @sincedb_collection = sincedb_collection + @observer = observer + end + + def handle(watched_file) + logger.debug("handling: #{watched_file.path}") + unless watched_file.has_listener? + watched_file.set_listener(@observer) + end + handle_specifically(watched_file) + end + + def handle_specifically(watched_file) + # some handlers don't need to define this method + end + + def update_existing_specifically(watched_file, sincedb_value) + # when a handler subclass does not implement this then do nothing + end + + private + + def read_to_eof(watched_file) + changed = false + # from a real config (has 102 file inputs) + # -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously + # -- If we put all log files in one file input glob we will have indexing delay, because Logstash waits until the first file becomes EOF + # by allowing the user to specify a combo of `file_chunk_count` X `file_chunk_size`... + # we enable the pseudo parallel processing of each file. + # user also has the option to specify a low `stat_interval` and a very high `discover_interval`to respond + # quicker to changing files and not allowing too much content to build up before reading it. + @settings.file_chunk_count.times do + begin + data = watched_file.file_read(@settings.file_chunk_size) + lines = watched_file.buffer_extract(data) + logger.warn("read_to_eof: no delimiter found in current chunk") if lines.empty? + changed = true + lines.each do |line| + watched_file.listener.accept(line) + sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size) + end + rescue EOFError + # it only makes sense to signal EOF in "read" mode not "tail" + break + rescue Errno::EWOULDBLOCK, Errno::EINTR + watched_file.listener.error + break + rescue => e + logger.error("read_to_eof: general error reading #{watched_file.path}", "error" => e.inspect, "backtrace" => e.backtrace.take(4)) + watched_file.listener.error + break + end + end + sincedb_collection.request_disk_flush if changed + end + + def open_file(watched_file) + return true if watched_file.file_open? + logger.debug("opening #{watched_file.path}") + begin + watched_file.open + rescue + # don't emit this message too often. if a file that we can't + # read is changing a lot, we'll try to open it more often, and spam the logs. + now = Time.now.to_i + logger.warn("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") + if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL + logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}") + watched_file.last_open_warning_at = now + else + logger.debug("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}") + end + watched_file.watch # set it back to watch so we can try it again + end + if watched_file.file_open? + watched_file.listener.opened + true + else + false + end + end + + def add_or_update_sincedb_collection(watched_file) + sincedb_value = @sincedb_collection.find(watched_file) + if sincedb_value.nil? + add_new_value_sincedb_collection(watched_file) + elsif sincedb_value.watched_file == watched_file + update_existing_sincedb_collection_value(watched_file, sincedb_value) + else + logger.warn? && logger.warn("mismatch on sincedb_value.watched_file, this should have been handled by Discoverer") + end + watched_file.initial_completed + end + + def update_existing_sincedb_collection_value(watched_file, sincedb_value) + logger.debug("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}") + update_existing_specifically(watched_file, sincedb_value) + end + + def add_new_value_sincedb_collection(watched_file) + sincedb_value = get_new_value_specifically(watched_file) + logger.debug("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position) + sincedb_collection.set(watched_file.sincedb_key, sincedb_value) + end + + def get_new_value_specifically(watched_file) + position = @settings.start_new_files_at == :beginning ? 0 : watched_file.last_stat_size + value = SincedbValue.new(position) + value.set_watched_file(watched_file) + watched_file.update_bytes_read(position) + value + end + end +end end end diff --git a/lib/filewatch/tail_mode/handlers/create.rb b/lib/filewatch/tail_mode/handlers/create.rb new file mode 100644 index 0000000..2b89c11 --- /dev/null +++ b/lib/filewatch/tail_mode/handlers/create.rb @@ -0,0 +1,17 @@ +# encoding: utf-8 + +module FileWatch module TailMode module Handlers + class Create < Base + def handle_specifically(watched_file) + if open_file(watched_file) + add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) + end + end + + def update_existing_specifically(watched_file, sincedb_value) + # sincedb_value is the source of truth + position = sincedb_value.position + watched_file.update_bytes_read(position) + end + end +end end end diff --git a/lib/filewatch/tail_mode/handlers/create_initial.rb b/lib/filewatch/tail_mode/handlers/create_initial.rb new file mode 100644 index 0000000..65c385b --- /dev/null +++ b/lib/filewatch/tail_mode/handlers/create_initial.rb @@ -0,0 +1,21 @@ +# encoding: utf-8 + +module FileWatch module TailMode module Handlers + class CreateInitial < Base + def handle_specifically(watched_file) + if open_file(watched_file) + add_or_update_sincedb_collection(watched_file) + end + end + + def update_existing_specifically(watched_file, sincedb_value) + position = watched_file.last_stat_size + if @settings.start_new_files_at == :beginning + position = 0 + end + logger.debug("update_existing_specifically - #{watched_file.path}: seeking to #{position}") + watched_file.update_bytes_read(position) + sincedb_value.update_position(position) + end + end +end end end diff --git a/lib/filewatch/tail_mode/handlers/delete.rb b/lib/filewatch/tail_mode/handlers/delete.rb new file mode 100644 index 0000000..39e4c74 --- /dev/null +++ b/lib/filewatch/tail_mode/handlers/delete.rb @@ -0,0 +1,11 @@ +# encoding: utf-8 + +module FileWatch module TailMode module Handlers + class Delete < Base + def handle_specifically(watched_file) + watched_file.listener.deleted + sincedb_collection.unset_watched_file(watched_file) + watched_file.file_close + end + end +end end end diff --git a/lib/filewatch/tail_mode/handlers/grow.rb b/lib/filewatch/tail_mode/handlers/grow.rb new file mode 100644 index 0000000..826017e --- /dev/null +++ b/lib/filewatch/tail_mode/handlers/grow.rb @@ -0,0 +1,11 @@ +# encoding: utf-8 + +module FileWatch module TailMode module Handlers + class Grow < Base + def handle_specifically(watched_file) + watched_file.file_seek(watched_file.bytes_read) + logger.debug("reading to eof: #{watched_file.path}") + read_to_eof(watched_file) + end + end +end end end diff --git a/lib/filewatch/tail_mode/handlers/shrink.rb b/lib/filewatch/tail_mode/handlers/shrink.rb new file mode 100644 index 0000000..9a7f0f0 --- /dev/null +++ b/lib/filewatch/tail_mode/handlers/shrink.rb @@ -0,0 +1,20 @@ +# encoding: utf-8 + +module FileWatch module TailMode module Handlers + class Shrink < Base + def handle_specifically(watched_file) + add_or_update_sincedb_collection(watched_file) + watched_file.file_seek(watched_file.bytes_read) + logger.debug("reading to eof: #{watched_file.path}") + read_to_eof(watched_file) + end + + def update_existing_specifically(watched_file, sincedb_value) + # we have a match but size is smaller + # set all to zero + logger.debug("update_existing_specifically: #{watched_file.path}: was truncated seeking to beginning") + watched_file.update_bytes_read(0) if watched_file.bytes_read != 0 + sincedb_value.update_position(0) + end + end +end end end diff --git a/lib/filewatch/tail_mode/handlers/timeout.rb b/lib/filewatch/tail_mode/handlers/timeout.rb new file mode 100644 index 0000000..248eee0 --- /dev/null +++ b/lib/filewatch/tail_mode/handlers/timeout.rb @@ -0,0 +1,10 @@ +# encoding: utf-8 + +module FileWatch module TailMode module Handlers + class Timeout < Base + def handle_specifically(watched_file) + watched_file.listener.timed_out + watched_file.file_close + end + end +end end end diff --git a/lib/filewatch/tail_mode/handlers/unignore.rb b/lib/filewatch/tail_mode/handlers/unignore.rb new file mode 100644 index 0000000..07cace1 --- /dev/null +++ b/lib/filewatch/tail_mode/handlers/unignore.rb @@ -0,0 +1,37 @@ +# encoding: utf-8 + +module FileWatch module TailMode module Handlers + class Unignore < Base + # a watched file can be put straight into the ignored state + # before any other handling has been done + # at a minimum we create or associate a sincedb value + def handle_specifically(watched_file) + add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) + end + + def get_new_value_specifically(watched_file) + # for file initially ignored their bytes_read was set to stat.size + # use this value not the `start_new_files_at` for the position + # logger.debug("get_new_value_specifically", "watched_file" => watched_file.inspect) + SincedbValue.new(watched_file.bytes_read).tap do |val| + val.set_watched_file(watched_file) + end + end + + def update_existing_specifically(watched_file, sincedb_value) + # when this watched_file was ignored it had it bytes_read set to eof + # now the file has changed (watched_file.size_changed?) + # it has been put into the watched state so when it becomes active + # we will handle grow or shrink + # for now we seek to where we were before the file got ignored (grow) + # or to the start (shrink) + position = 0 + if watched_file.shrunk? + watched_file.update_bytes_read(0) + else + position = watched_file.bytes_read + end + sincedb_value.update_position(position) + end + end +end end end diff --git a/lib/filewatch/tail_mode/processor.rb b/lib/filewatch/tail_mode/processor.rb new file mode 100644 index 0000000..8291280 --- /dev/null +++ b/lib/filewatch/tail_mode/processor.rb @@ -0,0 +1,209 @@ +# encoding: utf-8 +require "logstash/util/loggable" +require_relative "handlers/base" +require_relative "handlers/create_initial" +require_relative "handlers/create" +require_relative "handlers/delete" +require_relative "handlers/grow" +require_relative "handlers/shrink" +require_relative "handlers/timeout" +require_relative "handlers/unignore" + +module FileWatch module TailMode + # Must handle + # :create_initial - file is discovered and we have no record of it in the sincedb + # :create - file is discovered and we have seen it before in the sincedb + # :grow - file has more content + # :shrink - file has less content + # :delete - file can't be read + # :timeout - file is closable + # :unignore - file was ignored, but have now received new content + class Processor + include LogStash::Util::Loggable + + attr_reader :watch, :deletable_filepaths + + def initialize(settings) + @settings = settings + @deletable_filepaths = [] + end + + def add_watch(watch) + @watch = watch + self + end + + def initialize_handlers(sincedb_collection, observer) + @create_initial = Handlers::CreateInitial.new(sincedb_collection, observer, @settings) + @create = Handlers::Create.new(sincedb_collection, observer, @settings) + @grow = Handlers::Grow.new(sincedb_collection, observer, @settings) + @shrink = Handlers::Shrink.new(sincedb_collection, observer, @settings) + @delete = Handlers::Delete.new(sincedb_collection, observer, @settings) + @timeout = Handlers::Timeout.new(sincedb_collection, observer, @settings) + @unignore = Handlers::Unignore.new(sincedb_collection, observer, @settings) + end + + def create(watched_file) + @create.handle(watched_file) + end + + def create_initial(watched_file) + @create_initial.handle(watched_file) + end + + def grow(watched_file) + @grow.handle(watched_file) + end + + def shrink(watched_file) + @shrink.handle(watched_file) + end + + def delete(watched_file) + @delete.handle(watched_file) + end + + def timeout(watched_file) + @timeout.handle(watched_file) + end + + def unignore(watched_file) + @unignore.handle(watched_file) + end + + def process_closed(watched_files) + logger.debug("Closed processing") + # Handles watched_files in the closed state. + # if its size changed it is put into the watched state + watched_files.select {|wf| wf.closed? }.each do |watched_file| + path = watched_file.path + begin + watched_file.restat + if watched_file.size_changed? + # if the closed file changed, move it to the watched state + # not to active state because we want to respect the active files window. + watched_file.watch + end + rescue Errno::ENOENT + # file has gone away or we can't read it anymore. + common_deleted_reaction(watched_file, "Closed") + rescue => e + common_error_reaction(path, e, "Closed") + end + break if watch.quit? + end + end + + def process_ignored(watched_files) + logger.debug("Ignored processing") + # Handles watched_files in the ignored state. + # if its size changed: + # put it in the watched state + # invoke unignore + watched_files.select {|wf| wf.ignored? }.each do |watched_file| + path = watched_file.path + begin + watched_file.restat + if watched_file.size_changed? + watched_file.watch + unignore(watched_file) + end + rescue Errno::ENOENT + # file has gone away or we can't read it anymore. + common_deleted_reaction(watched_file, "Ignored") + rescue => e + common_error_reaction(path, e, "Ignored") + end + break if watch.quit? + end + end + + def process_watched(watched_files) + logger.debug("Watched processing") + # Handles watched_files in the watched state. + # for a slice of them: + # move to the active state + # and we allow the block to open the file and create a sincedb collection record if needed + # some have never been active and some have + # those that were active before but are watched now were closed under constraint + + # how much of the max active window is available + to_take = @settings.max_active - watched_files.count{|wf| wf.active?} + if to_take > 0 + watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file| + path = watched_file.path + begin + watched_file.restat + watched_file.activate + if watched_file.initial? + create_initial(watched_file) + else + create(watched_file) + end + rescue Errno::ENOENT + # file has gone away or we can't read it anymore. + common_deleted_reaction(watched_file, "Watched") + rescue => e + common_error_reaction(path, e, "Watched") + end + break if watch.quit? + end + else + now = Time.now.to_i + if (now - watch.lastwarn_max_files) > MAX_FILES_WARN_INTERVAL + waiting = watched_files.size - @settings.max_active + logger.warn(@settings.max_warn_msg + ", files yet to open: #{waiting}") + watch.lastwarn_max_files = now + end + end + end + + def process_active(watched_files) + logger.debug("Active processing") + # Handles watched_files in the active state. + # it has been read once - unless they were empty at the time + watched_files.select {|wf| wf.active? }.each do |watched_file| + path = watched_file.path + begin + watched_file.restat + rescue Errno::ENOENT + # file has gone away or we can't read it anymore. + common_deleted_reaction(watched_file, "Active") + next + rescue => e + common_error_reaction(path, e, "Active") + next + end + break if watch.quit? + if watched_file.grown? + logger.debug("Active - file grew: #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}") + grow(watched_file) + elsif watched_file.shrunk? + # we don't update the size here, its updated when we actually read + logger.debug("Active - file shrunk #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}") + shrink(watched_file) + else + # same size, do nothing + end + # can any active files be closed to make way for waiting files? + if watched_file.file_closable? + logger.debug("Watch each: active: file expired: #{path}") + timeout(watched_file) + watched_file.close + end + end + end + + def common_deleted_reaction(watched_file, action) + # file has gone away or we can't read it anymore. + watched_file.unwatch + delete(watched_file) + deletable_filepaths << watched_file.path + logger.debug("#{action} - stat failed: #{watched_file.path}, removing from collection") + end + + def common_error_reaction(path, error, action) + logger.error("#{action} - other error #{path}: (#{error.message}, #{error.backtrace.take(8).inspect})") + end + end +end end diff --git a/lib/filewatch/watch.rb b/lib/filewatch/watch.rb new file mode 100644 index 0000000..c544719 --- /dev/null +++ b/lib/filewatch/watch.rb @@ -0,0 +1,107 @@ +# encoding: utf-8 +require "logstash/util/loggable" + +module FileWatch + class Watch + include LogStash::Util::Loggable + + attr_accessor :lastwarn_max_files + attr_reader :discoverer, :watched_files_collection + + def initialize(discoverer, watched_files_collection, settings) + @settings = settings + # watch and iterate_on_state can be called from different threads. + @lock = Mutex.new + # we need to be threadsafe about the quit mutation + @quit = false + @quit_lock = Mutex.new + @lastwarn_max_files = 0 + @discoverer = discoverer + @watched_files_collection = watched_files_collection + end + + def add_processor(processor) + @processor = processor + @processor.add_watch(self) + self + end + + def watch(path) + synchronized do + @discoverer.add_path(path) + end + # don't return whatever @discoverer.add_path returns + return true + end + + def discover + synchronized do + @discoverer.discover + end + # don't return whatever @discoverer.discover returns + return true + end + + def subscribe(observer, sincedb_collection) + @processor.initialize_handlers(sincedb_collection, observer) + + glob = 0 + interval = @settings.discover_interval + reset_quit + until quit? + iterate_on_state + break if quit? + glob += 1 + if glob == interval + discover + glob = 0 + end + break if quit? + sleep(@settings.stat_interval) + end + @watched_files_collection.close_all + end # def subscribe + + # Read mode processor will handle watched_files in the closed, ignored, watched and active state + # differently from Tail mode - see the ReadMode::Processor and TailMode::Processor + def iterate_on_state + return if @watched_files_collection.empty? + synchronized do + begin + # creates this snapshot of watched_file values just once + watched_files = @watched_files_collection.values + @processor.process_closed(watched_files) + return if quit? + @processor.process_ignored(watched_files) + return if quit? + @processor.process_watched(watched_files) + return if quit? + @processor.process_active(watched_files) + ensure + @watched_files_collection.delete(@processor.deletable_filepaths) + @processor.deletable_filepaths.clear + end + end + end # def each + + def quit + @quit_lock.synchronize do + @quit = true + end + end # def quit + + def quit? + @quit_lock.synchronize { @quit } + end + + private + + def synchronized(&block) + @lock.synchronize { block.call } + end + + def reset_quit + @quit_lock.synchronize { @quit = false } + end + end +end diff --git a/lib/filewatch/watched_file.rb b/lib/filewatch/watched_file.rb new file mode 100644 index 0000000..dffdf29 --- /dev/null +++ b/lib/filewatch/watched_file.rb @@ -0,0 +1,226 @@ +# encoding: utf-8 + +module FileWatch + class WatchedFile + include InodeMixin # see bootstrap.rb at `if LogStash::Environment.windows?` + + attr_reader :bytes_read, :state, :file, :buffer, :recent_states + attr_reader :path, :filestat, :accessed_at, :modified_at, :pathname + attr_reader :sdb_key_v1, :last_stat_size, :listener + attr_accessor :last_open_warning_at + + # this class represents a file that has been discovered + def initialize(pathname, stat, settings) + @settings = settings + @pathname = Pathname.new(pathname) # given arg pathname might be a string or a Pathname object + @path = @pathname.to_path + @bytes_read = 0 + @last_stat_size = 0 + # the prepare_inode method is sourced from the mixed module above + @sdb_key_v1 = InodeStruct.new(*prepare_inode(path, stat)) + # initial as true means we have not associated this watched_file with a previous sincedb value yet. + # and we should read from the beginning if necessary + @initial = true + @recent_states = [] # keep last 8 states, managed in set_state + @state = :watched + set_stat(stat) # can change @last_stat_size + @listener = nil + @last_open_warning_at = nil + set_accessed_at + end + + def set_listener(observer) + @listener = observer.listener_for(@path) + end + + def unset_listener + @listener = nil + end + + def has_listener? + !@listener.nil? + end + + def sincedb_key + @sdb_key_v1 + end + + def initial_completed + @initial = false + end + + def set_accessed_at + @accessed_at = Time.now.to_f + end + + def initial? + @initial + end + + def compressed? + @path.end_with?('.gz','.gzip') + end + + def size_changed? + @last_stat_size != bytes_read + end + + def all_read? + @last_stat_size == bytes_read + end + + def open + file_add_opened(FileOpener.open(@path)) + end + + def file_add_opened(rubyfile) + @file = rubyfile + @buffer = BufferedTokenizer.new(@settings.delimiter) if @buffer.nil? + end + + def file_close + return if @file.nil? || @file.closed? + @file.close + @file = nil + end + + def file_seek(amount, whence = IO::SEEK_SET) + @file.sysseek(amount, whence) + end + + def file_read(amount) + set_accessed_at + @file.sysread(amount) + end + + def file_open? + !@file.nil? && !@file.closed? + end + + def reset_buffer + @buffer.flush + end + + def buffer_extract(data) + @buffer.extract(data) + end + + def increment_bytes_read(delta) + return if delta.nil? + @bytes_read += delta + end + + def update_bytes_read(total_bytes_read) + return if total_bytes_read.nil? + @bytes_read = total_bytes_read + end + + def update_path(_path) + @path = _path + end + + def update_stat(st) + set_stat(st) + end + + def activate + set_state :active + end + + def ignore + set_state :ignored + @bytes_read = @filestat.size + end + + def close + set_state :closed + end + + def watch + set_state :watched + end + + def unwatch + set_state :unwatched + end + + def active? + @state == :active + end + + def ignored? + @state == :ignored + end + + def closed? + @state == :closed + end + + def watched? + @state == :watched + end + + def unwatched? + @state == :unwatched + end + + def expiry_close_enabled? + !@settings.close_older.nil? + end + + def expiry_ignore_enabled? + !@settings.ignore_older.nil? + end + + def shrunk? + @last_stat_size < @bytes_read + end + + def grown? + @last_stat_size > @bytes_read + end + + def restat + set_stat(pathname.stat) + end + + def set_state(value) + @recent_states.shift if @recent_states.size == 8 + @recent_states << @state + @state = value + end + + def recent_state_history + @recent_states + Array(@state) + end + + def file_closable? + file_can_close? && all_read? + end + + def file_ignorable? + return false unless expiry_ignore_enabled? + # (Time.now - stat.mtime) <- in jruby, this does int and float + # conversions before the subtraction and returns a float. + # so use all floats upfront + (Time.now.to_f - @modified_at) > @settings.ignore_older + end + + def file_can_close? + return false unless expiry_close_enabled? + (Time.now.to_f - @accessed_at) > @settings.close_older + end + + def to_s + inspect + end + + private + + def set_stat(stat) + @modified_at = stat.mtime.to_f + @last_stat_size = stat.size + @filestat = stat + end + end +end diff --git a/lib/filewatch/watched_files_collection.rb b/lib/filewatch/watched_files_collection.rb new file mode 100644 index 0000000..e341aff --- /dev/null +++ b/lib/filewatch/watched_files_collection.rb @@ -0,0 +1,84 @@ +# encoding: utf-8 +module FileWatch + class WatchedFilesCollection + + def initialize(settings) + @sort_by = settings.file_sort_by # "last_modified" | "path" + @sort_direction = settings.file_sort_direction # "asc" | "desc" + @sort_method = method("#{@sort_by}_#{@sort_direction}".to_sym) + @files = [] + @pointers = {} + end + + def add(watched_file) + @files << watched_file + @sort_method.call + end + + def delete(paths) + Array(paths).each do |f| + index = @pointers.delete(f) + @files.delete_at(index) + end + @sort_method.call + end + + def close_all + @files.each(&:file_close) + end + + def empty? + @files.empty? + end + + def keys + @pointers.keys + end + + def values + @files + end + + def watched_file_by_path(path) + index = @pointers[path] + return nil unless index + @files[index] + end + + private + + def last_modified_asc + @files.sort! do |left, right| + left.modified_at <=> right.modified_at + end + refresh_pointers + end + + def last_modified_desc + @files.sort! do |left, right| + right.modified_at <=> left.modified_at + end + refresh_pointers + end + + def path_asc + @files.sort! do |left, right| + left.path <=> right.path + end + refresh_pointers + end + + def path_desc + @files.sort! do |left, right| + right.path <=> left.path + end + refresh_pointers + end + + def refresh_pointers + @files.each_with_index do |watched_file, index| + @pointers[watched_file.path] = index + end + end + end +end diff --git a/lib/filewatch/winhelper.rb b/lib/filewatch/winhelper.rb new file mode 100644 index 0000000..2440323 --- /dev/null +++ b/lib/filewatch/winhelper.rb @@ -0,0 +1,65 @@ +# encoding: utf-8 +require "ffi" + +module Winhelper + extend FFI::Library + + ffi_lib 'kernel32' + ffi_convention :stdcall + class FileTime < FFI::Struct + layout :lowDateTime, :uint, + :highDateTime, :uint + end + + #http://msdn.microsoft.com/en-us/library/windows/desktop/aa363788(v=vs.85).aspx + class FileInformation < FFI::Struct + layout :fileAttributes, :uint, #DWORD dwFileAttributes; + :createTime, FileTime, #FILETIME ftCreationTime; + :lastAccessTime, FileTime, #FILETIME ftLastAccessTime; + :lastWriteTime, FileTime, #FILETIME ftLastWriteTime; + :volumeSerialNumber, :uint, #DWORD dwVolumeSerialNumber; + :fileSizeHigh, :uint, #DWORD nFileSizeHigh; + :fileSizeLow, :uint, #DWORD nFileSizeLow; + :numberOfLinks, :uint, #DWORD nNumberOfLinks; + :fileIndexHigh, :uint, #DWORD nFileIndexHigh; + :fileIndexLow, :uint #DWORD nFileIndexLow; + end + + + #http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx + #HANDLE WINAPI CreateFile(_In_ LPCTSTR lpFileName,_In_ DWORD dwDesiredAccess,_In_ DWORD dwShareMode, + # _In_opt_ LPSECURITY_ATTRIBUTES lpSecurityAttributes,_In_ DWORD dwCreationDisposition, + # _In_ DWORD dwFlagsAndAttributes,_In_opt_ HANDLE hTemplateFile); + attach_function :GetOpenFileHandle, :CreateFileA, [:pointer, :uint, :uint, :pointer, :uint, :uint, :pointer], :pointer + + #http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952(v=vs.85).aspx + #BOOL WINAPI GetFileInformationByHandle(_In_ HANDLE hFile,_Out_ LPBY_HANDLE_FILE_INFORMATION lpFileInformation); + attach_function :GetFileInformationByHandle, [:pointer, :pointer], :int + + attach_function :CloseHandle, [:pointer], :int + + + def self.GetWindowsUniqueFileIdentifier(path) + handle = GetOpenFileHandle(path, 0, 7, nil, 3, 128, nil) + fileInfo = Winhelper::FileInformation.new + success = GetFileInformationByHandle(handle, fileInfo) + CloseHandle(handle) + if success == 1 + #args = [ + # fileInfo[:fileAttributes], fileInfo[:volumeSerialNumber], fileInfo[:fileSizeHigh], fileInfo[:fileSizeLow], + # fileInfo[:numberOfLinks], fileInfo[:fileIndexHigh], fileInfo[:fileIndexLow] + # ] + #p "Information: %u %u %u %u %u %u %u " % args + #this is only guaranteed on NTFS, for ReFS on windows 2012, GetFileInformationByHandleEx should be used with FILE_ID_INFO, which returns a 128 bit identifier + return "#{fileInfo[:volumeSerialNumber]}-#{fileInfo[:fileIndexLow]}-#{fileInfo[:fileIndexHigh]}" + else + #p "cannot retrieve file information, returning path" + return path; + end + end +end + +#fileId = Winhelper.GetWindowsUniqueFileIdentifier('C:\inetpub\logs\LogFiles\W3SVC1\u_ex1fdsadfsadfasdf30612.log') +#p "FileId: " + fileId +#p "outside function, sleeping" +#sleep(10) diff --git a/lib/logstash/inputs/delete_completed_file_handler.rb b/lib/logstash/inputs/delete_completed_file_handler.rb new file mode 100644 index 0000000..c6a3e91 --- /dev/null +++ b/lib/logstash/inputs/delete_completed_file_handler.rb @@ -0,0 +1,9 @@ +# encoding: utf-8 + +module LogStash module Inputs + class DeleteCompletedFileHandler + def handle(path) + Pathname.new(path).unlink rescue nil + end + end +end end diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index fec3aa7..cb81732 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -6,7 +6,12 @@ require "pathname" require "socket" # for Socket.gethostname require "fileutils" + require_relative "file/patch" +require_relative "file_listener" +require_relative "delete_completed_file_handler" +require_relative "log_completed_file_handler" +require "filewatch/bootstrap" # Stream events from files, normally by tailing them in a manner # similar to `tail -0F` but optionally reading them from the @@ -26,8 +31,8 @@ # # ==== Reading from remote network volumes # -# The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These -# remote filesystems typically have behaviors that are very different from local filesystems and +# The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These +# remote filesystems typically have behaviors that are very different from local filesystems and # are therefore unlikely to work correctly when used with the file input. # # ==== Tracking of current position in watched files @@ -77,8 +82,8 @@ # to the rotation and its reopening under the new name (an interval # determined by the `stat_interval` and `discover_interval` options) # will not get picked up. - -class LogStash::Inputs::File < LogStash::Inputs::Base +module LogStash module Inputs +class File < LogStash::Inputs::Base config_name "file" # The path(s) to the file(s) to use as an input. @@ -149,8 +154,8 @@ class LogStash::Inputs::File < LogStash::Inputs::Base # can be closed (allowing other files to be opened) but will be queued for # reopening when new data is detected. If reading, the file will be closed # after closed_older seconds from when the last bytes were read. - # The default is 1 hour - config :close_older, :validate => :number, :default => 1 * 60 * 60 + # By default, this option disabled + config :close_older, :validate => :number # What is the maximum number of file_handles that this input consumes # at any one time. Use close_older to close some files if you need to @@ -160,10 +165,66 @@ class LogStash::Inputs::File < LogStash::Inputs::Base # The default of 4095 is set in filewatch. config :max_open_files, :validate => :number + # What mode do you want the file input to operate in. + # Tail a few files or read many content-complete files + # The default is tail + # If "read" is specified then the following other settings are ignored + # `start_position` (files are always read from the beginning) + # `delimiter` (files are assumed to use \n or \r (or both) as line endings) + # `close_older` (files are automatically 'closed' when EOF is reached) + # If "read" is specified then the following settings are heeded + # `ignore_older` (older files are not processed) + # "read" mode now supports gzip file processing + config :mode, :validate => [ "tail", "read"], :default => "tail" + + # When in 'read' mode, what action should be carried out when a file is done with. + # If 'delete' is specified then the file will be deleted. + # If 'log' is specified then the full path of the file is logged to the file specified + # in the `file_completed_log_path` setting. + config :file_completed_action, :validate => ["delete", "log", "log_and_delete"], :default => "delete" + + # Which file should the completely read file paths be appended to. + # Only specify this path to a file when `file_completed_action` is 'log' or 'log_and_delete'. + # IMPORTANT: this file is appended to only - it could become very large. You are responsible for file rotation. + config :file_completed_log_path, :validate => :string + + # The sincedb entry now has a last active timestamp associated with it. + # If no changes are detected in tracked files in the last N days their sincedb + # tracking record will expire and not be persisted. + # This option protects against the well known inode recycling problem. (add reference) + config :sincedb_clean_after, :validate => :number, :default => 14 # days + + # File content is read off disk in blocks or chunks, then using whatever the set delimiter + # is, lines are extracted from the chunk. Specify the size in bytes of each chunk. + # See `file_chunk_count` to see why and when to change this from the default. + # The default set internally is 32768 (32KB) + config :file_chunk_size, :validate => :number, :default => FileWatch::FILE_READ_SIZE + + # When combined with the `file_chunk_size`, this option sets how many chunks + # are read from each file before moving to the next active file. + # e.g. a `chunk_count` of 32 with the default `file_chunk_size` will process + # 1MB from each active file. See the option `max_open_files` for more info. + # The default set internally is very large, 4611686018427387903. By default + # the file is read to the end before moving to the next active file. + config :file_chunk_count, :validate => :number, :default => FileWatch::FIXNUM_MAX + + # Which attribute of a discovered file should be used to sort the discovered files. + # Files can be sort by modified date or full path alphabetic. + # The default is `last_modified` + # Previously the processing order of the discovered files was OS dependent. + config :file_sort_by, :validate => ["last_modified", "path"], :default => "last_modified" + + # Choose between ascending and descending order when also choosing between + # `last_modified` and `path` file_sort_by options. + # If ingesting the newest data first is important then opt for last_modified + desc + # If ingesting the oldest data first is important then opt for last_modified + asc + # If you use a special naming convention for the file full paths then + # perhaps path + asc will help to achieve the goal of controlling the order of file ingestion + config :file_sort_direction, :validate => ["asc", "desc"], :default => "asc" + public def register require "addressable/uri" - require "filewatch/tail" require "digest/md5" @logger.trace("Registering file input", :path => @path) @host = Socket.gethostname.force_encoding(Encoding::UTF_8) @@ -171,7 +232,7 @@ def register # won't in older versions of Logstash, then we need to set it to nil. settings = defined?(LogStash::SETTINGS) ? LogStash::SETTINGS : nil - @tail_config = { + @filewatch_config = { :exclude => @exclude, :stat_interval => @stat_interval, :discover_interval => @discover_interval, @@ -179,9 +240,16 @@ def register :delimiter => @delimiter, :ignore_older => @ignore_older, :close_older => @close_older, - :max_open_files => @max_open_files + :max_open_files => @max_open_files, + :sincedb_clean_after => @sincedb_clean_after, + :file_chunk_count => @file_chunk_count, + :file_chunk_size => @file_chunk_size, + :file_sort_by => @file_sort_by, + :file_sort_direction => @file_sort_direction, } + @completed_file_handlers = [] + @path.each do |path| if Pathname.new(path).relative? raise ArgumentError.new("File paths must be absolute, relative path specified: #{path}") @@ -189,132 +257,84 @@ def register end if @sincedb_path.nil? - if settings - datapath = File.join(settings.get_value("path.data"), "plugins", "inputs", "file") - # Ensure that the filepath exists before writing, since it's deeply nested. - FileUtils::mkdir_p datapath - @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) + base_sincedb_path = build_sincedb_base_from_settings(settings) || build_sincedb_base_from_env + @sincedb_path = build_random_sincedb_filename(base_sincedb_path) + @logger.info('No sincedb_path set, generating one based on the "path" setting', :sincedb_path => @sincedb_path.to_s, :path => @path) + else + @sincedb_path = Pathname.new(@sincedb_path) + if @sincedb_path.directory? + raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"") end end - # This section is going to be deprecated eventually, as path.data will be - # the default, not an environment variable (SINCEDB_DIR or HOME) - if @sincedb_path.nil? # If it is _still_ nil... - if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil? - @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \ - "to keep track of the files I'm watching. Either set " \ - "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \ - "in your Logstash config for the file input with " \ - "path '#{@path.inspect}'") - raise # TODO(sissel): HOW DO I FAIL PROPERLY YO - end - - #pick SINCEDB_DIR if available, otherwise use HOME - sincedb_dir = ENV["SINCEDB_DIR"] || ENV["HOME"] - - # Join by ',' to make it easy for folks to know their own sincedb - # generated path (vs, say, inspecting the @path array) - @sincedb_path = File.join(sincedb_dir, ".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) - - # Migrate any old .sincedb to the new file (this is for version <=1.1.1 compatibility) - old_sincedb = File.join(sincedb_dir, ".sincedb") - if File.exists?(old_sincedb) - @logger.debug("Renaming old ~/.sincedb to new one", :old => old_sincedb, - :new => @sincedb_path) - File.rename(old_sincedb, @sincedb_path) + @filewatch_config[:sincedb_path] = @sincedb_path + + @filewatch_config[:start_new_files_at] = @start_position.to_sym + + if @file_completed_action.include?('log') + if @file_completed_log_path.nil? + raise ArgumentError.new('The "file_completed_log_path" setting must be provided when the "file_completed_action" is set to "log" or "log_and_delete"') + else + @file_completed_log_path = Pathname.new(@file_completed_log_path) + unless @file_completed_log_path.exist? + begin + FileUtils.touch(@file_completed_log_path) + rescue + raise ArgumentError.new("The \"file_completed_log_path\" file can't be created: #{@file_completed_log_path}") + end + end end - - @logger.info("No sincedb_path set, generating one based on the file path", - :sincedb_path => @sincedb_path, :path => @path) - end - - if File.directory?(@sincedb_path) - raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"") end - @tail_config[:sincedb_path] = @sincedb_path - - if @start_position == "beginning" - @tail_config[:start_new_files_at] = :beginning + if tail_mode? + @watcher_class = FileWatch::ObservingTail + else + @watcher_class = FileWatch::ObservingRead + if @file_completed_action.include?('log') + @completed_file_handlers << LogCompletedFileHandler.new(@file_completed_log_path) + end + if @file_completed_action.include?('delete') + @completed_file_handlers << DeleteCompletedFileHandler.new + end end - @codec = LogStash::Codecs::IdentityMapCodec.new(@codec) end # def register - class ListenerTail - # use attr_reader to define noop methods - attr_reader :input, :path, :data - attr_reader :deleted, :created, :error, :eof - - # construct with upstream state - def initialize(path, input) - @path, @input = path, input - end - - def timed_out - input.codec.evict(path) - end - - def accept(data) - # and push transient data filled dup listener downstream - input.log_line_received(path, data) - input.codec.accept(dup_adding_state(data)) - end - - def process_event(event) - event.set("[@metadata][path]", path) - event.set("path", path) if !event.include?("path") - input.post_process_this(event) - end - - def add_state(data) - @data = data - self - end - - private - - # duplicate and add state for downstream - def dup_adding_state(line) - self.class.new(path, input).add_state(line) - end - end - - class FlushableListener < ListenerTail - attr_writer :path - end - def listener_for(path) # path is the identity - ListenerTail.new(path, self) + FileListener.new(path, self) end - def begin_tailing + def start_processing # if the pipeline restarts this input, # make sure previous files are closed stop - # use observer listener api - @tail = FileWatch::Tail.new_observing(@tail_config) - @tail.logger = @logger - @path.each { |path| @tail.tail(path) } + @watcher = @watcher_class.new(@filewatch_config) + @path.each { |path| @watcher.watch_this(path) } end def run(queue) - begin_tailing + start_processing @queue = queue - @tail.subscribe(self) + @watcher.subscribe(self) # halts here until quit is called exit_flush end # def run def post_process_this(event) event.set("[@metadata][host]", @host) - event.set("host", @host) if !event.include?("host") + event.set("host", @host) unless event.include?("host") decorate(event) @queue << event end + def handle_deletable_path(path) + return if tail_mode? + return if @completed_file_handlers.empty? + @completed_file_handlers.each { |handler| handler.handle(path) } + end + def log_line_received(path, line) - return if !@logger.debug? + return unless @logger.debug? @logger.debug("Received line", :path => path, :text => line) end @@ -322,14 +342,50 @@ def stop # in filewatch >= 0.6.7, quit will closes and forget all files # but it will write their last read positions to since_db # beforehand - if @tail + if @watcher @codec.close - @tail.quit + @watcher.quit end end private + def build_sincedb_base_from_settings(settings) + logstash_data_path = settings.get_value("path.data") + Pathname.new(logstash_data_path).join("plugins", "inputs", "file").tap do |path| + # Ensure that the filepath exists before writing, since it's deeply nested. + path.mkpath + end + end + + def build_sincedb_base_from_env + # This section is going to be deprecated eventually, as path.data will be + # the default, not an environment variable (SINCEDB_DIR or LOGSTASH_HOME) + if ENV["SINCEDB_DIR"].nil? && ENV["LOGSTASH_HOME"].nil? + @logger.error("No SINCEDB_DIR or LOGSTASH_HOME environment variable set, I don't know where " \ + "to keep track of the files I'm watching. Either set " \ + "LOGSTASH_HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \ + "in your Logstash config for the file input with " \ + "path '#{@path.inspect}'") + raise ArgumentError.new('The "sincedb_path" setting was not given and the environment variables "SINCEDB_DIR" or "LOGSTASH_HOME" are not set so we cannot build a file path for the sincedb') + end + Pathname.new(ENV["SINCEDB_DIR"] || ENV["LOGSTASH_HOME"]) + end + + def build_random_sincedb_filename(pathname) + # Join by ',' to make it easy for folks to know their own sincedb + # generated path (vs, say, inspecting the @path array) + pathname.join(".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) + end + + def tail_mode? + @mode == "tail" + end + + def read_mode? + !tail_mode? + end + def exit_flush listener = FlushableListener.new("none", self) if @codec.identity_count.zero? @@ -345,4 +401,4 @@ def exit_flush @codec.flush_mapped(listener) end end -end # class LogStash::Inputs::File +end end end# class LogStash::Inputs::File diff --git a/lib/logstash/inputs/file_listener.rb b/lib/logstash/inputs/file_listener.rb new file mode 100644 index 0000000..71ddd42 --- /dev/null +++ b/lib/logstash/inputs/file_listener.rb @@ -0,0 +1,61 @@ +# encoding: utf-8 + +module LogStash module Inputs + # As and when a new WatchedFile is processed FileWatch asks for an instance of this class for the + # file path of that WatchedFile. All subsequent callbacks are sent via this listener instance. + # The file is essentially a stream and the path is the identity of that stream. + class FileListener + attr_reader :input, :path, :data + # construct with link back to the input plugin instance. + def initialize(path, input) + @path, @input = path, input + @data = nil + end + + def opened + end + + def eof + end + + def error + end + + def timed_out + input.codec.evict(path) + end + + def deleted + input.codec.evict(path) + input.handle_deletable_path(path) + end + + def accept(data) + # and push transient data filled dup listener downstream + input.log_line_received(path, data) + input.codec.accept(dup_adding_state(data)) + end + + def process_event(event) + event.set("[@metadata][path]", path) + event.set("path", path) unless event.include?("path") + input.post_process_this(event) + end + + def add_state(data) + @data = data + self + end + + private + + # duplicate and add state for downstream + def dup_adding_state(line) + self.class.new(path, input).add_state(line) + end + end + + class FlushableListener < FileListener + attr_writer :path + end +end end diff --git a/lib/logstash/inputs/log_completed_file_handler.rb b/lib/logstash/inputs/log_completed_file_handler.rb new file mode 100644 index 0000000..5210ac7 --- /dev/null +++ b/lib/logstash/inputs/log_completed_file_handler.rb @@ -0,0 +1,13 @@ +# encoding: utf-8 + +module LogStash module Inputs + class LogCompletedFileHandler + def initialize(log_completed_file_path) + @log_completed_file_path = Pathname.new(log_completed_file_path) + end + + def handle(path) + @log_completed_file_path.open("a") { |fd| fd.puts(path) } + end + end +end end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 4316146..274c1cb 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,8 +1,8 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.0.5' - s.licenses = ['Apache License (2.0)'] + s.version = '5.0.0' + s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elastic"] @@ -11,7 +11,7 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] # Files - s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] + s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "JAR_VERSION", "docs/**/*"] # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) @@ -24,11 +24,12 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'logstash-codec-plain' s.add_runtime_dependency 'addressable' - s.add_runtime_dependency 'filewatch', ['>= 0.8.1', '~> 0.8'] s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0'] s.add_development_dependency 'stud', ['~> 0.0.19'] s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'logstash-codec-json' s.add_development_dependency 'rspec-sequencing' + s.add_development_dependency "rspec-wait" + s.add_development_dependency 'timecop' end diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..73158f0 --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'filewatch' \ No newline at end of file diff --git a/spec/filewatch/buftok_spec.rb b/spec/filewatch/buftok_spec.rb new file mode 100644 index 0000000..17e7532 --- /dev/null +++ b/spec/filewatch/buftok_spec.rb @@ -0,0 +1,24 @@ +require_relative 'spec_helper' + +describe FileWatch::BufferedTokenizer do + + context "when using the default delimiter" do + it "splits the lines correctly" do + expect(subject.extract("hello\nworld\n")).to eq ["hello", "world"] + end + + it "holds partial lines back until a token is found" do + buffer = described_class.new + expect(buffer.extract("hello\nwor")).to eq ["hello"] + expect(buffer.extract("ld\n")).to eq ["world"] + end + end + + context "when passing a custom delimiter" do + subject { FileWatch::BufferedTokenizer.new("\r\n") } + + it "splits the lines correctly" do + expect(subject.extract("hello\r\nworld\r\n")).to eq ["hello", "world"] + end + end +end diff --git a/spec/filewatch/reading_spec.rb b/spec/filewatch/reading_spec.rb new file mode 100644 index 0000000..bb574d3 --- /dev/null +++ b/spec/filewatch/reading_spec.rb @@ -0,0 +1,128 @@ + +require 'stud/temporary' +require_relative 'spec_helper' +require 'filewatch/observing_read' + +module FileWatch + describe Watch do + before(:all) do + @thread_abort = Thread.abort_on_exception + Thread.abort_on_exception = true + end + + after(:all) do + Thread.abort_on_exception = @thread_abort + end + + let(:directory) { Stud::Temporary.directory } + let(:watch_dir) { ::File.join(directory, "*.log") } + let(:file_path) { ::File.join(directory, "1.log") } + let(:sincedb_path) { ::File.join(Stud::Temporary.directory, "reading.sdb") } + let(:stat_interval) { 0.1 } + let(:discover_interval) { 4 } + let(:start_new_files_at) { :end } # should be irrelevant for read mode + let(:opts) do + { + :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, + :delimiter => "\n", :discover_interval => discover_interval, + :ignore_older => 3600, :sincedb_path => sincedb_path + } + end + let(:observer) { TestObserver.new } + let(:reading) { ObservingRead.new(opts) } + let(:actions) do + RSpec::Sequencing.run_after(0.45, "quit after a short time") do + reading.quit + end + end + + after do + FileUtils.rm_rf(directory) unless directory =~ /fixture/ + end + + context "when watching a directory with files" do + let(:directory) { Stud::Temporary.directory } + let(:watch_dir) { ::File.join(directory, "*.log") } + let(:file_path) { ::File.join(directory, "1.log") } + + it "the file is read" do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + actions.activate + reading.watch_this(watch_dir) + reading.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + end + end + + context "when watching a directory with files using striped reading" do + let(:directory) { Stud::Temporary.directory } + let(:watch_dir) { ::File.join(directory, "*.log") } + let(:file_path1) { ::File.join(directory, "1.log") } + let(:file_path2) { ::File.join(directory, "2.log") } + # use a chunk size that does not align with the line boundaries + let(:opts) { super.merge(:file_chunk_size => 10, :file_chunk_count => 1)} + let(:lines) { [] } + let(:observer) { TestObserver.new(lines) } + + it "the files are read seemingly in parallel" do + File.open(file_path1, "w") { |file| file.write("string1\nstring2\n") } + File.open(file_path2, "w") { |file| file.write("stringA\nstringB\n") } + actions.activate + reading.watch_this(watch_dir) + reading.subscribe(observer) + if lines.first == "stringA" + expect(lines).to eq(%w(stringA string1 stringB string2)) + else + expect(lines).to eq(%w(string1 stringA string2 stringB)) + end + end + end + + describe "reading fixtures" do + let(:directory) { FIXTURE_DIR } + + context "for an uncompressed file" do + let(:watch_dir) { ::File.join(directory, "unc*.log") } + let(:file_path) { ::File.join(directory, 'uncompressed.log') } + + it "the file is read" do + FileWatch.make_fixture_current(file_path) + actions.activate + reading.watch_this(watch_dir) + reading.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(observer.listener_for(file_path).lines.size).to eq(2) + end + end + + context "for another uncompressed file" do + let(:watch_dir) { ::File.join(directory, "invalid*.log") } + let(:file_path) { ::File.join(directory, 'invalid_utf8.gbk.log') } + + it "the file is read" do + FileWatch.make_fixture_current(file_path) + actions.activate + reading.watch_this(watch_dir) + reading.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(observer.listener_for(file_path).lines.size).to eq(2) + end + end + + context "for a compressed file" do + let(:watch_dir) { ::File.join(directory, "compressed.*.gz") } + let(:file_path) { ::File.join(directory, 'compressed.log.gz') } + + it "the file is read" do + FileWatch.make_fixture_current(file_path) + actions.activate + reading.watch_this(watch_dir) + reading.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(observer.listener_for(file_path).lines.size).to eq(2) + end + end + end + end +end diff --git a/spec/filewatch/sincedb_record_serializer_spec.rb b/spec/filewatch/sincedb_record_serializer_spec.rb new file mode 100644 index 0000000..7beaba9 --- /dev/null +++ b/spec/filewatch/sincedb_record_serializer_spec.rb @@ -0,0 +1,71 @@ +# encoding: utf-8 +require_relative 'spec_helper' +require 'filewatch/settings' +require 'filewatch/sincedb_record_serializer' + +module FileWatch + describe SincedbRecordSerializer do + let(:opts) { Hash.new } + let(:io) { StringIO.new } + let(:db) { Hash.new } + + subject { described_class.new(Settings.days_to_seconds(14)) } + + context "deserialize from IO" do + it 'reads V1 records' do + io.write("5391299 1 4 12\n") + subject.deserialize(io) do |inode_struct, sincedb_value| + expect(inode_struct.inode).to eq("5391299") + expect(inode_struct.maj).to eq(1) + expect(inode_struct.min).to eq(4) + expect(sincedb_value.position).to eq(12) + end + end + + it 'reads V2 records from an IO object' do + now = Time.now.to_f + io.write("5391299 1 4 12 #{now} /a/path/to/1.log\n") + subject.deserialize(io) do |inode_struct, sincedb_value| + expect(inode_struct.inode).to eq("5391299") + expect(inode_struct.maj).to eq(1) + expect(inode_struct.min).to eq(4) + expect(sincedb_value.position).to eq(12) + expect(sincedb_value.last_changed_at).to eq(now) + expect(sincedb_value.path_in_sincedb).to eq("/a/path/to/1.log") + end + end + end + + context "serialize to IO" do + it "writes db entries" do + now = Time.now.to_f + inode_struct = InodeStruct.new("42424242", 2, 5) + sincedb_value = SincedbValue.new(42, now) + db[inode_struct] = sincedb_value + subject.serialize(db, io) + expect(io.string).to eq("42424242 2 5 42 #{now}\n") + end + + it "does not write expired db entries to an IO object" do + twelve_days_ago = Time.now.to_f - (12.0*24*3600) + sixteen_days_ago = twelve_days_ago - (4.0*24*3600) + db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, twelve_days_ago) + db[InodeStruct.new("18181818", 1, 6)] = SincedbValue.new(99, sixteen_days_ago) + subject.serialize(db, io) + expect(io.string).to eq("42424242 2 5 42 #{twelve_days_ago}\n") + end + end + + context "given a non default `sincedb_clean_after`" do + it "does not write expired db entries to an IO object" do + subject.update_sincedb_value_expiry_from_days(2) + one_day_ago = Time.now.to_f - (1.0*24*3600) + three_days_ago = one_day_ago - (2.0*24*3600) + db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago) + db[InodeStruct.new("18181818", 1, 6)] = SincedbValue.new(99, three_days_ago) + subject.serialize(db, io) + expect(io.string).to eq("42424242 2 5 42 #{one_day_ago}\n") + end + end + end +end \ No newline at end of file diff --git a/spec/filewatch/spec_helper.rb b/spec/filewatch/spec_helper.rb new file mode 100644 index 0000000..578cc46 --- /dev/null +++ b/spec/filewatch/spec_helper.rb @@ -0,0 +1,120 @@ +require "rspec_sequencing" +require 'rspec/wait' +require "logstash/devutils/rspec/spec_helper" +require "timecop" + +def formatted_puts(text) + cfg = RSpec.configuration + return unless cfg.formatters.first.is_a?( + RSpec::Core::Formatters::DocumentationFormatter) + txt = cfg.format_docstrings_block.call(text) + cfg.output_stream.puts " #{txt}" +end + +unless RSpec::Matchers.method_defined?(:receive_call_and_args) + RSpec::Matchers.define(:receive_call_and_args) do |m, args| + match do |actual| + actual.trace_for(m) == args + end + + failure_message do + "Expecting method #{m} to receive: #{args} but got: #{actual.trace_for(m)}" + end + end +end + +require 'filewatch/bootstrap' + +module FileWatch + + FIXTURE_DIR = File.join('spec', 'fixtures') + + def self.make_file_older(path, seconds) + time = Time.now.to_f - seconds + ::File.utime(time, time, path) + end + + def self.make_fixture_current(path, time = Time.now) + ::File.utime(time, time, path) + end + + class TracerBase + def initialize + @tracer = [] + end + + def trace_for(symbol) + params = @tracer.map {|k,v| k == symbol ? v : nil}.compact + params.empty? ? false : params + end + + def clear + @tracer.clear + end + end + + module NullCallable + def self.call + end + end + + class TestObserver + class Listener + attr_reader :path, :lines, :calls + + def initialize(path) + @path = path + @lines = [] + @calls = [] + end + + def add_lines(lines) + @lines = lines + self + end + + def accept(line) + @lines << line + @calls << :accept + end + + def deleted + @calls << :delete + end + + def opened + @calls << :open + end + + def error + @calls << :error + end + + def eof + @calls << :eof + end + + def timed_out + @calls << :timed_out + end + end + + attr_reader :listeners + + def initialize(combined_lines = nil) + listener_proc = if combined_lines.nil? + lambda{|k| Listener.new(k) } + else + lambda{|k| Listener.new(k).add_lines(combined_lines) } + end + @listeners = Hash.new {|hash, key| hash[key] = listener_proc.call(key) } + end + + def listener_for(path) + @listeners[path] + end + + def clear + @listeners.clear; end + end +end diff --git a/spec/filewatch/tailing_spec.rb b/spec/filewatch/tailing_spec.rb new file mode 100644 index 0000000..11f50eb --- /dev/null +++ b/spec/filewatch/tailing_spec.rb @@ -0,0 +1,440 @@ + +require 'stud/temporary' +require_relative 'spec_helper' +require 'filewatch/observing_tail' + +LogStash::Logging::Logger::configure_logging("WARN") +# LogStash::Logging::Logger::configure_logging("DEBUG") + +module FileWatch + describe Watch do + before(:all) do + @thread_abort = Thread.abort_on_exception + Thread.abort_on_exception = true + end + + after(:all) do + Thread.abort_on_exception = @thread_abort + end + + let(:directory) { Stud::Temporary.directory } + let(:watch_dir) { ::File.join(directory, "*.log") } + let(:file_path) { ::File.join(directory, "1.log") } + let(:max) { 4095 } + let(:stat_interval) { 0.1 } + let(:discover_interval) { 4 } + let(:start_new_files_at) { :beginning } + let(:sincedb_path) { ::File.join(directory, "tailing.sdb") } + let(:opts) do + { + :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_active => max, + :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path + } + end + let(:observer) { TestObserver.new } + let(:tailing) { ObservingTail.new(opts) } + + after do + FileUtils.rm_rf(directory) + end + + describe "max open files (set to 1)" do + let(:max) { 1 } + let(:file_path2) { File.join(directory, "2.log") } + let(:wait_before_quit) { 0.15 } + let(:stat_interval) { 0.01 } + let(:discover_interval) { 4 } + let(:actions) do + RSpec::Sequencing + .run_after(wait_before_quit, "quit after a short time") do + tailing.quit + end + end + + before do + ENV["FILEWATCH_MAX_FILES_WARN_INTERVAL"] = "0" + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + File.open(file_path2, "wb") { |file| file.write("lineA\nlineB\n") } + end + + context "when max_active is 1" do + + it "without close_older set, opens only 1 file" do + actions.activate + tailing.watch_this(watch_dir) + tailing.subscribe(observer) + expect(tailing.settings.max_active).to eq(max) + file1_calls = observer.listener_for(file_path).calls + file2_calls = observer.listener_for(file_path2).calls + # file glob order is OS dependent + if file1_calls.empty? + expect(observer.listener_for(file_path2).lines).to eq(["lineA", "lineB"]) + expect(file2_calls).to eq([:open, :accept, :accept]) + else + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + expect(file1_calls).to eq([:open, :accept, :accept]) + expect(file2_calls).to be_empty + end + end + end + + context "when close_older is set" do + let(:wait_before_quit) { 0.8 } + let(:opts) { super.merge(:close_older => 0.2, :max_active => 1, :stat_interval => 0.1) } + it "opens both files" do + actions.activate + tailing.watch_this(watch_dir) + tailing.subscribe(observer) + expect(tailing.settings.max_active).to eq(1) + filelistener_1 = observer.listener_for(file_path) + filelistener_2 = observer.listener_for(file_path2) + expect(filelistener_2.calls).to eq([:open, :accept, :accept, :timed_out]) + expect(filelistener_2.lines).to eq(["lineA", "lineB"]) + expect(filelistener_1.calls).to eq([:open, :accept, :accept, :timed_out]) + expect(filelistener_1.lines).to eq(["line1", "line2"]) + end + end + end + + context "when watching a directory with files" do + let(:start_new_files_at) { :beginning } + let(:actions) do + RSpec::Sequencing.run_after(0.45, "quit after a short time") do + tailing.quit + end + end + + it "the file is read" do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + actions.activate + tailing.watch_this(watch_dir) + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept]) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + end + end + + context "when watching a directory without files and one is added" do + let(:start_new_files_at) { :beginning } + before do + tailing.watch_this(watch_dir) + RSpec::Sequencing + .run_after(0.25, "create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then_after(0.45, "quit after a short time") do + tailing.quit + end + end + + it "the file is read" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept]) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + end + end + + describe "given a previously discovered file" do + # these tests rely on the fact that the 'filepath' does not exist on disk + # it simulates that the user deleted the file + # so when a stat is taken on the file an error is raised + let(:quit_after) { 0.2 } + let(:stat) { double("stat", :size => 100, :ctime => Time.now, :mtime => Time.now, :ino => 234567, :dev_major => 3, :dev_minor => 2) } + let(:watched_file) { WatchedFile.new(file_path, stat, tailing.settings) } + + before do + tailing.watch.watched_files_collection.add(watched_file) + watched_file.initial_completed + end + + context "when a close operation occurs" do + before { watched_file.close } + it "is removed from the watched_files_collection" do + expect(tailing.watch.watched_files_collection).not_to be_empty + RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit } + tailing.subscribe(observer) + expect(tailing.watch.watched_files_collection).to be_empty + expect(observer.listener_for(file_path).calls).to eq([:delete]) + end + end + + context "an ignore operation occurs" do + before { watched_file.ignore } + it "is removed from the watched_files_collection" do + RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit } + tailing.subscribe(observer) + expect(tailing.watch.watched_files_collection).to be_empty + expect(observer.listener_for(file_path).calls).to eq([:delete]) + end + end + + context "when subscribed and a watched file is no longer readable" do + before { watched_file.watch } + it "is removed from the watched_files_collection" do + RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit } + tailing.subscribe(observer) + expect(tailing.watch.watched_files_collection).to be_empty + expect(observer.listener_for(file_path).calls).to eq([:delete]) + end + end + + context "when subscribed and an active file is no longer readable" do + before { watched_file.activate } + it "is removed from the watched_files_collection" do + RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit } + tailing.subscribe(observer) + expect(tailing.watch.watched_files_collection).to be_empty + expect(observer.listener_for(file_path).calls).to eq([:delete]) + end + end + end + + context "when a processed file shrinks" do + let(:discover_interval) { 100 } + before do + RSpec::Sequencing + .run("create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\nline3\nline4\n") } + end + .then_after(0.25, "start watching after files are written") do + tailing.watch_this(watch_dir) + end + .then_after(0.25, "truncate file and write new content") do + File.truncate(file_path, 0) + File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") } + end + .then_after(0.25, "quit after a short time") do + tailing.quit + end + end + + it "new changes to the shrunk file are read from the beginning" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :accept, :accept, :accept, :accept]) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4", "lineA", "lineB"]) + end + end + + context "when watching a directory with files and a file is renamed to not match glob" do + let(:new_file_path) { file_path + ".old" } + before do + RSpec::Sequencing + .run("create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then_after(0.25, "start watching after files are written") do + tailing.watch_this(watch_dir) + end + .then_after(0.55, "rename file") do + FileUtils.mv(file_path, new_file_path) + end + .then_after(0.55, "then write to renamed file") do + File.open(new_file_path, "ab") { |file| file.write("line3\nline4\n") } + end + .then_after(0.45, "quit after a short time") do + tailing.quit + end + end + + it "changes to the renamed file are not read" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :delete]) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + expect(observer.listener_for(new_file_path).calls).to eq([]) + expect(observer.listener_for(new_file_path).lines).to eq([]) + end + end + + context "when watching a directory with files and a file is renamed to match glob" do + let(:new_file_path) { file_path + "2.log" } + let(:opts) { super.merge(:close_older => 0) } + before do + RSpec::Sequencing + .run("create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then_after(0.15, "start watching after files are written") do + tailing.watch_this(watch_dir) + end + .then_after(0.25, "rename file") do + FileUtils.mv(file_path, new_file_path) + end + .then("then write to renamed file") do + File.open(new_file_path, "ab") { |file| file.write("line3\nline4\n") } + end + .then_after(0.55, "quit after a short time") do + tailing.quit + end + end + + it "the first set of lines are not re-read" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out, :delete]) + expect(observer.listener_for(new_file_path).lines).to eq(["line3", "line4"]) + expect(observer.listener_for(new_file_path).calls).to eq([:open, :accept, :accept, :timed_out]) + end + end + + context "when watching a directory with files and data is appended" do + before do + RSpec::Sequencing + .run("create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then_after(0.25, "start watching after file is written") do + tailing.watch_this(watch_dir) + end + .then_after(0.45, "append more lines to the file") do + File.open(file_path, "ab") { |file| file.write("line3\nline4\n") } + end + .then_after(0.45, "quit after a short time") do + tailing.quit + end + end + + it "appended lines are read after an EOF" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :accept, :accept]) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4"]) + end + end + + context "when close older expiry is enabled" do + let(:opts) { super.merge(:close_older => 1) } + before do + RSpec::Sequencing + .run("create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then("start watching before file ages more than close_older") do + tailing.watch_this(watch_dir) + end + .then_after(2.1, "quit after allowing time to close the file") do + tailing.quit + end + end + + it "lines are read and the file times out" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out]) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + end + end + + context "when close older expiry is enabled and after timeout the file is appended-to" do + let(:opts) { super.merge(:close_older => 1) } + before do + RSpec::Sequencing + .run("create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then("start watching before file ages more than close_older") do + tailing.watch_this(watch_dir) + end + .then_after(2.1, "append more lines to file after file ages more than close_older") do + File.open(file_path, "ab") { |file| file.write("line3\nline4\n") } + end + .then_after(2.1, "quit after allowing time to close the file") do + tailing.quit + end + end + + it "all lines are read" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out, :open, :accept, :accept, :timed_out]) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4"]) + end + end + + context "when ignore older expiry is enabled and all files are already expired" do + let(:opts) { super.merge(:ignore_older => 1) } + before do + RSpec::Sequencing + .run("create file older than ignore_older and watch") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + FileWatch.make_file_older(file_path, 15) + tailing.watch_this(watch_dir) + end + .then_after(1.1, "quit") do + tailing.quit + end + end + + it "no files are read" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([]) + expect(observer.listener_for(file_path).lines).to eq([]) + end + end + + context "when ignore_older is less than close_older and all files are not expired" do + let(:opts) { super.merge(:ignore_older => 1, :close_older => 1.5) } + before do + RSpec::Sequencing + .run("create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then("start watching before file age reaches ignore_older") do + tailing.watch_this(watch_dir) + end + .then_after(1.75, "quit after allowing time to close the file") do + tailing.quit + end + end + + it "reads lines normally" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out]) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + end + end + + context "when ignore_older is less than close_older and all files are expired" do + let(:opts) { super.merge(:ignore_older => 10, :close_older => 1) } + before do + RSpec::Sequencing + .run("create file older than ignore_older and watch") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + FileWatch.make_file_older(file_path, 15) + tailing.watch_this(watch_dir) + end + .then_after(1.5, "quit after allowing time to check the files") do + tailing.quit + end + end + + it "no files are read" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([]) + expect(observer.listener_for(file_path).lines).to eq([]) + end + end + + context "when ignore older and close older expiry is enabled and after timeout the file is appended-to" do + let(:opts) { super.merge(:ignore_older => 20, :close_older => 1) } + before do + RSpec::Sequencing + .run("create file older than ignore_older and watch") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + FileWatch.make_file_older(file_path, 25) + tailing.watch_this(watch_dir) + end + .then_after(0.15, "append more lines to file after file ages more than ignore_older") do + File.open(file_path, "ab") { |file| file.write("line3\nline4\n") } + end + .then_after(1.25, "quit after allowing time to close the file") do + tailing.quit + end + end + + it "reads the added lines only" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).lines).to eq(["line3", "line4"]) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out]) + end + end + end +end + + diff --git a/spec/filewatch/watched_file_spec.rb b/spec/filewatch/watched_file_spec.rb new file mode 100644 index 0000000..69181e4 --- /dev/null +++ b/spec/filewatch/watched_file_spec.rb @@ -0,0 +1,38 @@ +require 'stud/temporary' +require_relative 'spec_helper' + +module FileWatch + describe WatchedFile do + let(:pathname) { Pathname.new(__FILE__) } + + context 'Given two instances of the same file' do + it 'their sincedb_keys should equate' do + wf_key1 = WatchedFile.new(pathname, pathname.stat, Settings.new).sincedb_key + hash_db = { wf_key1 => 42 } + wf_key2 = WatchedFile.new(pathname, pathname.stat, Settings.new).sincedb_key + expect(wf_key1).to eq(wf_key2) + expect(wf_key1).to eql(wf_key2) + expect(wf_key1.hash).to eq(wf_key2.hash) + expect(hash_db[wf_key2]).to eq(42) + end + end + + context 'Given a barrage of state changes' do + it 'only the previous N state changes are remembered' do + watched_file = WatchedFile.new(pathname, pathname.stat, Settings.new) + watched_file.ignore + watched_file.watch + watched_file.activate + watched_file.watch + watched_file.close + watched_file.watch + watched_file.activate + watched_file.unwatch + watched_file.activate + watched_file.close + expect(watched_file.closed?).to be_truthy + expect(watched_file.recent_states).to eq([:watched, :active, :watched, :closed, :watched, :active, :unwatched, :active]) + end + end + end +end diff --git a/spec/filewatch/watched_files_collection_spec.rb b/spec/filewatch/watched_files_collection_spec.rb new file mode 100644 index 0000000..c8fb491 --- /dev/null +++ b/spec/filewatch/watched_files_collection_spec.rb @@ -0,0 +1,73 @@ +require_relative 'spec_helper' + +module FileWatch + describe WatchedFilesCollection do + let(:time) { Time.now } + let(:stat1) { double("stat1", :size => 98, :ctime => time - 30, :mtime => time - 30, :ino => 234567, :dev_major => 3, :dev_minor => 2) } + let(:stat2) { double("stat2", :size => 99, :ctime => time - 20, :mtime => time - 20, :ino => 234568, :dev_major => 3, :dev_minor => 2) } + let(:stat3) { double("stat3", :size => 100, :ctime => time, :mtime => time, :ino => 234569, :dev_major => 3, :dev_minor => 2) } + let(:wf1) { WatchedFile.new("/var/log/z.log", stat1, Settings.new) } + let(:wf2) { WatchedFile.new("/var/log/m.log", stat2, Settings.new) } + let(:wf3) { WatchedFile.new("/var/log/a.log", stat3, Settings.new) } + + context "sort by last_modified in ascending order" do + let(:sort_by) { "last_modified" } + let(:sort_direction) { "asc" } + + it "sorts earliest modified first" do + collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction)) + collection.add(wf2) + expect(collection.values).to eq([wf2]) + collection.add(wf3) + expect(collection.values).to eq([wf2, wf3]) + collection.add(wf1) + expect(collection.values).to eq([wf1, wf2, wf3]) + end + end + + context "sort by path in ascending order" do + let(:sort_by) { "path" } + let(:sort_direction) { "asc" } + + it "sorts path A-Z" do + collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction)) + collection.add(wf2) + expect(collection.values).to eq([wf2]) + collection.add(wf1) + expect(collection.values).to eq([wf2, wf1]) + collection.add(wf3) + expect(collection.values).to eq([wf3, wf2, wf1]) + end + end + + context "sort by last_modified in descending order" do + let(:sort_by) { "last_modified" } + let(:sort_direction) { "desc" } + + it "sorts latest modified first" do + collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction)) + collection.add(wf2) + expect(collection.values).to eq([wf2]) + collection.add(wf1) + expect(collection.values).to eq([wf2, wf1]) + collection.add(wf3) + expect(collection.values).to eq([wf3, wf2, wf1]) + end + end + + context "sort by path in descending order" do + let(:sort_by) { "path" } + let(:sort_direction) { "desc" } + + it "sorts path Z-A" do + collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction)) + collection.add(wf2) + expect(collection.values).to eq([wf2]) + collection.add(wf1) + expect(collection.values).to eq([wf1, wf2]) + collection.add(wf3) + expect(collection.values).to eq([wf1, wf2, wf3]) + end + end + end +end diff --git a/spec/filewatch/winhelper_spec.rb b/spec/filewatch/winhelper_spec.rb new file mode 100644 index 0000000..f65e564 --- /dev/null +++ b/spec/filewatch/winhelper_spec.rb @@ -0,0 +1,22 @@ +require "stud/temporary" +require "fileutils" + +if Gem.win_platform? + require "lib/filewatch/winhelper" + + describe Winhelper do + let(:path) { Stud::Temporary.file.path } + + after do + FileUtils.rm_rf(path) + end + + it "return a unique file identifier" do + volume_serial, file_index_low, file_index_high = Winhelper.GetWindowsUniqueFileIdentifier(path).split("").map(&:to_i) + + expect(volume_serial).not_to eq(0) + expect(file_index_low).not_to eq(0) + expect(file_index_high).not_to eq(0) + end + end +end diff --git a/spec/fixtures/compressed.log.gz b/spec/fixtures/compressed.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..e8e0cb9e88053c5316ed9ebf445baaf7341cc7db GIT binary patch literal 303 zcmV+~0nq**iwFqt<MY;R`(<&d#b#2^$!`~8YES|tQX+=a|! z91ANQEo^K_c$lmtun9>vR`&j-Un1x@I^+I<1-u9MT$pz$COE-aLTD64G%GGhL9uTi zuCH=m2&V`mib%fmr`q&A)Gm;i1o{5PZMY;R`(<&d#b#2^$!`~8YES|tQX+=a|! z91ANQEo^K_c$lmtun9>vR`&j-Un1x@I^+I<1-u9MT$pz$COE-aLTD64G%GGhL9uTi zuCH=m2&V`mib%fmr`q&A)Gm;i1o{5PZ "blah" + path => "#{tmpfile_path}" + sincedb_path => "#{sincedb_path}" + delimiter => "#{FILE_DELIMITER}" + mode => "read" + file_completed_action => "delete" + } + } + CONFIG + + File.open(tmpfile_path, "a") do |fd| + fd.puts("hello") + fd.puts("world") + fd.fsync + end + + events = input(conf) do |pipeline, queue| + 2.times.collect { queue.pop } + end + + expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world") + expect(File.exist?(tmpfile_path)).to be_falsey + end + end + + describe "reading fixtures" do + let(:fixture_dir) { Pathname.new(FileInput::FIXTURE_DIR).expand_path } + + context "for a file without a final newline character" do + let(:file_path) { fixture_dir.join('no-final-newline.log') } + + it "the file is read and the path is logged to the `file_completed_log_path` file" do + tmpfile_path = fixture_dir.join("no-f*.log") + sincedb_path = Stud::Temporary.pathname + FileInput.make_fixture_current(file_path.to_path) + log_completed_path = Stud::Temporary.pathname + + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{tmpfile_path}" + sincedb_path => "#{sincedb_path}" + delimiter => "#{FILE_DELIMITER}" + mode => "read" + file_completed_action => "log" + file_completed_log_path => "#{log_completed_path}" + } + } + CONFIG + + events = input(conf) do |pipeline, queue| + 2.times.collect { queue.pop } + end + + expect(events[0].get("message")).to start_with("2010-03-12 23:51") + expect(events[1].get("message")).to start_with("2010-03-12 23:51") + expect(IO.read(log_completed_path)).to eq(file_path.to_s + "\n") + end + + end + + context "for an uncompressed file" do + let(:file_path) { fixture_dir.join('uncompressed.log') } + + it "the file is read and the path is logged to the `file_completed_log_path` file" do + tmpfile_path = fixture_dir.join("unc*.log") + sincedb_path = Stud::Temporary.pathname + FileInput.make_fixture_current(file_path.to_path) + log_completed_path = Stud::Temporary.pathname + + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{tmpfile_path}" + sincedb_path => "#{sincedb_path}" + delimiter => "#{FILE_DELIMITER}" + mode => "read" + file_completed_action => "log" + file_completed_log_path => "#{log_completed_path}" + } + } + CONFIG + + events = input(conf) do |pipeline, queue| + 2.times.collect { queue.pop } + end + + expect(events[0].get("message")).to start_with("2010-03-12 23:51") + expect(events[1].get("message")).to start_with("2010-03-12 23:51") + expect(IO.read(log_completed_path)).to eq(file_path.to_s + "\n") + end + end + + context "for a compressed file" do + it "the file is read" do + tmpfile_path = fixture_dir.join("compressed.*.*") + sincedb_path = Stud::Temporary.pathname + file_path = fixture_dir.join('compressed.log.gz') + file_path2 = fixture_dir.join('compressed.log.gzip') + FileInput.make_fixture_current(file_path.to_path) + log_completed_path = Stud::Temporary.pathname + + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{tmpfile_path}" + sincedb_path => "#{sincedb_path}" + delimiter => "#{FILE_DELIMITER}" + mode => "read" + file_completed_action => "log" + file_completed_log_path => "#{log_completed_path}" + } + } + CONFIG + + events = input(conf) do |pipeline, queue| + 4.times.collect { queue.pop } + end + + expect(events[0].get("message")).to start_with("2010-03-12 23:51") + expect(events[1].get("message")).to start_with("2010-03-12 23:51") + expect(events[2].get("message")).to start_with("2010-03-12 23:51") + expect(events[3].get("message")).to start_with("2010-03-12 23:51") + logged_completions = IO.read(log_completed_path).split + expect(logged_completions.first).to match(/compressed\.log\.(gzip|gz)$/) + expect(logged_completions.last).to match(/compressed\.log\.(gzip|gz)$/) + end + end + end +end diff --git a/spec/inputs/file_spec.rb b/spec/inputs/file_tail_spec.rb similarity index 79% rename from spec/inputs/file_spec.rb rename to spec/inputs/file_tail_spec.rb index 85df88d..369906e 100644 --- a/spec/inputs/file_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -1,14 +1,18 @@ # encoding: utf-8 -require_relative "../spec_helper" + +require "helpers/spec_helper" require "logstash/inputs/file" + require "tempfile" require "stud/temporary" require "logstash/codecs/multiline" -FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n" +# LogStash::Logging::Logger::configure_logging("DEBUG") + +TEST_FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n" describe LogStash::Inputs::File do - describe "testing with input(conf) do |pipeline, queue|" do + describe "'tail' mode testing with input(conf) do |pipeline, queue|" do it_behaves_like "an interruptible input plugin" do let(:config) do { @@ -29,7 +33,7 @@ path => "#{tmpfile_path}" start_position => "beginning" sincedb_path => "#{sincedb_path}" - delimiter => "#{FILE_DELIMITER}" + delimiter => "#{TEST_FILE_DELIMITER}" } } CONFIG @@ -43,12 +47,10 @@ events = input(conf) do |pipeline, queue| 2.times.collect { queue.pop } end - - insist { events[0].get("message") } == "hello" - insist { events[1].get("message") } == "world" + expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world") end - it "should restarts at the sincedb value" do + it "should restart at the sincedb value" do tmpfile_path = Stud::Temporary.pathname sincedb_path = Stud::Temporary.pathname @@ -59,7 +61,7 @@ path => "#{tmpfile_path}" start_position => "beginning" sincedb_path => "#{sincedb_path}" - delimiter => "#{FILE_DELIMITER}" + delimiter => "#{TEST_FILE_DELIMITER}" } } CONFIG @@ -73,8 +75,7 @@ 2.times.collect { queue.pop } end - insist { events[0].get("message") } == "hello3" - insist { events[1].get("message") } == "world3" + expect(events.map{|e| e.get("message")}).to contain_exactly("hello3", "world3") File.open(tmpfile_path, "a") do |fd| fd.puts("foo") @@ -86,10 +87,8 @@ events = input(conf) do |pipeline, queue| 3.times.collect { queue.pop } end - - insist { events[0].get("message") } == "foo" - insist { events[1].get("message") } == "bar" - insist { events[2].get("message") } == "baz" + messages = events.map{|e| e.get("message")} + expect(messages).to contain_exactly("foo", "bar", "baz") end it "should not overwrite existing path and host fields" do @@ -103,7 +102,7 @@ path => "#{tmpfile_path}" start_position => "beginning" sincedb_path => "#{sincedb_path}" - delimiter => "#{FILE_DELIMITER}" + delimiter => "#{TEST_FILE_DELIMITER}" codec => "json" } } @@ -119,13 +118,15 @@ 2.times.collect { queue.pop } end - insist { events[0].get("path") } == "my_path" - insist { events[0].get("host") } == "my_host" - insist { events[0].get("[@metadata][host]") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1] - insist { events[1].get("path") } == "#{tmpfile_path}" - insist { events[1].get("host") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" - insist { events[1].get("[@metadata][host]") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + expect(events[existing_path_index].get("path")).to eq "my_path" + expect(events[existing_path_index].get("host")).to eq "my_host" + expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + + expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}" + expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" end it "should read old files" do @@ -153,14 +154,14 @@ events = input(conf) do |pipeline, queue| 2.times.collect { queue.pop } end - - insist { events[0].get("path") } == "my_path" - insist { events[0].get("host") } == "my_host" - insist { events[0].get("[@metadata][host]") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" - - insist { events[1].get("path") } == "#{tmpfile_path}" - insist { events[1].get("host") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" - insist { events[1].get("[@metadata][host]") } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1] + expect(events[existing_path_index].get("path")).to eq "my_path" + expect(events[existing_path_index].get("host")).to eq "my_host" + expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + + expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}" + expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" end context "when sincedb_path is an existing directory" do @@ -207,17 +208,17 @@ "sincedb_path" => sincedb_path, "stat_interval" => 0.1, "codec" => mlcodec, - "delimiter" => FILE_DELIMITER) - subject.register + "delimiter" => TEST_FILE_DELIMITER) end it "reads the appended data only" do + subject.register RSpec::Sequencing - .run_after(0.1, "assert zero events then append two lines") do + .run_after(0.2, "assert zero events then append two lines") do expect(events.size).to eq(0) File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") } end - .then_after(0.25, "quit") do + .then_after(0.4, "quit") do subject.stop end @@ -250,7 +251,7 @@ "stat_interval" => 0.02, "codec" => codec, "close_older" => 0.5, - "delimiter" => FILE_DELIMITER) + "delimiter" => TEST_FILE_DELIMITER) subject.register end @@ -294,7 +295,7 @@ "stat_interval" => 0.02, "codec" => codec, "ignore_older" => 1, - "delimiter" => FILE_DELIMITER) + "delimiter" => TEST_FILE_DELIMITER) subject.register Thread.new { subject.run(events) } @@ -320,7 +321,7 @@ "sincedb_path" => sincedb_path, "stat_interval" => 0.05, "codec" => mlcodec, - "delimiter" => FILE_DELIMITER) + "delimiter" => TEST_FILE_DELIMITER) subject.register end @@ -355,13 +356,13 @@ if e1_message.start_with?('line1.1-of-z') expect(e1.get("path")).to match(/z.log/) expect(e2.get("path")).to match(/A.log/) - expect(e1_message).to eq("line1.1-of-z#{FILE_DELIMITER} line1.2-of-z#{FILE_DELIMITER} line1.3-of-z") - expect(e2_message).to eq("line1.1-of-a#{FILE_DELIMITER} line1.2-of-a#{FILE_DELIMITER} line1.3-of-a") + expect(e1_message).to eq("line1.1-of-z#{TEST_FILE_DELIMITER} line1.2-of-z#{TEST_FILE_DELIMITER} line1.3-of-z") + expect(e2_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") else expect(e1.get("path")).to match(/A.log/) expect(e2.get("path")).to match(/z.log/) - expect(e1_message).to eq("line1.1-of-a#{FILE_DELIMITER} line1.2-of-a#{FILE_DELIMITER} line1.3-of-a") - expect(e2_message).to eq("line1.1-of-z#{FILE_DELIMITER} line1.2-of-z#{FILE_DELIMITER} line1.3-of-z") + expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") + expect(e2_message).to eq("line1.1-of-z#{TEST_FILE_DELIMITER} line1.2-of-z#{TEST_FILE_DELIMITER} line1.3-of-z") end end subject.run(events) @@ -385,7 +386,7 @@ e1 = events.first e1_message = e1.get("message") expect(e1["path"]).to match(/a.log/) - expect(e1_message).to eq("line1.1-of-a#{FILE_DELIMITER} line1.2-of-a#{FILE_DELIMITER} line1.3-of-a") + expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") end .then("stop") do subject.stop @@ -400,7 +401,6 @@ context "when #run is called multiple times", :unix => true do let(:file_path) { "#{tmpdir_path}/a.log" } let(:buffer) { [] } - let(:lsof) { [] } let(:run_thread_proc) do lambda { Thread.new { subject.run(buffer) } } end @@ -424,17 +424,20 @@ end end - it "should only have one set of files open" do + it "should only actually open files when content changes are detected" do subject.register expect(lsof_proc.call).to eq("") + # first run processes the file and records sincedb progress run_thread_proc.call - sleep 0.25 - first_lsof = lsof_proc.call - expect(first_lsof.scan(file_path).size).to eq(1) + wait(1).for{lsof_proc.call.scan(file_path).size}.to eq(1) + # second run quits the first run + # sees the file has not changed size and does not open it run_thread_proc.call - sleep 0.25 - second_lsof = lsof_proc.call - expect(second_lsof.scan(file_path).size).to eq(1) + wait(1).for{lsof_proc.call.scan(file_path).size}.to eq(0) + # truncate and write less than before + File.open(file_path, "w"){ |fd| fd.puts('baz'); fd.fsync } + # sees the file has changed size and does open it + wait(1).for{lsof_proc.call.scan(file_path).size}.to eq(1) end end @@ -463,7 +466,7 @@ "stat_interval" => 0.1, "max_open_files" => 1, "start_position" => "beginning", - "delimiter" => FILE_DELIMITER) + "delimiter" => TEST_FILE_DELIMITER) subject.register end it "collects line events from only one file" do @@ -502,7 +505,7 @@ "max_open_files" => 1, "close_older" => 0.5, "start_position" => "beginning", - "delimiter" => FILE_DELIMITER) + "delimiter" => TEST_FILE_DELIMITER) subject.register end diff --git a/src/main/java/JrubyFileWatchService.java b/src/main/java/JrubyFileWatchService.java new file mode 100644 index 0000000..969147c --- /dev/null +++ b/src/main/java/JrubyFileWatchService.java @@ -0,0 +1,11 @@ +import org.jruby.Ruby; +import org.jruby.runtime.load.BasicLibraryService; +import org.logstash.filewatch.JrubyFileWatchLibrary; + +public class JrubyFileWatchService implements BasicLibraryService { + @Override + public final boolean basicLoad(final Ruby runtime) { + new JrubyFileWatchLibrary().load(runtime, false); + return true; + } +} diff --git a/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java b/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java new file mode 100644 index 0000000..7a859a1 --- /dev/null +++ b/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java @@ -0,0 +1,191 @@ +package org.logstash.filewatch; + +/** + * Created with IntelliJ IDEA. User: efrey Date: 6/11/13 Time: 11:00 AM To + * change this template use File | Settings | File Templates. + * + * http://bugs.sun.com/view_bug.do?bug_id=6357433 + * [Guy] modified original to be a proper JRuby class + * [Guy] do we need this anymore? JRuby 1.7+ uses new Java 7 File API + * + * + * fnv code extracted and modified from https://github.com/jakedouglas/fnv-java + */ + +import org.jruby.Ruby; +import org.jruby.RubyBignum; +import org.jruby.RubyClass; +import org.jruby.RubyFixnum; +import org.jruby.RubyIO; +import org.jruby.RubyInteger; +import org.jruby.RubyModule; +import org.jruby.RubyObject; +import org.jruby.RubyString; +import org.jruby.anno.JRubyClass; +import org.jruby.anno.JRubyMethod; +import org.jruby.runtime.Arity; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.runtime.load.Library; + +import java.io.IOException; +import java.math.BigInteger; +import java.nio.channels.Channel; +import java.nio.channels.FileChannel; +import java.nio.file.FileSystems; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +@SuppressWarnings("ClassUnconnectedToPackage") +public class JrubyFileWatchLibrary implements Library { + + private static final BigInteger INIT32 = new BigInteger("811c9dc5", 16); + private static final BigInteger INIT64 = new BigInteger("cbf29ce484222325", 16); + private static final BigInteger PRIME32 = new BigInteger("01000193", 16); + private static final BigInteger PRIME64 = new BigInteger("100000001b3", 16); + private static final BigInteger MOD32 = new BigInteger("2").pow(32); + private static final BigInteger MOD64 = new BigInteger("2").pow(64); + + @Override + public final void load(final Ruby runtime, final boolean wrap) { + final RubyModule module = runtime.defineModule("FileWatch"); + + RubyClass clazz = runtime.defineClassUnder("FileExt", runtime.getObject(), JrubyFileWatchLibrary.RubyFileExt::new, module); + clazz.defineAnnotatedMethods(JrubyFileWatchLibrary.RubyFileExt.class); + + clazz = runtime.defineClassUnder("Fnv", runtime.getObject(), JrubyFileWatchLibrary.Fnv::new, module); + clazz.defineAnnotatedMethods(JrubyFileWatchLibrary.Fnv.class); + + } + + @JRubyClass(name = "FileExt", parent = "Object") + public static class RubyFileExt extends RubyObject { + + public RubyFileExt(final Ruby runtime, final RubyClass metaClass) { + super(runtime, metaClass); + } + + public RubyFileExt(final RubyClass metaClass) { + super(metaClass); + } + + @JRubyMethod(name = "open", required = 1, meta = true) + public static RubyIO open(final ThreadContext context, final IRubyObject self, final RubyString path) throws IOException { + final Path javapath = FileSystems.getDefault().getPath(path.asJavaString()); + final OpenOption[] options = new OpenOption[1]; + options[0] = StandardOpenOption.READ; + final Channel channel = FileChannel.open(javapath, options); + return new RubyIO(Ruby.getGlobalRuntime(), channel); + } + } + + // This class may be used by fingerprinting in the future + @SuppressWarnings({"NewMethodNamingConvention", "ChainOfInstanceofChecks"}) + @JRubyClass(name = "Fnv", parent = "Object") + public static class Fnv extends RubyObject { + + private byte[] bytes; + private long size; + private boolean open; + + public Fnv(final Ruby runtime, final RubyClass metaClass) { + super(runtime, metaClass); + } + + public Fnv(final RubyClass metaClass) { + super(metaClass); + } + + @JRubyMethod(name = "coerce_bignum", meta = true, required = 1) + public static IRubyObject coerceBignum(final ThreadContext ctx, final IRubyObject recv, final IRubyObject i) { + if (i instanceof RubyBignum) { + return i; + } + if (i instanceof RubyFixnum) { + return RubyBignum.newBignum(ctx.runtime, ((RubyFixnum)i).getBigIntegerValue()); + } + throw ctx.runtime.newRaiseException(ctx.runtime.getClass("StandardError"), "Can't coerce"); + } + + // def initialize(data) + @JRubyMethod(name = "initialize", required = 1) + public IRubyObject rubyInitialize(final ThreadContext ctx, final RubyString data) { + bytes = data.getBytes(); + size = (long) bytes.length; + open = true; + return ctx.nil; + } + + @JRubyMethod(name = "close") + public IRubyObject close(final ThreadContext ctx) { + open = false; + bytes = null; + return ctx.nil; + } + + @JRubyMethod(name = "open?") + public IRubyObject open_p(final ThreadContext ctx) { + if(open) { + return ctx.runtime.getTrue(); + } + return ctx.runtime.getFalse(); + } + + @JRubyMethod(name = "closed?") + public IRubyObject closed_p(final ThreadContext ctx) { + if(open) { + return ctx.runtime.getFalse(); + } + return ctx.runtime.getTrue(); + } + + @JRubyMethod(name = "fnv1a32", optional = 1) + public IRubyObject fnv1a_32(final ThreadContext ctx, IRubyObject[] args) { + IRubyObject[] args1 = args; + if(open) { + args1 = Arity.scanArgs(ctx.runtime, args1, 0, 1); + return RubyBignum.newBignum(ctx.runtime, common_fnv(args1[0], INIT32, PRIME32, MOD32)); + } + throw ctx.runtime.newRaiseException(ctx.runtime.getClass("StandardError"), "Fnv instance is closed!"); + } + + @JRubyMethod(name = "fnv1a64", optional = 1) + public IRubyObject fnv1a_64(final ThreadContext ctx, IRubyObject[] args) { + IRubyObject[] args1 = args; + if(open) { + args1 = Arity.scanArgs(ctx.runtime, args1, 0, 1); + return RubyBignum.newBignum(ctx.runtime, common_fnv(args1[0], INIT64, PRIME64, MOD64)); + } + throw ctx.runtime.newRaiseException(ctx.runtime.getClass("StandardError"), "Fnv instance is closed!"); + } + + private long convertLong(final IRubyObject obj) { + if(obj instanceof RubyInteger) { + return ((RubyInteger)obj).getLongValue(); + } + if(obj instanceof RubyFixnum) { + return ((RubyFixnum)obj).getLongValue(); + } + + return size; + } + + private BigInteger common_fnv(final IRubyObject len, final BigInteger hash, final BigInteger prime, final BigInteger mod) { + long converted = convertLong(len); + + if (converted > size) { + converted = size; + } + + BigInteger tempHash = hash; + for (int idx = 0; (long) idx < converted; idx++) { + tempHash = tempHash.xor(BigInteger.valueOf((long) ((int) bytes[idx] & 0xff))); + tempHash = tempHash.multiply(prime).mod(mod); + } + + return tempHash; + } + } + +} From 86b59acfc7bdff5586071e910a170d613cdd729e Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Fri, 27 Apr 2018 09:00:39 +0100 Subject: [PATCH 27/91] Update docs for new features in v5.0.0 (#177) * Update docs for new features in v5.0.0 * Edit as suggested by review * changed filebeat ref --- docs/index.asciidoc | 234 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 196 insertions(+), 38 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 86044f3..5b846f7 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -25,21 +25,63 @@ Stream events from files, normally by tailing them in a manner similar to `tail -0F` but optionally reading them from the beginning. -By default, each event is assumed to be one line and a line is -taken to be the text before a newline character. Normally, logging will add a newline to the end of each line written. +By default, each event is assumed to be one line +and a line is taken to be the text before a newline character. If you would like to join multiple log lines into one event, -you'll want to use the multiline codec or filter. - -The plugin aims to track changing files and emit new content as it's -appended to each file. It's not well-suited for reading a file from -beginning to end and storing all of it in a single event (not even -with the multiline codec or filter). +you'll want to use the multiline codec. +The plugin loops between discovering new files and processing +each discovered file. Discovered files have a lifecycle, they start off +in the "watched" or "ignored" state. Other states in the lifecycle are: +"active", "closed" and "unwatched" + +By default, a window of 4095 files is used to limit the number of file handles in use. +The processing phase has a number of stages: + +* Checks whether "closed" or "ignored" files have changed in size since last time and +if so puts them in the "watched" state. +* Selects enough "watched" files to fill the available space in the window, these files +are made "active". +* The active files are opened and read, each file is read from the last known position +to the end of current content (EOF) by default. + +In some cases it is useful to be able to control which files are read first, sorting, +and whether files are read completely or banded/striped. +Complete reading is *all of* file A then file B then file C and so on. +Banded or striped reading is *some of* file A then file B then file C and so on looping around +to file A again until all files are read. Banded reading is specified by changing +<> and perhaps <>. +Banding and sorting may be useful if you want some events from all files to appear +in Kibana as early as possible. + +The plugin has two modes of operation, Tail mode and Read mode. + +===== Tail mode + +In this mode the plugin aims to track changing files and emit new content as it's +appended to each file. In this mode, files are seen as a never ending stream of +content and EOF has no special significance. The plugin always assumes that +there will be more content. When files are rotated, the smaller or zero size is +detected, the current position is reset to zero and streaming continues. +A delimiter must be seen before the accumulated characters can be emitted as a line. + +===== Read mode + +In this mode the plugin treats each file as if it is content complete, that is, +a finite stream of lines and now EOF is significant. A last delimiter is not +needed because EOF means that the accumulated characters can be emitted as a line. +Further, EOF here means that the file can be closed and put in the "unwatched" +state - this automatically frees up space in the active window. This mode also +makes it possible to process compressed files as they are content complete. +Read mode also allows for an action to take place after processing the file completely. + +In the past attempts to simulate a Read mode while still assuming infinite streams +was not ideal and a dedicated Read mode is an improvement. ==== Reading from remote network volumes -The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These -remote filesystems typically have behaviors that are very different from local filesystems and +The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These +remote filesystems typically have behaviors that are very different from local filesystems and are therefore unlikely to work correctly when used with the file input. ==== Tracking of current position in watched files @@ -50,29 +92,38 @@ possible to stop and restart Logstash and have it pick up where it left off without missing the lines that were added to the file while Logstash was stopped. -By default, the sincedb file is placed in the home directory of the -user running Logstash with a filename based on the filename patterns -being watched (i.e. the `path` option). Thus, changing the filename -patterns will result in a new sincedb file being used and any -existing current position state will be lost. If you change your -patterns with any frequency it might make sense to explicitly choose -a sincedb path with the `sincedb_path` option. +By default, the sincedb file is placed in the data directory of Logstash +with a filename based on the filename patterns being watched (i.e. the `path` option). +Thus, changing the filename patterns will result in a new sincedb file being used and +any existing current position state will be lost. If you change your patterns +with any frequency it might make sense to explicitly choose a sincedb path +with the `sincedb_path` option. A different `sincedb_path` must be used for each input. Using the same path will cause issues. The read checkpoints for each input must be stored in a different path so the information does not override. -Sincedb files are text files with four columns: +Sincedb records can now be expired meaning that read positions of older files +will not be remembered after a certain time period. File systems may need to reuse +inodes for new content. Ideally, we would not use the read position of old content, +but we have no reliable way to detect that inode reuse has occurred. This is more +relevant to Read mode where a great many files are tracked in the sincedb. +Bear in mind though, if a record has expired, a previously seen file will be read again. + +Sincedb files are text files with four (< v5.0.0), five or six columns: . The inode number (or equivalent). . The major device number of the file system (or equivalent). . The minor device number of the file system (or equivalent). . The current byte offset within the file. +. The last active timestamp (a floating point number) +. The last known path that this record was matched to (for +old sincedb records converted to the new format, this is blank. On non-Windows systems you can obtain the inode number of a file with e.g. `ls -li`. -==== File rotation +==== File rotation in Tail mode File rotation is detected and handled by this input, regardless of whether the file is rotated via a rename or a copy operation. To @@ -102,9 +153,17 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["delete", "log", "log_and_delete"]`|No +| <> |<>|No +| <> |<>, one of `["last_modified", "path"]`|No +| <> |<>, one of `["asc", "desc"]`|No | <> |<>|No | <> |<>|No +| <> |<>, one of `["tail", "read"]`|No | <> |<>|Yes +| <> |<>|No | <> |<>|No | <> |<>|No | <> |<>, one of `["beginning", "end"]`|No @@ -117,10 +176,10 @@ input plugins.   [id="plugins-{type}s-{plugin}-close_older"] -===== `close_older` +===== `close_older` * Value type is <> - * Default value is `3600` + * There is no default value for this setting The file input closes any files that were last read the specified timespan in seconds ago. @@ -129,18 +188,20 @@ read. If tailing, and there is a large time gap in incoming data the file can be closed (allowing other files to be opened) but will be queued for reopening when new data is detected. If reading, the file will be closed after closed_older seconds from when the last bytes were read. -The default is 1 hour +This setting is retained for backward compatibility if you upgrade the +plugin to 5.0.0+, are reading not tailing and do not switch to using Read mode. [id="plugins-{type}s-{plugin}-delimiter"] -===== `delimiter` +===== `delimiter` * Value type is <> * Default value is `"\n"` -set the new line delimiter, defaults to "\n" +set the new line delimiter, defaults to "\n". Note that when reading compressed files +this setting is not used, instead the standard Windows or Unix line endings are used. [id="plugins-{type}s-{plugin}-discover_interval"] -===== `discover_interval` +===== `discover_interval` * Value type is <> * Default value is `15` @@ -149,7 +210,7 @@ How often (in seconds) we expand the filename patterns in the `path` option to discover new files to watch. [id="plugins-{type}s-{plugin}-exclude"] -===== `exclude` +===== `exclude` * Value type is <> * There is no default value for this setting. @@ -159,12 +220,79 @@ patterns are valid here, too. For example, if you have [source,ruby] path => "/var/log/*" -You might want to exclude gzipped files: +In Tail mode, you might want to exclude gzipped files: [source,ruby] exclude => "*.gz" +[id="plugins-{type}s-{plugin}-file_chunk_count"] +===== `file_chunk_count` + + * Value type is <> + * Default value is `4611686018427387903` + +When combined with the `file_chunk_size`, this option sets how many chunks (bands or stripes) +are read from each file before moving to the next active file. +For example, a `file_chunk_count` of 32 and a `file_chunk_size` 32KB will process the next 1MB from each active file. +As the default is very large, the file is effectively read to EOF before moving to the next active file. + +[id="plugins-{type}s-{plugin}-file_chunk_size"] +===== `file_chunk_size` + + * Value type is <> + * Default value is `32768` (32KB) + +File content is read off disk in blocks or chunks and lines are extracted from the chunk. +See <> to see why and when to change this setting +from the default. + +[id="plugins-{type}s-{plugin}-file_completed_action"] +===== `file_completed_action` + + * Value can be any of: `delete`, `log`, `log_and_delete` + * The default is `delete`. + +When in `read` mode, what action should be carried out when a file is done with. +If 'delete' is specified then the file will be deleted. If 'log' is specified +then the full path of the file is logged to the file specified in the +`file_completed_log_path` setting. If `log_and_delete` is specified then +both above actions take place. + +[id="plugins-{type}s-{plugin}-file_completed_log_path"] +===== `file_completed_log_path` + + * Value type is <> + * There is no default value for this setting. + +Which file should the completely read file paths be appended to. Only specify +this path to a file when `file_completed_action` is 'log' or 'log_and_delete'. +IMPORTANT: this file is appended to only - it could become very large. You are +responsible for file rotation. + +[id="plugins-{type}s-{plugin}-file_sort_by"] +===== `file_sort_by` + + * Value can be any of: `last_modified`, `path` + * The default is `last_modified`. + +Which attribute of a "watched" file should be used to sort them by. +Files can be sorted by modified date or full path alphabetic. +Previously the processing order of the discovered and therefore +"watched" files was OS dependent. + +[id="plugins-{type}s-{plugin}-file_sort_direction"] +===== `file_sort_direction` + + * Value can be any of: `asc`, `desc` + * The default is `asc`. + +Select between ascending and descending order when sorting "watched" files. +If oldest data first is important then the defaults of `last_modified` + `asc` are good. +If newest data first is more important then opt for `last_modified` + `desc`. +If you use special naming conventions for the file full paths then perhaps +`path` + `asc` will help to control the order of file processing. + [id="plugins-{type}s-{plugin}-ignore_older"] -===== `ignore_older` +===== `ignore_older` * Value type is <> * There is no default value for this setting. @@ -176,7 +304,7 @@ longer ignored and any new data is read. By default, this option is disabled. Note this unit is in seconds. [id="plugins-{type}s-{plugin}-max_open_files"] -===== `max_open_files` +===== `max_open_files` * Value type is <> * There is no default value for this setting. @@ -186,10 +314,29 @@ at any one time. Use close_older to close some files if you need to process more files than this number. This should not be set to the maximum the OS can do because file handles are needed for other LS plugins and OS processes. -The default of 4095 is set in filewatch. +A default of 4095 is set in internally. + +[id="plugins-{type}s-{plugin}-mode"] +===== `mode` + + * Value can be either `tail` or `read`. + * The default value is `tail`. + +What mode do you want the file input to operate in. Tail a few files or +read many content-complete files. Read mode now supports gzip file processing. +If "read" is specified then the following other settings are ignored: + +. `start_position` (files are always read from the beginning) +. `close_older` (files are automatically 'closed' when EOF is reached) + +If "read" is specified then the following settings are heeded: + +. `ignore_older` (older files are not processed) +. `file_completed_action` (what action should be taken when the file is processed) +. `file_completed_log_path` (which file should the completed file path be logged to) [id="plugins-{type}s-{plugin}-path"] -===== `path` +===== `path` * This is a required setting. * Value type is <> @@ -204,9 +351,21 @@ Paths must be absolute and cannot be relative. You may also configure multiple paths. See an example on the {logstash-ref}/configuration-file-structure.html#array[Logstash configuration page]. +[id="plugins-{type}s-{plugin}-sincedb_clean_after"] +===== `sincedb_clean_after` + + * Value type is <> + * The default value for this setting is 14. + * This unit is in *days* and can be decimal e.g. 0.5 is 12 hours. + +The sincedb record now has a last active timestamp associated with it. +If no changes are detected in a tracked file in the last N days its sincedb +tracking record expires and will not be persisted. +This option helps protect against the inode recycling problem. +Filebeat has a {filebeat-ref}/faq.html#inode-reuse-issue[FAQ about inode recycling]. [id="plugins-{type}s-{plugin}-sincedb_path"] -===== `sincedb_path` +===== `sincedb_path` * Value type is <> * There is no default value for this setting. @@ -217,7 +376,7 @@ The default will write sincedb files to `/plugins/inputs/file` NOTE: it must be a file path and not a directory path [id="plugins-{type}s-{plugin}-sincedb_write_interval"] -===== `sincedb_write_interval` +===== `sincedb_write_interval` * Value type is <> * Default value is `15` @@ -226,7 +385,7 @@ How often (in seconds) to write a since database with the current position of monitored log files. [id="plugins-{type}s-{plugin}-start_position"] -===== `start_position` +===== `start_position` * Value can be any of: `beginning`, `end` * Default value is `"end"` @@ -243,7 +402,7 @@ has already been seen before, this option has no effect and the position recorded in the sincedb file will be used. [id="plugins-{type}s-{plugin}-stat_interval"] -===== `stat_interval` +===== `stat_interval` * Value type is <> * Default value is `1` @@ -252,9 +411,8 @@ How often (in seconds) we stat files to see if they have been modified. Increasing this interval will decrease the number of system calls we make, but increase the time to detect new log lines. - - [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] -:default_codec!: \ No newline at end of file +:default_codec!: + From eb1f016088f9da456dab96cf32f7f2189e4b3e4b Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Fri, 27 Apr 2018 18:23:27 +0100 Subject: [PATCH 28/91] Add changelog entry, use version 4.1.0, revert "close_older" default change. (#178) * add changelog entry, revert `close_older` default change. * change release version to 4.1.0 * reword changelog --- CHANGELOG.md | 28 +++++++++++++++++++++++++--- docs/index.asciidoc | 2 +- lib/logstash/inputs/file.rb | 7 +++---- logstash-input-file.gemspec | 2 +- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff6c0b1..a179ec7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +## 4.1.0 + - Move Filewatch code into the plugin folder, rework Filewatch code to use + Logstash facilities like logging and environment. + - New feature: `mode` setting. Introduces two modes, `tail` mode is the + existing behaviour for tailing, `read` mode is new behaviour that is + optimized for the read complete content scenario. Please read the docs to + fully appreciate the benefits of `read` mode. + - New feature: File completion actions. Settings `file_completed_action` + and `file_completed_log_path` control what actions to do after a file is + completely read. Applicable: `read` mode only. + - New feature: in `read` mode, compressed files can be processed, GZIP only. + - New feature: Files are sorted after being discovered. Settings `file_sort_by` + and `file_sort_direction` control the sort order. Applicable: any mode. + - New feature: Banded or striped file processing. Settings: `file_chunk_size` + and `file_chunk_count` control banded or striped processing. Applicable: any mode. + - New feature: `sincedb_clean_after` setting. Introduces expiry of sincedb + records. The default is 14 days. If, after `sincedb_clean_after` days, no + activity has been detected on a file (inode) the record expires and is not + written to disk. The persisted record now includes the "last activity seen" + timestamp. Applicable: any mode. + - Docs: extensive additions to introduce the new features. + ## 4.0.5 - Docs: Set the default_codec doc attribute. @@ -12,9 +34,9 @@ - Fix an issue with the rspec suite not finding log4j ## 4.0.0 - - Breaking: `ignore_older` settings is disabled by default. Previously if the file was older than - 24 hours (the default for ignore_older), it would be ignored. This confused new users a lot, specially - when they were reading new files with Logstash (with `start_position => beginning`). This setting also + - Breaking: `ignore_older` settings is disabled by default. Previously if the file was older than + 24 hours (the default for ignore_older), it would be ignored. This confused new users a lot, specially + when they were reading new files with Logstash (with `start_position => beginning`). This setting also makes it consistent with Filebeat. ## 3.1.2 diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 5b846f7..0689137 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -179,7 +179,7 @@ input plugins. ===== `close_older` * Value type is <> - * There is no default value for this setting + * Default value is `3600` The file input closes any files that were last read the specified timespan in seconds ago. diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index cb81732..2649442 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -149,13 +149,12 @@ class File < LogStash::Inputs::Base # The file input closes any files that were last read the specified # timespan in seconds ago. - # This has different implications depending on if a file is being tailed or - # read. If tailing, and there is a large time gap in incoming data the file + # If tailing, and there is a large time gap in incoming data the file # can be closed (allowing other files to be opened) but will be queued for # reopening when new data is detected. If reading, the file will be closed # after closed_older seconds from when the last bytes were read. - # By default, this option disabled - config :close_older, :validate => :number + # The default is 1 hour + config :close_older, :validate => :number, :default => 1 * 60 * 60 # What is the maximum number of file_handles that this input consumes # at any one time. Use close_older to close some files if you need to diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 274c1cb..e583e81 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '5.0.0' + s.version = '4.1.0' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 606fcb86ecbf538a834fc1a34a6d5fc079568f94 Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Tue, 1 May 2018 16:17:46 +0100 Subject: [PATCH 29/91] Critical fixes for two show stopper bugs #180 and #182 (#183) * Fix JAR_VERSION read problem, add test and fix /dev/null write IO error * bump version and add changelog entry Fix #182 Fix #180 --- CHANGELOG.md | 6 ++++++ lib/filewatch/bootstrap.rb | 2 +- lib/filewatch/sincedb_collection.rb | 2 +- logstash-input-file.gemspec | 2 +- spec/filewatch/reading_spec.rb | 16 ++++++++++++++++ 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a179ec7..9d20448 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 4.1.1 + - Fix JAR_VERSION read problem, prevented Logstash from starting. + [Issue #180](https://github.com/logstash-plugins/logstash-input-file/issues/180) + - Fix sincedb write error when using /dev/null, repeatedly causes a plugin restart. + [Issue #182](https://github.com/logstash-plugins/logstash-input-file/issues/182) + ## 4.1.0 - Move Filewatch code into the plugin folder, rework Filewatch code to use Logstash facilities like logging and environment. diff --git a/lib/filewatch/bootstrap.rb b/lib/filewatch/bootstrap.rb index 5eac4fa..a37d9df 100644 --- a/lib/filewatch/bootstrap.rb +++ b/lib/filewatch/bootstrap.rb @@ -30,7 +30,7 @@ def prepare_inode(path, stat) end end - jar_version = IO.read("JAR_VERSION").strip + jar_version = Pathname.new(__FILE__).dirname.join("../../JAR_VERSION").realpath.read.strip require "java" require_relative "../../lib/jars/filewatch-#{jar_version}.jar" diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb index 6d5dba2..d9b7b3d 100644 --- a/lib/filewatch/sincedb_collection.rb +++ b/lib/filewatch/sincedb_collection.rb @@ -207,7 +207,7 @@ def atomic_write end def non_atomic_write - IO.open(@full_path, 0) do |io| + IO.open(IO.sysopen(@full_path, "w+")) do |io| @serializer.serialize(@sincedb, io) end end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index e583e81..70cb7bd 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.0' + s.version = '4.1.1' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filewatch/reading_spec.rb b/spec/filewatch/reading_spec.rb index bb574d3..cf56ef7 100644 --- a/spec/filewatch/reading_spec.rb +++ b/spec/filewatch/reading_spec.rb @@ -55,6 +55,22 @@ module FileWatch end end + context "when watching a directory with files and sincedb_path is /dev/null or NUL" do + let(:directory) { Stud::Temporary.directory } + let(:sincedb_path) { File::NULL } + let(:watch_dir) { ::File.join(directory, "*.log") } + let(:file_path) { ::File.join(directory, "1.log") } + + it "the file is read" do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + actions.activate + reading.watch_this(watch_dir) + reading.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + end + end + context "when watching a directory with files using striped reading" do let(:directory) { Stud::Temporary.directory } let(:watch_dir) { ::File.join(directory, "*.log") } From ab152f20cc573c658e08b890a1169bab8e0fd7ad Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Thu, 3 May 2018 19:31:32 +0100 Subject: [PATCH 30/91] Important v4.1.1 Fixes 184 and 185 (#186) * make winhelper require work. * Fix for 185 when no delimiter is found in a chunk, the chunk is reread - no forward progress * Bump version and changelog * Add missing # encoding: utf-8 to spec files * fix typo * review changes * don't need loggable no mo Fixes #184 Fixes #185 --- CHANGELOG.md | 7 ++++ lib/filewatch/bootstrap.rb | 4 ++- lib/filewatch/read_mode/handlers/read_file.rb | 18 +++++----- lib/filewatch/sincedb_value.rb | 31 +++++++++-------- lib/filewatch/tail_mode/handlers/base.rb | 10 ++++-- lib/filewatch/watched_file.rb | 15 ++++++++- lib/logstash/inputs/file.rb | 2 +- logstash-input-file.gemspec | 2 +- spec/filewatch/buftok_spec.rb | 1 + spec/filewatch/reading_spec.rb | 21 +++++++++++- spec/filewatch/spec_helper.rb | 6 ++++ spec/filewatch/tailing_spec.rb | 33 ++++++++++++++++--- spec/filewatch/watched_file_spec.rb | 1 + .../watched_files_collection_spec.rb | 1 + spec/filewatch/winhelper_spec.rb | 1 + 15 files changed, 115 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d20448..0c33058 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 4.1.2 + - Fix `require winhelper` error in WINDOWS. + [Issue #184](https://github.com/logstash-plugins/logstash-input-file/issues/184) + - Fix when no delimiter is found in a chunk, the chunk is reread - no forward progress + is made in the file. + [Issue #185](https://github.com/logstash-plugins/logstash-input-file/issues/185) + ## 4.1.1 - Fix JAR_VERSION read problem, prevented Logstash from starting. [Issue #180](https://github.com/logstash-plugins/logstash-input-file/issues/180) diff --git a/lib/filewatch/bootstrap.rb b/lib/filewatch/bootstrap.rb index a37d9df..99c196d 100644 --- a/lib/filewatch/bootstrap.rb +++ b/lib/filewatch/bootstrap.rb @@ -37,7 +37,7 @@ def prepare_inode(path, stat) require "jruby_file_watch" if LogStash::Environment.windows? - require "winhelper" + require_relative "winhelper" FileOpener = FileExt InodeMixin = WindowsInode else @@ -53,6 +53,8 @@ def to_s end end + BufferExtractResult = Struct.new(:lines, :warning, :additional) + class NoSinceDBPathGiven < StandardError; end # how often (in seconds) we logger.warn a failed file open, per path. diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index cff1a76..bfd9b45 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -5,28 +5,28 @@ class ReadFile < Base def handle_specifically(watched_file) if open_file(watched_file) add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) - # if the `file_chunk_count` * `file_chunk_size` is less than the file size - # then this method will be executed multiple times - # and the seek is moved to just after a line boundary as recorded in the sincedb - # for each run - so we reset the buffer - watched_file.reset_buffer - watched_file.file_seek(watched_file.bytes_read) changed = false @settings.file_chunk_count.times do begin - lines = watched_file.buffer_extract(watched_file.file_read(@settings.file_chunk_size)) - logger.warn("read_to_eof: no delimiter found in current chunk") if lines.empty? + data = watched_file.file_read(@settings.file_chunk_size) + result = watched_file.buffer_extract(data) # expect BufferExtractResult + logger.info(result.warning, result.additional) unless result.warning.empty? changed = true - lines.each do |line| + result.lines.each do |line| watched_file.listener.accept(line) + # sincedb position is independent from the watched_file bytes_read sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size) end + # instead of tracking the bytes_read line by line we need to track by the data read size. + # because we initially seek to the bytes_read not the sincedb position + watched_file.increment_bytes_read(data.bytesize) rescue EOFError # flush the buffer now in case there is no final delimiter line = watched_file.buffer.flush watched_file.listener.accept(line) unless line.empty? watched_file.listener.eof watched_file.file_close + # unset_watched_file will set sincedb_value.position to be watched_file.bytes_read sincedb_collection.unset_watched_file(watched_file) watched_file.listener.deleted watched_file.unwatch diff --git a/lib/filewatch/sincedb_value.rb b/lib/filewatch/sincedb_value.rb index d5fa921..f97c44a 100644 --- a/lib/filewatch/sincedb_value.rb +++ b/lib/filewatch/sincedb_value.rb @@ -2,8 +2,14 @@ module FileWatch # Tracks the position and expiry of the offset of a file-of-interest + # NOTE: the `watched_file.bytes_read` and this `sincedb_value.position` can diverge + # At any given moment IF the `watched_file.bytes_read` is greater than `sincedb_value.position` + # then it is larger to account for bytes held in the `watched_file.buffer` + # in Tail mode if we quit the buffer is not flushed and we restart from + # the `sincedb_value.position` (end of the last line read). + # in Read mode the buffer is flushed as a line and both values should be the same. class SincedbValue - attr_reader :last_changed_at, :watched_file, :path_in_sincedb + attr_reader :last_changed_at, :watched_file, :path_in_sincedb, :position def initialize(position, last_changed_at = nil, watched_file = nil) @position = position # this is the value read from disk @@ -21,27 +27,19 @@ def last_changed_at_expires(duration) @last_changed_at + duration end - def position - # either the value from disk or the current wf position - @watched_file.nil? ? @position : @watched_file.bytes_read - end - def update_position(pos) + # called when we reset the position to bof or eof on shrink or file read complete touch - if @watched_file.nil? - @position = pos - else - @watched_file.update_bytes_read(pos) - end + @position = pos + @watched_file.update_bytes_read(pos) unless @watched_file.nil? end def increment_position(pos) + # called when actual lines are sent to the observer listener + # this gets serialized as its a more true indication of position than + # chunk read size touch - if watched_file.nil? - @position += pos - else - @watched_file.increment_bytes_read(pos) - end + @position += pos end def set_watched_file(watched_file) @@ -69,6 +67,7 @@ def clear_watched_file end def unset_watched_file + # called in read mode only because we flushed any remaining bytes as a final line. # cache the position # we don't cache the path here because we know we are done with this file. # either due via the `delete` handling diff --git a/lib/filewatch/tail_mode/handlers/base.rb b/lib/filewatch/tail_mode/handlers/base.rb index c8bc7a0..a769f5a 100644 --- a/lib/filewatch/tail_mode/handlers/base.rb +++ b/lib/filewatch/tail_mode/handlers/base.rb @@ -42,13 +42,17 @@ def read_to_eof(watched_file) @settings.file_chunk_count.times do begin data = watched_file.file_read(@settings.file_chunk_size) - lines = watched_file.buffer_extract(data) - logger.warn("read_to_eof: no delimiter found in current chunk") if lines.empty? + result = watched_file.buffer_extract(data) # expect BufferExtractResult + logger.info(result.warning, result.additional) unless result.warning.empty? changed = true - lines.each do |line| + result.lines.each do |line| watched_file.listener.accept(line) + # sincedb position is now independent from the watched_file bytes_read sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size) end + # instead of tracking the bytes_read line by line we need to track by the data read size. + # because we seek to the bytes_read not the sincedb position + watched_file.increment_bytes_read(data.bytesize) rescue EOFError # it only makes sense to signal EOF in "read" mode not "tail" break diff --git a/lib/filewatch/watched_file.rb b/lib/filewatch/watched_file.rb index dffdf29..a80a93c 100644 --- a/lib/filewatch/watched_file.rb +++ b/lib/filewatch/watched_file.rb @@ -102,7 +102,20 @@ def reset_buffer end def buffer_extract(data) - @buffer.extract(data) + warning, additional = "", {} + lines = @buffer.extract(data) + if lines.empty? + warning.concat("buffer_extract: a delimiter can't be found in current chunk") + warning.concat(", maybe there are no more delimiters or the delimiter is incorrect") + warning.concat(" or the text before the delimiter, a 'line', is very large") + warning.concat(", if this message is logged often try increasing the `file_chunk_size` setting.") + additional["delimiter"] = @settings.delimiter + additional["read_position"] = @bytes_read + additional["bytes_read_count"] = data.bytesize + additional["last_known_file_size"] = @last_stat_size + additional["file_path"] = @path + end + BufferExtractResult.new(lines, warning, additional) end def increment_bytes_read(delta) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 2649442..e3d499a 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -142,7 +142,7 @@ class File < LogStash::Inputs::Base # When the file input discovers a file that was last modified # before the specified timespan in seconds, the file is ignored. - # After it's discovery, if an ignored file is modified it is no + # After its discovery, if an ignored file is modified it is no # longer ignored and any new data is read. By default, this option is # disabled. Note this unit is in seconds. config :ignore_older, :validate => :number diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 70cb7bd..c37c4b4 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.1' + s.version = '4.1.2' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filewatch/buftok_spec.rb b/spec/filewatch/buftok_spec.rb index 17e7532..c27cccb 100644 --- a/spec/filewatch/buftok_spec.rb +++ b/spec/filewatch/buftok_spec.rb @@ -1,3 +1,4 @@ +# encoding: utf-8 require_relative 'spec_helper' describe FileWatch::BufferedTokenizer do diff --git a/spec/filewatch/reading_spec.rb b/spec/filewatch/reading_spec.rb index cf56ef7..4f1f845 100644 --- a/spec/filewatch/reading_spec.rb +++ b/spec/filewatch/reading_spec.rb @@ -1,4 +1,4 @@ - +# encoding: utf-8 require 'stud/temporary' require_relative 'spec_helper' require 'filewatch/observing_read' @@ -95,6 +95,25 @@ module FileWatch end end + context "when a non default delimiter is specified and it is not in the content" do + let(:opts) { super.merge(:delimiter => "\nø") } + + it "the file is opened, data is read, but no lines are found initially, at EOF the whole file becomes the line" do + File.open(file_path, "wb") { |file| file.write("line1\nline2") } + actions.activate + reading.watch_this(watch_dir) + reading.subscribe(observer) + listener = observer.listener_for(file_path) + expect(listener.calls).to eq([:open, :accept, :eof, :delete]) + expect(listener.lines).to eq(["line1\nline2"]) + sincedb_record_fields = File.read(sincedb_path).split(" ") + position_field_index = 3 + # tailing, no delimiter, we are expecting one, if it grows we read from the start. + # there is an info log telling us that no lines were seen but we can't test for it. + expect(sincedb_record_fields[position_field_index]).to eq("11") + end + end + describe "reading fixtures" do let(:directory) { FIXTURE_DIR } diff --git a/spec/filewatch/spec_helper.rb b/spec/filewatch/spec_helper.rb index 578cc46..f58e767 100644 --- a/spec/filewatch/spec_helper.rb +++ b/spec/filewatch/spec_helper.rb @@ -1,3 +1,4 @@ +# encoding: utf-8 require "rspec_sequencing" require 'rspec/wait' require "logstash/devutils/rspec/spec_helper" @@ -118,3 +119,8 @@ def clear @listeners.clear; end end end + +ENV["LOG_AT"].tap do |level| + LogStash::Logging::Logger::configure_logging(level) unless level.nil? +end + diff --git a/spec/filewatch/tailing_spec.rb b/spec/filewatch/tailing_spec.rb index 11f50eb..091eff0 100644 --- a/spec/filewatch/tailing_spec.rb +++ b/spec/filewatch/tailing_spec.rb @@ -1,11 +1,8 @@ - +# encoding: utf-8 require 'stud/temporary' require_relative 'spec_helper' require 'filewatch/observing_tail' -LogStash::Logging::Logger::configure_logging("WARN") -# LogStash::Logging::Logger::configure_logging("DEBUG") - module FileWatch describe Watch do before(:all) do @@ -58,7 +55,6 @@ module FileWatch end context "when max_active is 1" do - it "without close_older set, opens only 1 file" do actions.activate tailing.watch_this(watch_dir) @@ -434,6 +430,33 @@ module FileWatch expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out]) end end + + context "when a non default delimiter is specified and it is not in the content" do + let(:opts) { super.merge(:ignore_older => 20, :close_older => 1, :delimiter => "\nø") } + before do + RSpec::Sequencing + .run("create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2") } + end + .then("start watching before file ages more than close_older") do + tailing.watch_this(watch_dir) + end + .then_after(2.1, "quit after allowing time to close the file") do + tailing.quit + end + end + + it "the file is opened, data is read, but no lines are found, the file times out" do + tailing.subscribe(observer) + expect(observer.listener_for(file_path).calls).to eq([:open, :timed_out]) + expect(observer.listener_for(file_path).lines).to eq([]) + sincedb_record_fields = File.read(sincedb_path).split(" ") + position_field_index = 3 + # tailing, no delimiter, we are expecting one, if it grows we read from the start. + # there is an info log telling us that no lines were seen but we can't test for it. + expect(sincedb_record_fields[position_field_index]).to eq("0") + end + end end end diff --git a/spec/filewatch/watched_file_spec.rb b/spec/filewatch/watched_file_spec.rb index 69181e4..adf0a7e 100644 --- a/spec/filewatch/watched_file_spec.rb +++ b/spec/filewatch/watched_file_spec.rb @@ -1,3 +1,4 @@ +# encoding: utf-8 require 'stud/temporary' require_relative 'spec_helper' diff --git a/spec/filewatch/watched_files_collection_spec.rb b/spec/filewatch/watched_files_collection_spec.rb index c8fb491..06eba28 100644 --- a/spec/filewatch/watched_files_collection_spec.rb +++ b/spec/filewatch/watched_files_collection_spec.rb @@ -1,3 +1,4 @@ +# encoding: utf-8 require_relative 'spec_helper' module FileWatch diff --git a/spec/filewatch/winhelper_spec.rb b/spec/filewatch/winhelper_spec.rb index f65e564..9fb2b78 100644 --- a/spec/filewatch/winhelper_spec.rb +++ b/spec/filewatch/winhelper_spec.rb @@ -1,3 +1,4 @@ +# encoding: utf-8 require "stud/temporary" require "fileutils" From 323e669fafd99f6762ed703c559e0229d2a2a280 Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Tue, 12 Jun 2018 08:45:08 +0100 Subject: [PATCH 31/91] In read mode, request sincedb disk flush on each read loop iteration. (#196) * In read mode, request sincedb disk flush on each read loop iteration. The actual write occurs at the `sincedb_write interval`. Note: the chunk size vs the events-per-second affect how often a request is made. e.g. a large chunk size (1MB), lines of say 128 bytes and 256 events/s means each iteration takes 32 seconds, and an eps of 32768 takes 250 ms. * changelog and version bump * make spec a tad more reliable. * add a reference to watch so we can react to quit in the read loop * remove immediate write and add a call to write in the discover/stat loop Fix #195 --- CHANGELOG.md | 7 ++++ lib/filewatch/observing_read.rb | 1 - lib/filewatch/read_mode/handlers/base.rb | 7 +++- lib/filewatch/read_mode/handlers/read_file.rb | 5 +-- .../read_mode/handlers/read_zip_file.rb | 13 +++--- lib/filewatch/read_mode/processor.rb | 6 ++- lib/filewatch/sincedb_collection.rb | 32 ++++++++++----- lib/filewatch/watch.rb | 1 + lib/logstash/inputs/file.rb | 4 +- logstash-input-file.gemspec | 2 +- .../read_mode_handlers_read_file_spec.rb | 40 +++++++++++++++++++ spec/filewatch/spec_helper.rb | 26 ++++++++++++ spec/filewatch/tailing_spec.rb | 2 +- 13 files changed, 118 insertions(+), 28 deletions(-) create mode 100644 spec/filewatch/read_mode_handlers_read_file_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c33058..929f0a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 4.1.3 + - Fixed `read` mode of regular files sincedb write is requested in each read loop + iteration rather than waiting for the end-of-file to be reached. Note: for gz files, + the sincedb entry can only be updated at the end of the file as it is not possible + to seek into a compressed file and begin reading from that position. + [Issue #196](https://github.com/logstash-plugins/logstash-input-file/pull/196) + ## 4.1.2 - Fix `require winhelper` error in WINDOWS. [Issue #184](https://github.com/logstash-plugins/logstash-input-file/issues/184) diff --git a/lib/filewatch/observing_read.rb b/lib/filewatch/observing_read.rb index 272fee9..2a6316e 100644 --- a/lib/filewatch/observing_read.rb +++ b/lib/filewatch/observing_read.rb @@ -18,6 +18,5 @@ def subscribe(observer) def build_specific_processor(settings) ReadMode::Processor.new(settings) end - end end diff --git a/lib/filewatch/read_mode/handlers/base.rb b/lib/filewatch/read_mode/handlers/base.rb index 3a8680e..8146808 100644 --- a/lib/filewatch/read_mode/handlers/base.rb +++ b/lib/filewatch/read_mode/handlers/base.rb @@ -7,12 +7,17 @@ class Base attr_reader :sincedb_collection - def initialize(sincedb_collection, observer, settings) + def initialize(processor, sincedb_collection, observer, settings) @settings = settings + @processor = processor @sincedb_collection = sincedb_collection @observer = observer end + def quit? + @processor.watch.quit? + end + def handle(watched_file) logger.debug("handling: #{watched_file.path}") unless watched_file.has_listener? diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index bfd9b45..818e0f7 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -5,13 +5,12 @@ class ReadFile < Base def handle_specifically(watched_file) if open_file(watched_file) add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) - changed = false @settings.file_chunk_count.times do + break if quit? begin data = watched_file.file_read(@settings.file_chunk_size) result = watched_file.buffer_extract(data) # expect BufferExtractResult logger.info(result.warning, result.additional) unless result.warning.empty? - changed = true result.lines.each do |line| watched_file.listener.accept(line) # sincedb position is independent from the watched_file bytes_read @@ -20,6 +19,7 @@ def handle_specifically(watched_file) # instead of tracking the bytes_read line by line we need to track by the data read size. # because we initially seek to the bytes_read not the sincedb position watched_file.increment_bytes_read(data.bytesize) + sincedb_collection.request_disk_flush rescue EOFError # flush the buffer now in case there is no final delimiter line = watched_file.buffer.flush @@ -40,7 +40,6 @@ def handle_specifically(watched_file) break end end - sincedb_collection.request_disk_flush if changed end end end diff --git a/lib/filewatch/read_mode/handlers/read_zip_file.rb b/lib/filewatch/read_mode/handlers/read_zip_file.rb index 75689fc..a0d7fb8 100644 --- a/lib/filewatch/read_mode/handlers/read_zip_file.rb +++ b/lib/filewatch/read_mode/handlers/read_zip_file.rb @@ -13,10 +13,6 @@ def handle_specifically(watched_file) add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) # can't really stripe read a zip file, its all or nothing. watched_file.listener.opened - # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading) - # should we track lines read in the sincedb and - # fast forward through the lines until we reach unseen content? - # meaning that we can quit in the middle of a zip file begin file_stream = FileInputStream.new(watched_file.path) gzip_stream = GZIPInputStream.new(file_stream) @@ -24,14 +20,19 @@ def handle_specifically(watched_file) buffered = BufferedReader.new(decoder) while (line = buffered.readLine(false)) watched_file.listener.accept(line) + # can't quit, if we did then we would incorrectly write a 'completed' sincedb entry + # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading) + # should we track lines read in the sincedb and + # fast forward through the lines until we reach unseen content? + # meaning that we can quit in the middle of a zip file end watched_file.listener.eof rescue ZipException => e logger.error("Cannot decompress the gzip file at path: #{watched_file.path}") watched_file.listener.error else - sincedb_collection.store_last_read(watched_file.sincedb_key, watched_file.last_stat_size) - sincedb_collection.request_disk_flush + watched_file.update_bytes_read(watched_file.last_stat_size) + sincedb_collection.unset_watched_file(watched_file) watched_file.listener.deleted watched_file.unwatch ensure diff --git a/lib/filewatch/read_mode/processor.rb b/lib/filewatch/read_mode/processor.rb index 5a8c615..90ca05b 100644 --- a/lib/filewatch/read_mode/processor.rb +++ b/lib/filewatch/read_mode/processor.rb @@ -25,8 +25,10 @@ def add_watch(watch) end def initialize_handlers(sincedb_collection, observer) - @read_file = Handlers::ReadFile.new(sincedb_collection, observer, @settings) - @read_zip_file = Handlers::ReadZipFile.new(sincedb_collection, observer, @settings) + # we deviate from the tail mode handler initialization here + # by adding a reference to self so we can read the quit flag during a (depth first) read loop + @read_file = Handlers::ReadFile.new(self, sincedb_collection, observer, @settings) + @read_zip_file = Handlers::ReadZipFile.new(self, sincedb_collection, observer, @settings) end def read_file(watched_file) diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb index d9b7b3d..90b59b0 100644 --- a/lib/filewatch/sincedb_collection.rb +++ b/lib/filewatch/sincedb_collection.rb @@ -20,14 +20,21 @@ def initialize(settings) @write_method = LogStash::Environment.windows? || @path.chardev? || @path.blockdev? ? method(:non_atomic_write) : method(:atomic_write) @full_path = @path.to_path FileUtils.touch(@full_path) + @write_requested = false + end + + def write_requested? + @write_requested end def request_disk_flush - now = Time.now.to_i - delta = now - @sincedb_last_write - if delta >= @settings.sincedb_write_interval - logger.debug("writing sincedb (delta since last write = #{delta})") - sincedb_write(now) + @write_requested = true + flush_at_interval + end + + def write_if_requested + if write_requested? + flush_at_interval end end @@ -51,7 +58,6 @@ def open #No existing sincedb to load logger.debug("open: error: #{path}: #{e.inspect}") end - end def associate(watched_file) @@ -130,10 +136,6 @@ def rewind(key) @sincedb[key].update_position(0) end - def store_last_read(key, last_read) - @sincedb[key].update_position(last_read) - end - def increment(key, amount) @sincedb[key].increment_position(amount) end @@ -167,6 +169,15 @@ def watched_file_unset?(key) private + def flush_at_interval + now = Time.now.to_i + delta = now - @sincedb_last_write + if delta >= @settings.sincedb_write_interval + logger.debug("writing sincedb (delta since last write = #{delta})") + sincedb_write(now) + end + end + def handle_association(sincedb_value, watched_file) watched_file.update_bytes_read(sincedb_value.position) sincedb_value.set_watched_file(watched_file) @@ -193,6 +204,7 @@ def sincedb_write(time = Time.now.to_i) logger.debug("sincedb_write: cleaned", "key" => "'#{key}'") end @sincedb_last_write = time + @write_requested = false rescue Errno::EACCES # no file handles free perhaps # maybe it will work next time diff --git a/lib/filewatch/watch.rb b/lib/filewatch/watch.rb index c544719..6256ae4 100644 --- a/lib/filewatch/watch.rb +++ b/lib/filewatch/watch.rb @@ -51,6 +51,7 @@ def subscribe(observer, sincedb_collection) until quit? iterate_on_state break if quit? + sincedb_collection.write_if_requested glob += 1 if glob == interval discover diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index e3d499a..d7e068e 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -316,6 +316,7 @@ def run(queue) start_processing @queue = queue @watcher.subscribe(self) # halts here until quit is called + # last action of the subscribe call is to write the sincedb exit_flush end # def run @@ -338,9 +339,6 @@ def log_line_received(path, line) end def stop - # in filewatch >= 0.6.7, quit will closes and forget all files - # but it will write their last read positions to since_db - # beforehand if @watcher @codec.close @watcher.quit diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index c37c4b4..38e89de 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.2' + s.version = '4.1.3' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filewatch/read_mode_handlers_read_file_spec.rb b/spec/filewatch/read_mode_handlers_read_file_spec.rb new file mode 100644 index 0000000..4667b36 --- /dev/null +++ b/spec/filewatch/read_mode_handlers_read_file_spec.rb @@ -0,0 +1,40 @@ +# encoding: utf-8 +require_relative 'spec_helper' + +module FileWatch + describe ReadMode::Handlers::ReadFile do + let(:settings) do + Settings.from_options( + :sincedb_write_interval => 0, + :sincedb_path => File::NULL + ) + end + let(:sdb_collection) { SincedbCollection.new(settings) } + let(:directory) { Pathname.new(FIXTURE_DIR) } + let(:pathname) { directory.join('uncompressed.log') } + let(:watched_file) { WatchedFile.new(pathname, pathname.stat, settings) } + let(:processor) { ReadMode::Processor.new(settings).add_watch(watch) } + let(:file) { DummyFileReader.new(settings.file_chunk_size, 2) } + + context "simulate reading a 64KB file with a default chunk size of 32KB and a zero sincedb write interval" do + let(:watch) { double("watch", :quit? => false) } + it "calls 'sincedb_write' exactly 2 times" do + allow(FileOpener).to receive(:open).with(watched_file.path).and_return(file) + expect(sdb_collection).to receive(:sincedb_write).exactly(2).times + watched_file.activate + processor.initialize_handlers(sdb_collection, TestObserver.new) + processor.read_file(watched_file) + end + end + + context "simulate reading a 64KB file with a default chunk size of 32KB and a zero sincedb write interval" do + let(:watch) { double("watch", :quit? => true) } + it "calls 'sincedb_write' exactly 0 times as shutdown is in progress" do + expect(sdb_collection).to receive(:sincedb_write).exactly(0).times + watched_file.activate + processor.initialize_handlers(sdb_collection, TestObserver.new) + processor.read_file(watched_file) + end + end + end +end diff --git a/spec/filewatch/spec_helper.rb b/spec/filewatch/spec_helper.rb index f58e767..0b8bfb7 100644 --- a/spec/filewatch/spec_helper.rb +++ b/spec/filewatch/spec_helper.rb @@ -28,6 +28,32 @@ def formatted_puts(text) module FileWatch + class DummyFileReader + def initialize(read_size, iterations) + @read_size = read_size + @iterations = iterations + @closed = false + @accumulated = 0 + end + def file_seek(*) + end + def close() + @closed = true + end + def closed? + @closed + end + def sysread(amount) + @accumulated += amount + if @accumulated > @read_size * @iterations + raise EOFError.new + end + string = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcde\n" + multiplier = amount / string.length + string * multiplier + end + end + FIXTURE_DIR = File.join('spec', 'fixtures') def self.make_file_older(path, seconds) diff --git a/spec/filewatch/tailing_spec.rb b/spec/filewatch/tailing_spec.rb index 091eff0..5e082a6 100644 --- a/spec/filewatch/tailing_spec.rb +++ b/spec/filewatch/tailing_spec.rb @@ -76,7 +76,7 @@ module FileWatch context "when close_older is set" do let(:wait_before_quit) { 0.8 } - let(:opts) { super.merge(:close_older => 0.2, :max_active => 1, :stat_interval => 0.1) } + let(:opts) { super.merge(:close_older => 0.15, :max_active => 1, :stat_interval => 0.1) } it "opens both files" do actions.activate tailing.watch_this(watch_dir) From 28c0534e0d61ed724d867e68c55ad19aa9835b45 Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Wed, 13 Jun 2018 10:41:34 +0100 Subject: [PATCH 32/91] Add string based time duration support. (#194) * add string based time duration support. * redo the returned object as a Struct with a `to_a` method. Fixes #187 --- lib/logstash/inputs/file.rb | 25 ++++++-- lib/logstash/inputs/friendly_durations.rb | 45 ++++++++++++++ spec/inputs/friendly_durations_spec.rb | 71 +++++++++++++++++++++++ 3 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 lib/logstash/inputs/friendly_durations.rb create mode 100644 spec/inputs/friendly_durations_spec.rb diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index d7e068e..81b04b4 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -11,6 +11,7 @@ require_relative "file_listener" require_relative "delete_completed_file_handler" require_relative "log_completed_file_handler" +require_relative "friendly_durations" require "filewatch/bootstrap" # Stream events from files, normally by tailing them in a manner @@ -109,7 +110,7 @@ class File < LogStash::Inputs::Base # How often (in seconds) we stat files to see if they have been modified. # Increasing this interval will decrease the number of system calls we make, # but increase the time to detect new log lines. - config :stat_interval, :validate => :number, :default => 1 + config :stat_interval, :validate => [FriendlyDurations, "seconds"], :default => 1 # How often (in seconds) we expand the filename patterns in the # `path` option to discover new files to watch. @@ -123,7 +124,7 @@ class File < LogStash::Inputs::Base # How often (in seconds) to write a since database with the current position of # monitored log files. - config :sincedb_write_interval, :validate => :number, :default => 15 + config :sincedb_write_interval, :validate => [FriendlyDurations, "seconds"], :default => 15 # Choose where Logstash starts initially reading files: at the beginning or # at the end. The default behavior treats files like live streams and thus @@ -145,7 +146,7 @@ class File < LogStash::Inputs::Base # After its discovery, if an ignored file is modified it is no # longer ignored and any new data is read. By default, this option is # disabled. Note this unit is in seconds. - config :ignore_older, :validate => :number + config :ignore_older, :validate => [FriendlyDurations, "seconds"] # The file input closes any files that were last read the specified # timespan in seconds ago. @@ -154,7 +155,7 @@ class File < LogStash::Inputs::Base # reopening when new data is detected. If reading, the file will be closed # after closed_older seconds from when the last bytes were read. # The default is 1 hour - config :close_older, :validate => :number, :default => 1 * 60 * 60 + config :close_older, :validate => [FriendlyDurations, "seconds"], :default => "1 hour" # What is the maximum number of file_handles that this input consumes # at any one time. Use close_older to close some files if you need to @@ -191,7 +192,7 @@ class File < LogStash::Inputs::Base # If no changes are detected in tracked files in the last N days their sincedb # tracking record will expire and not be persisted. # This option protects against the well known inode recycling problem. (add reference) - config :sincedb_clean_after, :validate => :number, :default => 14 # days + config :sincedb_clean_after, :validate => [FriendlyDurations, "days"], :default => "14 days" # days # File content is read off disk in blocks or chunks, then using whatever the set delimiter # is, lines are extracted from the chunk. Specify the size in bytes of each chunk. @@ -222,6 +223,20 @@ class File < LogStash::Inputs::Base config :file_sort_direction, :validate => ["asc", "desc"], :default => "asc" public + + class << self + alias_method :old_validate_value, :validate_value + + def validate_value(value, validator) + if validator.is_a?(Array) && validator.size == 2 && validator.first.respond_to?(:call) + callable, units = *validator + # returns a ValidatedStruct having a `to_a` method suitable to return to the config mixin caller + return callable.call(value, units).to_a + end + old_validate_value(value, validator) + end + end + def register require "addressable/uri" require "digest/md5" diff --git a/lib/logstash/inputs/friendly_durations.rb b/lib/logstash/inputs/friendly_durations.rb new file mode 100644 index 0000000..07d0ee7 --- /dev/null +++ b/lib/logstash/inputs/friendly_durations.rb @@ -0,0 +1,45 @@ +# encoding: utf-8 + +module LogStash module Inputs + module FriendlyDurations + NUMBERS_RE = /^(?\d+(\.\d+)?)\s?(?s((ec)?(ond)?)(s)?|m((in)?(ute)?)(s)?|h(our)?(s)?|d(ay)?(s)?|w(eek)?(s)?|us(ec)?(s)?|ms(ec)?(s)?)?$/ + HOURS = 3600 + DAYS = 24 * HOURS + MEGA = 10**6 + KILO = 10**3 + + ValidatedStruct = Struct.new(:value, :error_message) do + def to_a + error_message.nil? ? [true, value] : [false, error_message] + end + end + + def self.call(value, unit = "sec") + # coerce into seconds + val_string = value.to_s.strip + matched = NUMBERS_RE.match(val_string) + if matched.nil? + failed_message = "Value '#{val_string}' is not a valid duration string e.g. 200 usec, 250ms, 60 sec, 18h, 21.5d, 1 day, 2w, 6 weeks" + return ValidatedStruct.new(nil, failed_message) + end + multiplier = matched[:units] || unit + numeric = matched[:number].to_f + case multiplier + when "m","min","mins","minute","minutes" + ValidatedStruct.new(numeric * 60, nil) + when "h","hour","hours" + ValidatedStruct.new(numeric * HOURS, nil) + when "d","day","days" + ValidatedStruct.new(numeric * DAYS, nil) + when "w","week","weeks" + ValidatedStruct.new(numeric * 7 * DAYS, nil) + when "ms","msec","msecs" + ValidatedStruct.new(numeric / KILO, nil) + when "us","usec","usecs" + ValidatedStruct.new(numeric / MEGA, nil) + else + ValidatedStruct.new(numeric, nil) + end + end + end +end end diff --git a/spec/inputs/friendly_durations_spec.rb b/spec/inputs/friendly_durations_spec.rb new file mode 100644 index 0000000..586bd86 --- /dev/null +++ b/spec/inputs/friendly_durations_spec.rb @@ -0,0 +1,71 @@ +# encoding: utf-8 + +require "helpers/spec_helper" +require "logstash/inputs/friendly_durations" + +describe "FriendlyDurations module function call" do + context "unacceptable strings" do + it "gives an error message for 'foobar'" do + result = LogStash::Inputs::FriendlyDurations.call("foobar","sec") + expect(result.error_message).to start_with("Value 'foobar' is not a valid duration string e.g. 200 usec") + end + it "gives an error message for '5 5 days'" do + result = LogStash::Inputs::FriendlyDurations.call("5 5 days","sec") + expect(result.error_message).to start_with("Value '5 5 days' is not a valid duration string e.g. 200 usec") + end + end + + context "when a unit is not specified, a unit override will affect the result" do + it "coerces 14 to 1209600.0s as days" do + result = LogStash::Inputs::FriendlyDurations.call(14,"d") + expect(result.error_message).to eq(nil) + expect(result.value).to eq(1209600.0) + end + it "coerces '30' to 1800.0s as minutes" do + result = LogStash::Inputs::FriendlyDurations.call("30","minutes") + expect(result.to_a).to eq([true, 1800.0]) + end + end + + context "acceptable strings" do + [ + ["10", 10.0], + ["10.5 s", 10.5], + ["10.75 secs", 10.75], + ["11 second", 11.0], + ["10 seconds", 10.0], + ["500 ms", 0.5], + ["750.9 msec", 0.7509], + ["750.9 msecs", 0.7509], + ["750.9 us", 0.0007509], + ["750.9 usec", 0.0007509], + ["750.9 usecs", 0.0007509], + ["1.5m", 90.0], + ["2.5 m", 150.0], + ["1.25 min", 75.0], + ["1 minute", 60.0], + ["2.5 minutes", 150.0], + ["2h", 7200.0], + ["2 h", 7200.0], + ["1 hour", 3600.0], + ["1hour", 3600.0], + ["3 hours", 10800.0], + ["0.5d", 43200.0], + ["1day", 86400.0], + ["1 day", 86400.0], + ["2days", 172800.0], + ["14 days", 1209600.0], + ["1w", 604800.0], + ["1 w", 604800.0], + ["1 week", 604800.0], + ["2weeks", 1209600.0], + ["2 weeks", 1209600.0], + ["1.5 weeks", 907200.0], + ].each do |input, coerced| + it "coerces #{input.inspect.rjust(16)} to #{coerced.inspect}" do + result = LogStash::Inputs::FriendlyDurations.call(input,"sec") + expect(result.to_a).to eq([true, coerced]) + end + end + end +end From c3c78471bb522e383af9cfd0a0805eb9e561c2de Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Wed, 13 Jun 2018 18:38:39 +0100 Subject: [PATCH 33/91] Added info about the string duration variant on settings values. Added a changelog entry for string durations. Fixes #197 --- CHANGELOG.md | 4 ++- docs/index.asciidoc | 88 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 71 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 929f0a4..4cc7611 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,9 @@ iteration rather than waiting for the end-of-file to be reached. Note: for gz files, the sincedb entry can only be updated at the end of the file as it is not possible to seek into a compressed file and begin reading from that position. - [Issue #196](https://github.com/logstash-plugins/logstash-input-file/pull/196) + [#196](https://github.com/logstash-plugins/logstash-input-file/pull/196) + - Added support for String Durations in some settings e.g. `stat_interval => "750 ms"` + [#194](https://github.com/logstash-plugins/logstash-input-file/pull/194) ## 4.1.2 - Fix `require winhelper` error in WINDOWS. diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 0689137..aab3dab 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -146,10 +146,15 @@ will not get picked up. This plugin supports the following configuration options plus the <> described later. +[NOTE] +Duration settings can be specified in text form e.g. "250 ms", this string will be converted into +decimal seconds. There are quite a few supported natural and abbreviated durations, +see <> for the details. + [cols="<,<,<",options="header",] |======================================================================= |Setting |Input type|Required -| <> |<>|No +| <> |<> or <>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -159,15 +164,15 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>, one of `["last_modified", "path"]`|No | <> |<>, one of `["asc", "desc"]`|No -| <> |<>|No +| <> |<> or <>|No | <> |<>|No | <> |<>, one of `["tail", "read"]`|No | <> |<>|Yes -| <> |<>|No +| <> |<> or <>|No | <> |<>|No -| <> |<>|No +| <> |<> or <>|No | <> |<>, one of `["beginning", "end"]`|No -| <> |<>|No +| <> |<> or <>|No |======================================================================= Also see <> for a list of options supported by all @@ -178,18 +183,18 @@ input plugins. [id="plugins-{type}s-{plugin}-close_older"] ===== `close_older` - * Value type is <> - * Default value is `3600` + * Value type is <> or <> + * Default value is `"1 hour"` The file input closes any files that were last read the specified -timespan in seconds ago. +duration (seconds if a number is specified) ago. This has different implications depending on if a file is being tailed or read. If tailing, and there is a large time gap in incoming data the file can be closed (allowing other files to be opened) but will be queued for reopening when new data is detected. If reading, the file will be closed after closed_older seconds from when the last bytes were read. This setting is retained for backward compatibility if you upgrade the -plugin to 5.0.0+, are reading not tailing and do not switch to using Read mode. +plugin to 4.1.0+, are reading not tailing and do not switch to using Read mode. [id="plugins-{type}s-{plugin}-delimiter"] ===== `delimiter` @@ -206,8 +211,10 @@ this setting is not used, instead the standard Windows or Unix line endings are * Value type is <> * Default value is `15` -How often (in seconds) we expand the filename patterns in the -`path` option to discover new files to watch. +How often we expand the filename patterns in the `path` option to discover new files to watch. +This value is a multiple to `stat_interval`, e.g. if `stat_interval` is "500 ms" then new files +files could be discovered every 15 X 500 milliseconds - 7.5 seconds. +In practice, this will be the best case because the time taken to read new content needs to be factored in. [id="plugins-{type}s-{plugin}-exclude"] ===== `exclude` @@ -294,11 +301,11 @@ If you use special naming conventions for the file full paths then perhaps [id="plugins-{type}s-{plugin}-ignore_older"] ===== `ignore_older` - * Value type is <> + * Value type is <> or <> * There is no default value for this setting. When the file input discovers a file that was last modified -before the specified timespan in seconds, the file is ignored. +before the specified duration (seconds if a number is specified), the file is ignored. After it's discovery, if an ignored file is modified it is no longer ignored and any new data is read. By default, this option is disabled. Note this unit is in seconds. @@ -354,9 +361,9 @@ on the {logstash-ref}/configuration-file-structure.html#array[Logstash configura [id="plugins-{type}s-{plugin}-sincedb_clean_after"] ===== `sincedb_clean_after` - * Value type is <> - * The default value for this setting is 14. - * This unit is in *days* and can be decimal e.g. 0.5 is 12 hours. + * Value type is <> or <> + * The default value for this setting is "2 weeks". + * If a number is specified then it is interpreted as *days* and can be decimal e.g. 0.5 is 12 hours. The sincedb record now has a last active timestamp associated with it. If no changes are detected in a tracked file in the last N days its sincedb @@ -378,8 +385,8 @@ NOTE: it must be a file path and not a directory path [id="plugins-{type}s-{plugin}-sincedb_write_interval"] ===== `sincedb_write_interval` - * Value type is <> - * Default value is `15` + * Value type is <> or <> + * Default value is `"15 seconds"` How often (in seconds) to write a since database with the current position of monitored log files. @@ -404,15 +411,56 @@ position recorded in the sincedb file will be used. [id="plugins-{type}s-{plugin}-stat_interval"] ===== `stat_interval` - * Value type is <> - * Default value is `1` + * Value type is <> or <> + * Default value is `"1 second"` How often (in seconds) we stat files to see if they have been modified. Increasing this interval will decrease the number of system calls we make, but increase the time to detect new log lines. +[NOTE] +Discovering new files and checking whether they have grown/or shrunk occurs in a loop. +This loop will sleep for `stat_interval` seconds before looping again. However, if files +have grown, the new content is read and lines are enqueued. +Reading and enqueuing across all grown files can take time, especially if +the pipeline is congested. So the overall loop time is a combination of the +`stat_interval` and the file read time. [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] :default_codec!: +[id="string_duration"] +// Move this to the includes when we make string durations available generally. +==== String Durations + +Format is `number` `string` and the space between these is optional. +So "45s" and "45 s" are both valid. +[TIP] +Use the most suitable duration, for example, "3 days" rather than "72 hours". + +===== Weeks +Supported values: `w` `week` `weeks`, e.g. "2 w", "1 week", "4 weeks". + +===== Days +Supported values: `d` `day` `days`, e.g. "2 d", "1 day", "2.5 days". + +===== Hours +Supported values: `h` `hour` `hours`, e.g. "4 h", "1 hour", "0.5 hours". + +===== Minutes +Supported values: `m` `min` `minute` `minutes`, e.g. "45 m", "35 min", "1 minute", "6 minutes". + +===== Seconds +Supported values: `s` `sec` `second` `seconds`, e.g. "45 s", "15 sec", "1 second", "2.5 seconds". + +===== Milliseconds +Supported values: `ms` `msec` `msecs`, e.g. "500 ms", "750 msec", "50 msecs +[NOTE] +`milli` `millis` and `milliseconds` are not supported + +===== Microseconds +Supported values: `us` `usec` `usecs`, e.g. "600 us", "800 usec", "900 usecs" +[NOTE] +`micro` `micros` and `microseconds` are not supported + From 498eb5e8c84e8b68201e731f271fb7bbd445c0ae Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Thu, 5 Jul 2018 19:04:58 +0100 Subject: [PATCH 34/91] Add better support for file rotation (#192) * Add better support for file rotation * Move common_restat for watched and active to one iteration to better handle when a file is rotated that has never been active. Change the way @previous_inode_size is cached, it needs to be set only when an inode change is detected. Change discover associate behaviour if the matched watched_file is open. Add spec for when a file is rotated that has never been active. Alter specs because we can rely on ordering with "file_sort_by" * Ensure single file `path` option is ok, reverts earlier change in this PR * Improve and expand our use of FFI on Windows * jnr and ffi interop, yay * put self back in * use fieldId label * handle rotations better. especially in the case where the rotated files are discoverable - in this case we were not the moving the state and the sincedb record correctly. Next - abstract the *nix and Windows stat calls and structure into two classes with the same API. * fixes for rebase from master * Abstract Stat part 1 * Abstract Stat part 2 * Finally have (all) the kinks worked out * Try to fix travis failures. * Try to fix travis failures 2 * Try fix travis 3 * Try fix travis 4 * Try fix travis 5 * Try fix travis 6 * Remove io based stat reliance. travis jruby 1.7.27 should pass now. Move all tail processing state iteration into its own method Clean up rspec sequencing usage. Remove all Mutex use in favour of AtomicBoolean, AtomicArray, AtomicHash Add wait for completely_stopped before removing sincedb in file_tail spec * Some windows fixes * more windows fixes * windows changes 2 * rename rspec run tag in ci/build.sh * move one trace logging line * add first run discovery methods * fix regression on files seen after inital run, travis 2 use docker. * add execute permissions * fix path ordering travis failures * fix jar loading so it works for tests and when installed in LS * reorder the jar require statements Fixes #198 --- .travis.yml | 26 +- JAR_VERSION | 2 +- README.md | 3 - ci/build.sh | 21 - ci/setup.sh | 26 - ci/unit/Dockerfile | 11 + ci/unit/docker-compose.yml | 17 + ci/unit/docker-run.sh | 7 + ci/unit/docker-setup.sh | 31 + ci/unit/run.sh | 6 + lib/filewatch/bootstrap.rb | 30 +- lib/filewatch/discoverer.rb | 63 ++- lib/filewatch/observing_base.rb | 3 +- lib/filewatch/read_mode/handlers/base.rb | 25 +- lib/filewatch/read_mode/handlers/read_file.rb | 22 +- .../read_mode/handlers/read_zip_file.rb | 11 +- lib/filewatch/read_mode/processor.rb | 16 +- lib/filewatch/settings.rb | 2 +- lib/filewatch/sincedb_collection.rb | 98 ++-- lib/filewatch/sincedb_value.rb | 6 + lib/filewatch/stat/generic.rb | 34 ++ lib/filewatch/stat/windows_path.rb | 32 ++ lib/filewatch/tail_mode/handlers/base.rb | 60 +- lib/filewatch/tail_mode/handlers/create.rb | 3 +- .../tail_mode/handlers/create_initial.rb | 3 +- lib/filewatch/tail_mode/handlers/delete.rb | 14 +- lib/filewatch/tail_mode/handlers/grow.rb | 1 - lib/filewatch/tail_mode/handlers/shrink.rb | 8 +- lib/filewatch/tail_mode/handlers/unignore.rb | 6 +- lib/filewatch/tail_mode/processor.rb | 205 +++++-- lib/filewatch/watch.rb | 50 +- lib/filewatch/watched_file.rb | 250 +++++++-- lib/filewatch/watched_files_collection.rb | 4 +- lib/filewatch/winhelper.rb | 192 ++++++- lib/logstash/inputs/file.rb | 11 +- logstash-input-file.gemspec | 11 +- run_until_fail.sh | 4 + spec/file_ext/file_ext_windows_spec.rb | 36 ++ .../read_mode_handlers_read_file_spec.rb | 2 +- spec/filewatch/reading_spec.rb | 157 ++++-- spec/filewatch/rotate_spec.rb | 451 +++++++++++++++ spec/filewatch/spec_helper.rb | 43 +- spec/filewatch/tailing_spec.rb | 424 +++++++++----- spec/filewatch/watched_file_spec.rb | 6 +- .../watched_files_collection_spec.rb | 6 +- spec/filewatch/winhelper_spec.rb | 9 +- spec/helpers/logging_level_helper.rb | 8 + spec/helpers/rspec_wait_handler_helper.rb | 38 ++ spec/helpers/spec_helper.rb | 8 +- spec/inputs/file_read_spec.rb | 78 ++- spec/inputs/file_tail_spec.rb | 530 ++++++++---------- .../WindowsFileInformationByHandle.java | 24 + .../filewatch/JrubyFileWatchLibrary.java | 164 +++++- .../org/logstash/filewatch/RubyWinIO.java | 65 +++ 54 files changed, 2417 insertions(+), 946 deletions(-) delete mode 100755 ci/build.sh delete mode 100755 ci/setup.sh create mode 100644 ci/unit/Dockerfile create mode 100644 ci/unit/docker-compose.yml create mode 100755 ci/unit/docker-run.sh create mode 100755 ci/unit/docker-setup.sh create mode 100755 ci/unit/run.sh create mode 100644 lib/filewatch/stat/generic.rb create mode 100644 lib/filewatch/stat/windows_path.rb create mode 100755 run_until_fail.sh create mode 100644 spec/file_ext/file_ext_windows_spec.rb create mode 100644 spec/filewatch/rotate_spec.rb create mode 100644 spec/helpers/logging_level_helper.rb create mode 100644 spec/helpers/rspec_wait_handler_helper.rb create mode 100644 src/main/java/jnr/posix/windows/WindowsFileInformationByHandle.java create mode 100644 src/main/java/org/logstash/filewatch/RubyWinIO.java diff --git a/.travis.yml b/.travis.yml index 1458a3b..b5cfb80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,16 @@ --- -sudo: false -language: ruby -cache: bundler +sudo: required +services: docker +addons: + apt: + packages: + - docker-ce matrix: include: - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=master - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.x - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.0 - - rvm: jruby-1.7.27 - env: LOGSTASH_BRANCH=5.6 + - env: ELASTIC_STACK_VERSION=5.6.10 + - env: ELASTIC_STACK_VERSION=6.3.0 + - env: ELASTIC_STACK_VERSION=6.4.0-SNAPSHOT + - env: ELASTIC_STACK_VERSION=7.0.0-alpha1-SNAPSHOT fast_finish: true -install: true -script: ci/build.sh -jdk: oraclejdk8 +install: ci/unit/docker-setup.sh +script: ci/unit/docker-run.sh diff --git a/JAR_VERSION b/JAR_VERSION index 3eefcb9..7dea76e 100644 --- a/JAR_VERSION +++ b/JAR_VERSION @@ -1 +1 @@ -1.0.0 +1.0.1 diff --git a/README.md b/README.md index a48c73a..5153942 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,6 @@ Travis Build [![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-file.svg)](https://travis-ci.org/logstash-plugins/logstash-input-file) -Jenkins Build -[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-file.svg)](https://travis-ci.org/logstash-plugins/logstash-input-file) - This is a plugin for [Logstash](https://github.com/elastic/logstash). It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. diff --git a/ci/build.sh b/ci/build.sh deleted file mode 100755 index 06caffd..0000000 --- a/ci/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# version: 1 -######################################################## -# -# AUTOMATICALLY GENERATED! DO NOT EDIT -# -######################################################## -set -e - -echo "Starting build process in: `pwd`" -source ./ci/setup.sh - -if [[ -f "ci/run.sh" ]]; then - echo "Running custom build script in: `pwd`/ci/run.sh" - source ./ci/run.sh -else - echo "Running default build scripts in: `pwd`/ci/build.sh" - bundle install - bundle exec rake vendor - bundle exec rspec spec -fi diff --git a/ci/setup.sh b/ci/setup.sh deleted file mode 100755 index 835fa43..0000000 --- a/ci/setup.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# version: 1 -######################################################## -# -# AUTOMATICALLY GENERATED! DO NOT EDIT -# -######################################################## -set -e -if [ "$LOGSTASH_BRANCH" ]; then - echo "Building plugin using Logstash source" - BASE_DIR=`pwd` - echo "Checking out branch: $LOGSTASH_BRANCH" - git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1 - printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)" - cd ../../logstash - echo "Building plugins with Logstash version:" - cat versions.yml - echo "---" - # We need to build the jars for that specific version - echo "Running gradle assemble in: `pwd`" - ./gradlew assemble - cd $BASE_DIR - export LOGSTASH_SOURCE=1 -else - echo "Building plugin using released gems on rubygems" -fi diff --git a/ci/unit/Dockerfile b/ci/unit/Dockerfile new file mode 100644 index 0000000..b2fc1c6 --- /dev/null +++ b/ci/unit/Dockerfile @@ -0,0 +1,11 @@ +ARG ELASTIC_STACK_VERSION +FROM docker.elastic.co/logstash/logstash:$ELASTIC_STACK_VERSION +WORKDIR /usr/share/logstash/logstash-core +RUN cp versions-gem-copy.yml ../logstash-core-plugin-api/versions-gem-copy.yml +COPY --chown=logstash:logstash . /usr/share/plugins/this +WORKDIR /usr/share/plugins/this +ENV PATH=/usr/share/logstash/vendor/jruby/bin:${PATH} +ENV LOGSTASH_SOURCE 1 +RUN jruby -S gem install bundler +RUN jruby -S bundle install --jobs=3 --retry=3 +RUN jruby -S bundle exec rake vendor diff --git a/ci/unit/docker-compose.yml b/ci/unit/docker-compose.yml new file mode 100644 index 0000000..1dd26fd --- /dev/null +++ b/ci/unit/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3' + +# run tests: docker-compose -f ci/unit/docker-compose.yml up --build --force-recreate +# only set up: docker-compose -f ci/unit/docker-compose.yml up --build --no-start --force-recreate +# start manually: docker-compose -f ci/unit/docker-compose.yml run logstash +services: + logstash: + build: + context: ../../ + dockerfile: ci/unit/Dockerfile + args: + - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION + command: /usr/share/plugins/this/ci/unit/run.sh + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + OSS: "true" + tty: true diff --git a/ci/unit/docker-run.sh b/ci/unit/docker-run.sh new file mode 100755 index 0000000..4007c55 --- /dev/null +++ b/ci/unit/docker-run.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh` +# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable. +set -e + +docker-compose -f ci/unit/docker-compose.yml run logstash diff --git a/ci/unit/docker-setup.sh b/ci/unit/docker-setup.sh new file mode 100755 index 0000000..37bbcd0 --- /dev/null +++ b/ci/unit/docker-setup.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh` +# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable. +set -e + +if [ "$ELASTIC_STACK_VERSION" ]; then + echo "Testing against version: $ELASTIC_STACK_VERSION" + + if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then + cd /tmp + wget https://snapshots.elastic.co/docker/logstash-"$ELASTIC_STACK_VERSION".tar.gz + tar xfvz logstash-"$ELASTIC_STACK_VERSION".tar.gz repositories + echo "Loading docker image: " + cat repositories + docker load < logstash-"$ELASTIC_STACK_VERSION".tar.gz + rm logstash-"$ELASTIC_STACK_VERSION".tar.gz + cd - + fi + + if [ -f Gemfile.lock ]; then + rm Gemfile.lock + fi + + docker-compose -f ci/unit/docker-compose.yml down + docker-compose -f ci/unit/docker-compose.yml up --no-start --build --force-recreate logstash +else + echo "Please set the ELASTIC_STACK_VERSION environment variable" + echo "For example: export ELASTIC_STACK_VERSION=6.2.4" + exit 1 +fi diff --git a/ci/unit/run.sh b/ci/unit/run.sh new file mode 100755 index 0000000..91e54bb --- /dev/null +++ b/ci/unit/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# This is intended to be run inside the docker container as the command of the docker-compose. +set -ex + +bundle exec rspec -fd --pattern spec/**/*_spec.rb,spec/**/*_specs.rb diff --git a/lib/filewatch/bootstrap.rb b/lib/filewatch/bootstrap.rb index 99c196d..51e9b37 100644 --- a/lib/filewatch/bootstrap.rb +++ b/lib/filewatch/bootstrap.rb @@ -1,7 +1,5 @@ # encoding: utf-8 -require "rbconfig" require "pathname" -# require "logstash/environment" ## Common setup # all the required constants and files @@ -13,36 +11,26 @@ module FileWatch # this is used in the read loop e.g. # @opts[:file_chunk_count].times do # where file_chunk_count defaults to this constant - FIXNUM_MAX = (2**(0.size * 8 - 2) - 1) + MAX_ITERATIONS = (2**(0.size * 8 - 2) - 2) / 32768 require_relative "helper" - module WindowsInode - def prepare_inode(path, stat) - fileId = Winhelper.GetWindowsUniqueFileIdentifier(path) - [fileId, 0, 0] # dev_* doesn't make sense on Windows - end - end - - module UnixInode - def prepare_inode(path, stat) - [stat.ino.to_s, stat.dev_major, stat.dev_minor] - end - end - - jar_version = Pathname.new(__FILE__).dirname.join("../../JAR_VERSION").realpath.read.strip - + gem_root_dir = Pathname.new(__FILE__).dirname.join("../../").realpath + jar_version = gem_root_dir.join("JAR_VERSION").read.strip + fullpath = gem_root_dir.join("lib/jars/filewatch-#{jar_version}.jar").expand_path.to_path require "java" - require_relative "../../lib/jars/filewatch-#{jar_version}.jar" + require fullpath require "jruby_file_watch" if LogStash::Environment.windows? require_relative "winhelper" + require_relative "stat/windows_path" + PathStatClass = Stat::WindowsPath FileOpener = FileExt - InodeMixin = WindowsInode else + require_relative "stat/generic" + PathStatClass = Stat::Generic FileOpener = ::File - InodeMixin = UnixInode end # Structs can be used as hash keys because they compare by value diff --git a/lib/filewatch/discoverer.rb b/lib/filewatch/discoverer.rb index b98f77d..f73b1a3 100644 --- a/lib/filewatch/discoverer.rb +++ b/lib/filewatch/discoverer.rb @@ -10,8 +10,8 @@ class Discoverer include LogStash::Util::Loggable def initialize(watched_files_collection, sincedb_collection, settings) - @watching = [] - @exclude = [] + @watching = Concurrent::Array.new + @exclude = Concurrent::Array.new @watched_files_collection = watched_files_collection @sincedb_collection = sincedb_collection @settings = settings @@ -21,13 +21,13 @@ def initialize(watched_files_collection, sincedb_collection, settings) def add_path(path) return if @watching.member?(path) @watching << path - discover_files(path) + discover_files_new_path(path) self end def discover @watching.each do |path| - discover_files(path) + discover_files_ongoing(path) end end @@ -37,7 +37,7 @@ def can_exclude?(watched_file, new_discovery) @exclude.each do |pattern| if watched_file.pathname.fnmatch?(pattern) if new_discovery - logger.debug("Discoverer can_exclude?: #{watched_file.path}: skipping " + + logger.trace("Discoverer can_exclude?: #{watched_file.path}: skipping " + "because it matches exclude #{pattern}") end watched_file.unwatch @@ -47,45 +47,52 @@ def can_exclude?(watched_file, new_discovery) false end - def discover_files(path) - globbed = Dir.glob(path) - globbed = [path] if globbed.empty? - logger.debug("Discoverer found files, count: #{globbed.size}") - globbed.each do |file| - logger.debug("Discoverer found file, path: #{file}") + def discover_files_new_path(path) + discover_any_files(path, false) + end + + def discover_files_ongoing(path) + discover_any_files(path, true) + end + + def discover_any_files(path, ongoing) + fileset = Dir.glob(path).select{|f| File.file?(f) && !File.symlink?(f)} + logger.trace("discover_files", "count" => fileset.size) + fileset.each do |file| pathname = Pathname.new(file) - next unless pathname.file? - next if pathname.symlink? new_discovery = false watched_file = @watched_files_collection.watched_file_by_path(file) if watched_file.nil? - logger.debug("Discoverer discover_files: #{path}: new: #{file} (exclude is #{@exclude.inspect})") new_discovery = true - watched_file = WatchedFile.new(pathname, pathname.stat, @settings) + watched_file = WatchedFile.new(pathname, PathStatClass.new(pathname), @settings) end # if it already unwatched or its excluded then we can skip next if watched_file.unwatched? || can_exclude?(watched_file, new_discovery) + logger.trace("discover_files handling:", "new discovery"=> new_discovery, "watched_file details" => watched_file.details) + if new_discovery - if watched_file.file_ignorable? - logger.debug("Discoverer discover_files: #{file}: skipping because it was last modified more than #{@settings.ignore_older} seconds ago") - # on discovery ignorable watched_files are put into the ignored state and that - # updates the size from the internal stat - # so the existing contents are not read. - # because, normally, a newly discovered file will - # have a watched_file size of zero - # they are still added to the collection so we know they are there for the next periodic discovery - watched_file.ignore - end - # now add the discovered file to the watched_files collection and adjust the sincedb collections - @watched_files_collection.add(watched_file) + watched_file.initial_completed if ongoing # initially when the sincedb collection is filled with records from the persistence file # each value is not associated with a watched file # a sincedb_value can be: # unassociated # associated with this watched_file # associated with a different watched_file - @sincedb_collection.associate(watched_file) + if @sincedb_collection.associate(watched_file) + if watched_file.file_ignorable? + logger.trace("Discoverer discover_files: #{file}: skipping because it was last modified more than #{@settings.ignore_older} seconds ago") + # on discovery ignorable watched_files are put into the ignored state and that + # updates the size from the internal stat + # so the existing contents are not read. + # because, normally, a newly discovered file will + # have a watched_file size of zero + # they are still added to the collection so we know they are there for the next periodic discovery + watched_file.ignore_as_unread + end + # now add the discovered file to the watched_files collection and adjust the sincedb collections + @watched_files_collection.add(watched_file) + end end # at this point the watched file is created, is in the db but not yet opened or being processed end diff --git a/lib/filewatch/observing_base.rb b/lib/filewatch/observing_base.rb index 387ba77..7f9be79 100644 --- a/lib/filewatch/observing_base.rb +++ b/lib/filewatch/observing_base.rb @@ -44,7 +44,8 @@ def initialize(opts={}) :exclude => [], :start_new_files_at => :end, :delimiter => "\n", - :file_chunk_count => FIXNUM_MAX, + :file_chunk_count => MAX_ITERATIONS, + :file_chunk_size => FILE_READ_SIZE, :file_sort_by => "last_modified", :file_sort_direction => "asc", }.merge(opts) diff --git a/lib/filewatch/read_mode/handlers/base.rb b/lib/filewatch/read_mode/handlers/base.rb index 8146808..6a086d8 100644 --- a/lib/filewatch/read_mode/handlers/base.rb +++ b/lib/filewatch/read_mode/handlers/base.rb @@ -19,7 +19,7 @@ def quit? end def handle(watched_file) - logger.debug("handling: #{watched_file.path}") + logger.trace("handling: #{watched_file.path}") unless watched_file.has_listener? watched_file.set_listener(@observer) end @@ -34,7 +34,7 @@ def handle_specifically(watched_file) def open_file(watched_file) return true if watched_file.file_open? - logger.debug("opening #{watched_file.path}") + logger.trace("opening #{watched_file.path}") begin watched_file.open rescue @@ -46,7 +46,7 @@ def open_file(watched_file) logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}") watched_file.last_open_warning_at = now else - logger.debug("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}") + logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}") end watched_file.watch # set it back to watch so we can try it again end @@ -65,13 +65,26 @@ def add_or_update_sincedb_collection(watched_file) elsif sincedb_value.watched_file == watched_file update_existing_sincedb_collection_value(watched_file, sincedb_value) else - logger.warn? && logger.warn("mismatch on sincedb_value.watched_file, this should have been handled by Discoverer") + msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file" + logger.trace(msg) + existing_watched_file = sincedb_value.watched_file + if existing_watched_file.nil? + sincedb_value.set_watched_file(watched_file) + logger.trace("add_or_update_sincedb_collection: switching as new file") + watched_file.rotate_as_file + watched_file.update_bytes_read(sincedb_value.position) + else + sincedb_value.set_watched_file(watched_file) + logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details) + watched_file.rotate_from(existing_watched_file) + end + end watched_file.initial_completed end def update_existing_sincedb_collection_value(watched_file, sincedb_value) - logger.debug("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}") + logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}") # sincedb_value is the source of truth watched_file.update_bytes_read(sincedb_value.position) end @@ -79,7 +92,7 @@ def update_existing_sincedb_collection_value(watched_file, sincedb_value) def add_new_value_sincedb_collection(watched_file) sincedb_value = SincedbValue.new(0) sincedb_value.set_watched_file(watched_file) - logger.debug("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position) + logger.trace("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position) sincedb_collection.set(watched_file.sincedb_key, sincedb_value) end end diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index 818e0f7..9219792 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -5,20 +5,21 @@ class ReadFile < Base def handle_specifically(watched_file) if open_file(watched_file) add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) - @settings.file_chunk_count.times do + changed = false + logger.trace("reading...", "amount" => watched_file.read_bytesize_description, "filename" => watched_file.filename) + watched_file.read_loop_count.times do break if quit? begin - data = watched_file.file_read(@settings.file_chunk_size) - result = watched_file.buffer_extract(data) # expect BufferExtractResult - logger.info(result.warning, result.additional) unless result.warning.empty? + # expect BufferExtractResult + result = watched_file.read_extract_lines + # read_extract_lines will increment bytes_read + logger.trace(result.warning, result.additional) unless result.warning.empty? + changed = true result.lines.each do |line| watched_file.listener.accept(line) # sincedb position is independent from the watched_file bytes_read sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size) end - # instead of tracking the bytes_read line by line we need to track by the data read size. - # because we initially seek to the bytes_read not the sincedb position - watched_file.increment_bytes_read(data.bytesize) sincedb_collection.request_disk_flush rescue EOFError # flush the buffer now in case there is no final delimiter @@ -26,8 +27,9 @@ def handle_specifically(watched_file) watched_file.listener.accept(line) unless line.empty? watched_file.listener.eof watched_file.file_close - # unset_watched_file will set sincedb_value.position to be watched_file.bytes_read - sincedb_collection.unset_watched_file(watched_file) + key = watched_file.sincedb_key + sincedb_collection.reading_completed(key) + sincedb_collection.clear_watched_file(key) watched_file.listener.deleted watched_file.unwatch break @@ -35,7 +37,7 @@ def handle_specifically(watched_file) watched_file.listener.error break rescue => e - logger.error("read_to_eof: general error reading #{watched_file.path} - error: #{e.inspect}") + logger.error("read_to_eof: general error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) watched_file.listener.error break end diff --git a/lib/filewatch/read_mode/handlers/read_zip_file.rb b/lib/filewatch/read_mode/handlers/read_zip_file.rb index a0d7fb8..3a0f764 100644 --- a/lib/filewatch/read_mode/handlers/read_zip_file.rb +++ b/lib/filewatch/read_mode/handlers/read_zip_file.rb @@ -13,6 +13,11 @@ def handle_specifically(watched_file) add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) # can't really stripe read a zip file, its all or nothing. watched_file.listener.opened + # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading) + # should we track lines read in the sincedb and + # fast forward through the lines until we reach unseen content? + # meaning that we can quit in the middle of a zip file + key = watched_file.sincedb_key begin file_stream = FileInputStream.new(watched_file.path) gzip_stream = GZIPInputStream.new(file_stream) @@ -31,8 +36,8 @@ def handle_specifically(watched_file) logger.error("Cannot decompress the gzip file at path: #{watched_file.path}") watched_file.listener.error else - watched_file.update_bytes_read(watched_file.last_stat_size) - sincedb_collection.unset_watched_file(watched_file) + sincedb_collection.store_last_read(key, watched_file.last_stat_size) + sincedb_collection.request_disk_flush watched_file.listener.deleted watched_file.unwatch ensure @@ -42,7 +47,7 @@ def handle_specifically(watched_file) close_and_ignore_ioexception(gzip_stream) unless gzip_stream.nil? close_and_ignore_ioexception(file_stream) unless file_stream.nil? end - sincedb_collection.unset_watched_file(watched_file) + sincedb_collection.clear_watched_file(key) end private diff --git a/lib/filewatch/read_mode/processor.rb b/lib/filewatch/read_mode/processor.rb index 90ca05b..88e8505 100644 --- a/lib/filewatch/read_mode/processor.rb +++ b/lib/filewatch/read_mode/processor.rb @@ -39,16 +39,16 @@ def read_zip_file(watched_file) @read_zip_file.handle(watched_file) end - def process_closed(watched_files) - # do not process watched_files in the closed state. + def process_all_states(watched_files) + process_watched(watched_files) + return if watch.quit? + process_active(watched_files) end - def process_ignored(watched_files) - # do not process watched_files in the ignored state. - end + private def process_watched(watched_files) - logger.debug("Watched processing") + logger.trace("Watched processing") # Handles watched_files in the watched state. # for a slice of them: # move to the active state @@ -81,7 +81,7 @@ def process_watched(watched_files) end def process_active(watched_files) - logger.debug("Active processing") + logger.trace("Active processing") # Handles watched_files in the active state. watched_files.select {|wf| wf.active? }.each do |watched_file| path = watched_file.path @@ -109,7 +109,7 @@ def common_deleted_reaction(watched_file, action) # file has gone away or we can't read it anymore. watched_file.unwatch deletable_filepaths << watched_file.path - logger.debug("#{action} - stat failed: #{watched_file.path}, removing from collection") + logger.trace("#{action} - stat failed: #{watched_file.path}, removing from collection") end def common_error_reaction(path, error, action) diff --git a/lib/filewatch/settings.rb b/lib/filewatch/settings.rb index 5d7b61f..1ec08e9 100644 --- a/lib/filewatch/settings.rb +++ b/lib/filewatch/settings.rb @@ -22,7 +22,7 @@ def initialize :delimiter => "\n", :file_chunk_size => FILE_READ_SIZE, :max_active => 4095, - :file_chunk_count => FIXNUM_MAX, + :file_chunk_count => MAX_ITERATIONS, :sincedb_clean_after => 14, :exclude => [], :stat_interval => 1, diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb index 90b59b0..78c4e12 100644 --- a/lib/filewatch/sincedb_collection.rb +++ b/lib/filewatch/sincedb_collection.rb @@ -39,7 +39,7 @@ def write_if_requested end def write(reason=nil) - logger.debug("caller requested sincedb write (#{reason})") + logger.trace("caller requested sincedb write (#{reason})") sincedb_write end @@ -47,73 +47,72 @@ def open @time_sdb_opened = Time.now.to_f begin path.open do |file| - logger.debug("open: reading from #{path}") + logger.trace("open: reading from #{path}") @serializer.deserialize(file) do |key, value| - logger.debug("open: importing ... '#{key}' => '#{value}'") + logger.trace("open: importing ... '#{key}' => '#{value}'") set_key_value(key, value) end end - logger.debug("open: count of keys read: #{@sincedb.keys.size}") + logger.trace("open: count of keys read: #{@sincedb.keys.size}") rescue => e #No existing sincedb to load - logger.debug("open: error: #{path}: #{e.inspect}") + logger.trace("open: error: #{path}: #{e.inspect}") end end def associate(watched_file) - logger.debug("associate: finding: #{watched_file.path}") + logger.trace("associate: finding", "inode" => watched_file.sincedb_key.inode, "path" => watched_file.path) sincedb_value = find(watched_file) if sincedb_value.nil? # sincedb has no record of this inode # and due to the window handling of many files # this file may not be opened in this session. # a new value will be added when the file is opened - return + logger.trace("associate: unmatched") + return true end + logger.trace("associate: found sincedb record", "filename" => watched_file.filename, "sincedb key" => watched_file.sincedb_key,"sincedb_value" => sincedb_value) if sincedb_value.watched_file.nil? # not associated if sincedb_value.path_in_sincedb.nil? - # old v1 record, assume its the same file handle_association(sincedb_value, watched_file) - return + logger.trace("associate: inode matched but no path in sincedb") + return true end if sincedb_value.path_in_sincedb == watched_file.path # the path on disk is the same as discovered path # and the inode is the same. handle_association(sincedb_value, watched_file) - return + logger.trace("associate: inode and path matched") + return true end # the path on disk is different from discovered unassociated path # but they have the same key (inode) # treat as a new file, a new value will be added when the file is opened - logger.debug("associate: matched but allocated to another - #{sincedb_value}") sincedb_value.clear_watched_file delete(watched_file.sincedb_key) - return + logger.trace("associate: matched but allocated to another") + return true end if sincedb_value.watched_file.equal?(watched_file) # pointer equals - logger.debug("associate: already associated - #{sincedb_value}, for path: #{watched_file.path}") - return + logger.trace("associate: already associated") + return true end - # sincedb_value.watched_file is not the discovered watched_file but they have the same key (inode) - # this means that the filename was changed during this session. - # logout the history of the old sincedb_value and remove it - # a new value will be added when the file is opened - # TODO notify about done-ness of old sincedb_value and watched_file - old_watched_file = sincedb_value.watched_file - sincedb_value.clear_watched_file - if logger.debug? - logger.debug("associate: matched but allocated to another - #{sincedb_value}") - logger.debug("associate: matched but allocated to another - old watched_file history - #{old_watched_file.recent_state_history.join(', ')}") - logger.debug("associate: matched but allocated to another - DELETING value at key `#{old_watched_file.sincedb_key}`") - end - delete(old_watched_file.sincedb_key) + # sincedb_value.watched_file is not this discovered watched_file but they have the same key (inode) + # this means that the filename path was changed during this session. + # renamed file can be discovered... + # before the original is detected as deleted: state is `active` + # after the original is detected as deleted but before it is actually deleted: state is `delayed_delete` + # after the original is deleted + # are not yet in the delete phase, let this play out + existing_watched_file = sincedb_value.watched_file + logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename", "this watched_file details" => watched_file.details, "other watched_file details" => existing_watched_file.details) + watched_file.rotation_in_progress + true end def find(watched_file) - get(watched_file.sincedb_key).tap do |obj| - logger.debug("find for path: #{watched_file.path}, found: '#{!obj.nil?}'") - end + get(watched_file.sincedb_key) end def member?(key) @@ -124,6 +123,11 @@ def get(key) @sincedb[key] end + def set(key, value) + @sincedb[key] = value + value + end + def delete(key) @sincedb.delete(key) end @@ -144,11 +148,23 @@ def set_watched_file(key, watched_file) @sincedb[key].set_watched_file(watched_file) end - def unset_watched_file(watched_file) + def watched_file_deleted(watched_file) return unless member?(watched_file.sincedb_key) get(watched_file.sincedb_key).unset_watched_file end + def store_last_read(key, pos) + @sincedb[key].update_position(pos) + end + + def clear_watched_file(key) + @sincedb[key].clear_watched_file + end + + def reading_completed(key) + @sincedb[key].reading_completed + end + def clear @sincedb.clear end @@ -157,11 +173,6 @@ def keys @sincedb.keys end - def set(key, value) - @sincedb[key] = value - value - end - def watched_file_unset?(key) return false unless member?(key) get(key).watched_file.nil? @@ -182,33 +193,36 @@ def handle_association(sincedb_value, watched_file) watched_file.update_bytes_read(sincedb_value.position) sincedb_value.set_watched_file(watched_file) watched_file.initial_completed - watched_file.ignore if watched_file.all_read? + if watched_file.all_read? + watched_file.ignore + logger.trace("handle_association fully read, ignoring.....", "watched file" => watched_file.details, "sincedb value" => sincedb_value) + end end def set_key_value(key, value) if @time_sdb_opened < value.last_changed_at_expires(@settings.sincedb_expiry_duration) - logger.debug("open: setting #{key.inspect} to #{value.inspect}") + logger.trace("open: setting #{key.inspect} to #{value.inspect}") set(key, value) else - logger.debug("open: record has expired, skipping: #{key.inspect} #{value.inspect}") + logger.trace("open: record has expired, skipping: #{key.inspect} #{value.inspect}") end end def sincedb_write(time = Time.now.to_i) - logger.debug("sincedb_write: to: #{path}") + logger.trace("sincedb_write: to: #{path}") begin @write_method.call @serializer.expired_keys.each do |key| @sincedb[key].unset_watched_file delete(key) - logger.debug("sincedb_write: cleaned", "key" => "'#{key}'") + logger.trace("sincedb_write: cleaned", "key" => "'#{key}'") end @sincedb_last_write = time @write_requested = false rescue Errno::EACCES # no file handles free perhaps # maybe it will work next time - logger.debug("sincedb_write: error: #{path}: #{$!}") + logger.trace("sincedb_write: error: #{path}: #{$!}") end end diff --git a/lib/filewatch/sincedb_value.rb b/lib/filewatch/sincedb_value.rb index f97c44a..56ac484 100644 --- a/lib/filewatch/sincedb_value.rb +++ b/lib/filewatch/sincedb_value.rb @@ -66,6 +66,12 @@ def clear_watched_file @watched_file = nil end + def reading_completed + touch + @path_in_sincedb = @watched_file.path + @position = @watched_file.bytes_read + end + def unset_watched_file # called in read mode only because we flushed any remaining bytes as a final line. # cache the position diff --git a/lib/filewatch/stat/generic.rb b/lib/filewatch/stat/generic.rb new file mode 100644 index 0000000..6d83a72 --- /dev/null +++ b/lib/filewatch/stat/generic.rb @@ -0,0 +1,34 @@ +# encoding: utf-8 + +module FileWatch module Stat + class Generic + + attr_reader :identifier, :inode, :modified_at, :size, :inode_struct + + def initialize(source) + @source = source + @identifier = nil + restat + end + + def add_identifier(identifier) self; end + + def restat + @inner_stat = @source.stat + @inode = @inner_stat.ino.to_s + @modified_at = @inner_stat.mtime.to_f + @size = @inner_stat.size + @dev_major = @inner_stat.dev_major + @dev_minor = @inner_stat.dev_minor + @inode_struct = InodeStruct.new(@inode, @dev_major, @dev_minor) + end + + def windows? + false + end + + def inspect + "" + end + end +end end diff --git a/lib/filewatch/stat/windows_path.rb b/lib/filewatch/stat/windows_path.rb new file mode 100644 index 0000000..de773a4 --- /dev/null +++ b/lib/filewatch/stat/windows_path.rb @@ -0,0 +1,32 @@ +# encoding: utf-8 + +module FileWatch module Stat + class WindowsPath + + attr_reader :identifier, :inode, :modified_at, :size, :inode_struct + + def initialize(source) + @source = source + @inode = Winhelper.identifier_from_path(@source.to_path) + @dev_major = 0 + @dev_minor = 0 + # in windows the dev hi and low are in the identifier + @inode_struct = InodeStruct.new(@inode, @dev_major, @dev_minor) + restat + end + + def restat + @inner_stat = @source.stat + @modified_at = @inner_stat.mtime.to_f + @size = @inner_stat.size + end + + def windows? + true + end + + def inspect + "" + end + end +end end diff --git a/lib/filewatch/tail_mode/handlers/base.rb b/lib/filewatch/tail_mode/handlers/base.rb index a769f5a..783eb4e 100644 --- a/lib/filewatch/tail_mode/handlers/base.rb +++ b/lib/filewatch/tail_mode/handlers/base.rb @@ -13,7 +13,7 @@ def initialize(sincedb_collection, observer, settings) end def handle(watched_file) - logger.debug("handling: #{watched_file.path}") + logger.trace("handling: #{watched_file.filename}") unless watched_file.has_listener? watched_file.set_listener(@observer) end @@ -31,6 +31,7 @@ def update_existing_specifically(watched_file, sincedb_value) private def read_to_eof(watched_file) + logger.trace("reading...", "amount" => watched_file.read_bytesize_description, "filename" => watched_file.filename) changed = false # from a real config (has 102 file inputs) # -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously @@ -39,20 +40,16 @@ def read_to_eof(watched_file) # we enable the pseudo parallel processing of each file. # user also has the option to specify a low `stat_interval` and a very high `discover_interval`to respond # quicker to changing files and not allowing too much content to build up before reading it. - @settings.file_chunk_count.times do + watched_file.read_loop_count.times do begin - data = watched_file.file_read(@settings.file_chunk_size) - result = watched_file.buffer_extract(data) # expect BufferExtractResult - logger.info(result.warning, result.additional) unless result.warning.empty? + result = watched_file.read_extract_lines # expect BufferExtractResult + logger.trace(result.warning, result.additional) unless result.warning.empty? changed = true result.lines.each do |line| watched_file.listener.accept(line) # sincedb position is now independent from the watched_file bytes_read sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size) end - # instead of tracking the bytes_read line by line we need to track by the data read size. - # because we seek to the bytes_read not the sincedb position - watched_file.increment_bytes_read(data.bytesize) rescue EOFError # it only makes sense to signal EOF in "read" mode not "tail" break @@ -70,7 +67,7 @@ def read_to_eof(watched_file) def open_file(watched_file) return true if watched_file.file_open? - logger.debug("opening #{watched_file.path}") + logger.trace("opening #{watched_file.filename}") begin watched_file.open rescue @@ -82,43 +79,64 @@ def open_file(watched_file) logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}") watched_file.last_open_warning_at = now else - logger.debug("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}") + logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}") end watched_file.watch # set it back to watch so we can try it again - end - if watched_file.file_open? - watched_file.listener.opened - true else - false + watched_file.listener.opened end + watched_file.file_open? end def add_or_update_sincedb_collection(watched_file) sincedb_value = @sincedb_collection.find(watched_file) if sincedb_value.nil? - add_new_value_sincedb_collection(watched_file) + sincedb_value = add_new_value_sincedb_collection(watched_file) + watched_file.initial_completed elsif sincedb_value.watched_file == watched_file update_existing_sincedb_collection_value(watched_file, sincedb_value) + watched_file.initial_completed else - logger.warn? && logger.warn("mismatch on sincedb_value.watched_file, this should have been handled by Discoverer") + msg = "add_or_update_sincedb_collection: found sincedb record" + logger.trace(msg, + "sincedb key" => watched_file.sincedb_key, + "sincedb value" => sincedb_value + ) + # detected a rotation, Discoverer can't handle this because this watched file is not a new discovery. + # we must handle it here, by transferring state and have the sincedb value track this watched file + # rotate_as_file and rotate_from will switch the sincedb key to the inode that the path is now pointing to + # and pickup the sincedb_value from before. + msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file" + logger.trace(msg) + existing_watched_file = sincedb_value.watched_file + if existing_watched_file.nil? + sincedb_value.set_watched_file(watched_file) + logger.trace("add_or_update_sincedb_collection: switching as new file") + watched_file.rotate_as_file + watched_file.update_bytes_read(sincedb_value.position) + else + sincedb_value.set_watched_file(watched_file) + logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details) + watched_file.rotate_from(existing_watched_file) + end end - watched_file.initial_completed + sincedb_value end def update_existing_sincedb_collection_value(watched_file, sincedb_value) - logger.debug("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}") + logger.trace("update_existing_sincedb_collection_value: #{watched_file.filename}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}") update_existing_specifically(watched_file, sincedb_value) end def add_new_value_sincedb_collection(watched_file) sincedb_value = get_new_value_specifically(watched_file) - logger.debug("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position) + logger.trace("add_new_value_sincedb_collection", "position" => sincedb_value.position, "watched_file details" => watched_file.details) sincedb_collection.set(watched_file.sincedb_key, sincedb_value) + sincedb_value end def get_new_value_specifically(watched_file) - position = @settings.start_new_files_at == :beginning ? 0 : watched_file.last_stat_size + position = watched_file.position_for_new_sincedb_value value = SincedbValue.new(position) value.set_watched_file(watched_file) watched_file.update_bytes_read(position) diff --git a/lib/filewatch/tail_mode/handlers/create.rb b/lib/filewatch/tail_mode/handlers/create.rb index 2b89c11..167bc2c 100644 --- a/lib/filewatch/tail_mode/handlers/create.rb +++ b/lib/filewatch/tail_mode/handlers/create.rb @@ -10,8 +10,7 @@ def handle_specifically(watched_file) def update_existing_specifically(watched_file, sincedb_value) # sincedb_value is the source of truth - position = sincedb_value.position - watched_file.update_bytes_read(position) + watched_file.update_bytes_read(sincedb_value.position) end end end end end diff --git a/lib/filewatch/tail_mode/handlers/create_initial.rb b/lib/filewatch/tail_mode/handlers/create_initial.rb index 65c385b..8e01f9d 100644 --- a/lib/filewatch/tail_mode/handlers/create_initial.rb +++ b/lib/filewatch/tail_mode/handlers/create_initial.rb @@ -4,6 +4,7 @@ module FileWatch module TailMode module Handlers class CreateInitial < Base def handle_specifically(watched_file) if open_file(watched_file) + logger.trace("handle_specifically opened file handle: #{watched_file.file.fileno}, path: #{watched_file.filename}") add_or_update_sincedb_collection(watched_file) end end @@ -13,7 +14,7 @@ def update_existing_specifically(watched_file, sincedb_value) if @settings.start_new_files_at == :beginning position = 0 end - logger.debug("update_existing_specifically - #{watched_file.path}: seeking to #{position}") + logger.trace("update_existing_specifically - #{watched_file.path}: seeking to #{position}") watched_file.update_bytes_read(position) sincedb_value.update_position(position) end diff --git a/lib/filewatch/tail_mode/handlers/delete.rb b/lib/filewatch/tail_mode/handlers/delete.rb index 39e4c74..cc7a79b 100644 --- a/lib/filewatch/tail_mode/handlers/delete.rb +++ b/lib/filewatch/tail_mode/handlers/delete.rb @@ -2,9 +2,21 @@ module FileWatch module TailMode module Handlers class Delete < Base + DATA_LOSS_WARNING = "watched file path was deleted or rotated before all content was read, if the file is found again it will be read from the last position" def handle_specifically(watched_file) + # TODO consider trying to find the renamed file - it will have the same inode. + # Needs a rotate scheme rename hint from user e.g. "-YYYY-MM-DD-N." or "..N" + # send the found content to the same listener (stream identity) + logger.trace("info", + "watched_file details" => watched_file.details, + "path" => watched_file.path) + if watched_file.bytes_unread > 0 + logger.warn(DATA_LOSS_WARNING, "unread_bytes" => watched_file.bytes_unread, "path" => watched_file.path) + end watched_file.listener.deleted - sincedb_collection.unset_watched_file(watched_file) + # no need to worry about data in the buffer + # if found it will be associated by inode and read from last position + sincedb_collection.watched_file_deleted(watched_file) watched_file.file_close end end diff --git a/lib/filewatch/tail_mode/handlers/grow.rb b/lib/filewatch/tail_mode/handlers/grow.rb index 826017e..4c93fe9 100644 --- a/lib/filewatch/tail_mode/handlers/grow.rb +++ b/lib/filewatch/tail_mode/handlers/grow.rb @@ -4,7 +4,6 @@ module FileWatch module TailMode module Handlers class Grow < Base def handle_specifically(watched_file) watched_file.file_seek(watched_file.bytes_read) - logger.debug("reading to eof: #{watched_file.path}") read_to_eof(watched_file) end end diff --git a/lib/filewatch/tail_mode/handlers/shrink.rb b/lib/filewatch/tail_mode/handlers/shrink.rb index 9a7f0f0..54fc3ed 100644 --- a/lib/filewatch/tail_mode/handlers/shrink.rb +++ b/lib/filewatch/tail_mode/handlers/shrink.rb @@ -3,18 +3,18 @@ module FileWatch module TailMode module Handlers class Shrink < Base def handle_specifically(watched_file) - add_or_update_sincedb_collection(watched_file) + sdbv = add_or_update_sincedb_collection(watched_file) watched_file.file_seek(watched_file.bytes_read) - logger.debug("reading to eof: #{watched_file.path}") read_to_eof(watched_file) + logger.trace("handle_specifically: after read_to_eof", "watched file" => watched_file.details, "sincedb value" => sdbv) end def update_existing_specifically(watched_file, sincedb_value) # we have a match but size is smaller # set all to zero - logger.debug("update_existing_specifically: #{watched_file.path}: was truncated seeking to beginning") - watched_file.update_bytes_read(0) if watched_file.bytes_read != 0 + watched_file.reset_bytes_unread sincedb_value.update_position(0) + logger.trace("update_existing_specifically: was truncated seeking to beginning", "watched file" => watched_file.details, "sincedb value" => sincedb_value) end end end end end diff --git a/lib/filewatch/tail_mode/handlers/unignore.rb b/lib/filewatch/tail_mode/handlers/unignore.rb index 07cace1..7b510fe 100644 --- a/lib/filewatch/tail_mode/handlers/unignore.rb +++ b/lib/filewatch/tail_mode/handlers/unignore.rb @@ -6,15 +6,16 @@ class Unignore < Base # before any other handling has been done # at a minimum we create or associate a sincedb value def handle_specifically(watched_file) - add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) + add_or_update_sincedb_collection(watched_file) end def get_new_value_specifically(watched_file) # for file initially ignored their bytes_read was set to stat.size # use this value not the `start_new_files_at` for the position - # logger.debug("get_new_value_specifically", "watched_file" => watched_file.inspect) + # logger.trace("get_new_value_specifically", "watched_file" => watched_file.inspect) SincedbValue.new(watched_file.bytes_read).tap do |val| val.set_watched_file(watched_file) + logger.trace("-------------------- >>>>> get_new_value_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => val) end end @@ -25,6 +26,7 @@ def update_existing_specifically(watched_file, sincedb_value) # we will handle grow or shrink # for now we seek to where we were before the file got ignored (grow) # or to the start (shrink) + logger.trace("-------------------- >>>>> update_existing_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => sincedb_value) position = 0 if watched_file.shrunk? watched_file.update_bytes_read(0) diff --git a/lib/filewatch/tail_mode/processor.rb b/lib/filewatch/tail_mode/processor.rb index 8291280..ed990c7 100644 --- a/lib/filewatch/tail_mode/processor.rb +++ b/lib/filewatch/tail_mode/processor.rb @@ -34,6 +34,7 @@ def add_watch(watch) end def initialize_handlers(sincedb_collection, observer) + @sincedb_collection = sincedb_collection @create_initial = Handlers::CreateInitial.new(sincedb_collection, observer, @settings) @create = Handlers::Create.new(sincedb_collection, observer, @settings) @grow = Handlers::Grow.new(sincedb_collection, observer, @settings) @@ -71,80 +72,148 @@ def unignore(watched_file) @unignore.handle(watched_file) end + def process_all_states(watched_files) + process_closed(watched_files) + return if watch.quit? + process_ignored(watched_files) + return if watch.quit? + process_delayed_delete(watched_files) + return if watch.quit? + process_restat_for_watched_and_active(watched_files) + return if watch.quit? + process_rotation_in_progress(watched_files) + return if watch.quit? + process_watched(watched_files) + return if watch.quit? + process_active(watched_files) + end + + private + def process_closed(watched_files) - logger.debug("Closed processing") + # logger.trace("Closed processing") # Handles watched_files in the closed state. # if its size changed it is put into the watched state watched_files.select {|wf| wf.closed? }.each do |watched_file| - path = watched_file.path - begin - watched_file.restat + common_restat_with_delay(watched_file, "Closed") do + # it won't do this if rotation is detected if watched_file.size_changed? # if the closed file changed, move it to the watched state # not to active state because we want to respect the active files window. watched_file.watch end - rescue Errno::ENOENT - # file has gone away or we can't read it anymore. - common_deleted_reaction(watched_file, "Closed") - rescue => e - common_error_reaction(path, e, "Closed") end break if watch.quit? end end def process_ignored(watched_files) - logger.debug("Ignored processing") + # logger.trace("Ignored processing") # Handles watched_files in the ignored state. # if its size changed: # put it in the watched state # invoke unignore watched_files.select {|wf| wf.ignored? }.each do |watched_file| - path = watched_file.path - begin - watched_file.restat + common_restat_with_delay(watched_file, "Ignored") do + # it won't do this if rotation is detected if watched_file.size_changed? watched_file.watch unignore(watched_file) end - rescue Errno::ENOENT - # file has gone away or we can't read it anymore. - common_deleted_reaction(watched_file, "Ignored") - rescue => e - common_error_reaction(path, e, "Ignored") end break if watch.quit? end end + def process_delayed_delete(watched_files) + # defer the delete to one loop later to ensure that the stat really really can't find a renamed file + # because a `stat` can be called right in the middle of the rotation rename cascade + logger.trace("Delayed Delete processing") + watched_files.select {|wf| wf.delayed_delete?}.each do |watched_file| + logger.trace(">>> Delayed Delete", "path" => watched_file.filename) + common_restat_without_delay(watched_file, ">>> Delayed Delete") do + logger.trace(">>> Delayed Delete: file at path found again", "watched_file" => watched_file.details) + watched_file.file_at_path_found_again + end + end + end + + def process_restat_for_watched_and_active(watched_files) + # do restat on all watched and active states once now. closed and ignored have been handled already + logger.trace("Watched + Active restat processing") + watched_files.select {|wf| wf.watched? || wf.active?}.each do |watched_file| + common_restat_with_delay(watched_file, "Watched") + end + end + + def process_rotation_in_progress(watched_files) + logger.trace("Rotation In Progress processing") + watched_files.select {|wf| wf.rotation_in_progress?}.each do |watched_file| + if !watched_file.all_read? + if watched_file.file_open? + # rotated file but original opened file is not fully read + # we need to keep reading the open file, if we close it we lose it because the path is now pointing at a different file. + logger.trace(">>> Rotation In Progress - inode change detected and original content is not fully read, reading all", "watched_file details" => watched_file.details) + # need to fully read open file while we can + watched_file.set_depth_first_read_loop + grow(watched_file) + watched_file.set_user_defined_read_loop + else + logger.warn(">>> Rotation In Progress - inode change detected and original content is not fully read, file is closed and path points to new content", "watched_file details" => watched_file.details) + end + end + current_key = watched_file.sincedb_key + sdb_value = @sincedb_collection.get(current_key) + potential_key = watched_file.stat_sincedb_key + potential_sdb_value = @sincedb_collection.get(potential_key) + logger.trace(">>> Rotation In Progress", "watched_file" => watched_file.details, "found_sdb_value" => sdb_value, "potential_key" => potential_key, "potential_sdb_value" => potential_sdb_value) + if potential_sdb_value.nil? + if sdb_value.nil? + logger.trace("---------- >>> Rotation In Progress: rotating as initial file, no potential sincedb value AND no found sincedb value") + watched_file.rotate_as_initial_file + else + logger.trace("---------- >>>> Rotation In Progress: rotating as existing file, no potential sincedb value BUT found sincedb value") + watched_file.rotate_as_file + sdb_value.clear_watched_file + end + new_sdb_value = SincedbValue.new(0) + new_sdb_value.set_watched_file(watched_file) + @sincedb_collection.set(potential_key, new_sdb_value) + else + other_watched_file = potential_sdb_value.watched_file + if other_watched_file.nil? + logger.trace("---------- >>>> Rotation In Progress: rotating as existing file WITH potential sincedb value that does not have a watched file reference !!!!!!!!!!!!!!!!!") + watched_file.rotate_as_file(potential_sdb_value.position) + sdb_value.clear_watched_file unless sdb_value.nil? + potential_sdb_value.set_watched_file(watched_file) + else + logger.trace("---------- >>>> Rotation In Progress: rotating from...", "this watched_file details" => watched_file.details, "other watched_file details" => other_watched_file.details) + watched_file.rotate_from(other_watched_file) + sdb_value.clear_watched_file unless sdb_value.nil? + potential_sdb_value.set_watched_file(watched_file) + end + end + logger.trace("---------- >>>> Rotation In Progress: after handling rotation", "this watched_file details" => watched_file.details, "sincedb_value" => (potential_sdb_value || sdb_value)) + end + end + def process_watched(watched_files) - logger.debug("Watched processing") # Handles watched_files in the watched state. # for a slice of them: # move to the active state # and we allow the block to open the file and create a sincedb collection record if needed # some have never been active and some have # those that were active before but are watched now were closed under constraint - + logger.trace("Watched processing") # how much of the max active window is available to_take = @settings.max_active - watched_files.count{|wf| wf.active?} if to_take > 0 watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file| - path = watched_file.path - begin - watched_file.restat - watched_file.activate - if watched_file.initial? - create_initial(watched_file) - else - create(watched_file) - end - rescue Errno::ENOENT - # file has gone away or we can't read it anymore. - common_deleted_reaction(watched_file, "Watched") - rescue => e - common_error_reaction(path, e, "Watched") + watched_file.activate + if watched_file.initial? + create_initial(watched_file) + else + create(watched_file) end break if watch.quit? end @@ -159,51 +228,71 @@ def process_watched(watched_files) end def process_active(watched_files) - logger.debug("Active processing") + # logger.trace("Active processing") # Handles watched_files in the active state. - # it has been read once - unless they were empty at the time + # files have been opened at this point watched_files.select {|wf| wf.active? }.each do |watched_file| - path = watched_file.path - begin - watched_file.restat - rescue Errno::ENOENT - # file has gone away or we can't read it anymore. - common_deleted_reaction(watched_file, "Active") - next - rescue => e - common_error_reaction(path, e, "Active") - next - end break if watch.quit? + path = watched_file.filename if watched_file.grown? - logger.debug("Active - file grew: #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}") + logger.trace("Active - file grew: #{path}: new size is #{watched_file.last_stat_size}, bytes read #{watched_file.bytes_read}") grow(watched_file) elsif watched_file.shrunk? + if watched_file.bytes_unread > 0 + logger.warn("Active - shrunk: DATA LOSS!! truncate detected with #{watched_file.bytes_unread} unread bytes: #{path}") + end # we don't update the size here, its updated when we actually read - logger.debug("Active - file shrunk #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}") + logger.trace("Active - file shrunk #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}") shrink(watched_file) else # same size, do nothing + logger.trace("Active - no change", "watched_file" => watched_file.details) end # can any active files be closed to make way for waiting files? if watched_file.file_closable? - logger.debug("Watch each: active: file expired: #{path}") + logger.trace("Watch each: active: file expired: #{path}") timeout(watched_file) watched_file.close end end end - def common_deleted_reaction(watched_file, action) - # file has gone away or we can't read it anymore. - watched_file.unwatch - delete(watched_file) - deletable_filepaths << watched_file.path - logger.debug("#{action} - stat failed: #{watched_file.path}, removing from collection") + def common_restat_with_delay(watched_file, action, &block) + common_restat(watched_file, action, true, &block) end - def common_error_reaction(path, error, action) - logger.error("#{action} - other error #{path}: (#{error.message}, #{error.backtrace.take(8).inspect})") + def common_restat_without_delay(watched_file, action, &block) + common_restat(watched_file, action, false, &block) + end + + def common_restat(watched_file, action, delay, &block) + all_ok = true + begin + watched_file.restat + if watched_file.rotation_in_progress? + logger.trace("-------------------- >>>>> restat - rotation_detected", "watched_file details" => watched_file.details, "new sincedb key" => watched_file.stat_sincedb_key) + # don't yield to closed and ignore processing + else + yield if block_given? + end + rescue Errno::ENOENT + if delay + logger.trace("#{action} - delaying the stat fail on: #{watched_file.filename}") + watched_file.delay_delete + else + # file has gone away or we can't read it anymore. + logger.trace("#{action} - after a delay, really can't find this file: #{watched_file.filename}") + watched_file.unwatch + logger.trace("#{action} - removing from collection: #{watched_file.filename}") + delete(watched_file) + deletable_filepaths << watched_file.path + all_ok = false + end + rescue => e + logger.error("#{action} - other error #{watched_file.path}: (#{e.message}, #{e.backtrace.take(8).inspect})") + all_ok = false + end + all_ok end end end end diff --git a/lib/filewatch/watch.rb b/lib/filewatch/watch.rb index 6256ae4..ea5b9f7 100644 --- a/lib/filewatch/watch.rb +++ b/lib/filewatch/watch.rb @@ -10,11 +10,8 @@ class Watch def initialize(discoverer, watched_files_collection, settings) @settings = settings - # watch and iterate_on_state can be called from different threads. - @lock = Mutex.new # we need to be threadsafe about the quit mutation - @quit = false - @quit_lock = Mutex.new + @quit = Concurrent::AtomicBoolean.new(false) @lastwarn_max_files = 0 @discoverer = discoverer @watched_files_collection = watched_files_collection @@ -27,17 +24,13 @@ def add_processor(processor) end def watch(path) - synchronized do - @discoverer.add_path(path) - end + @discoverer.add_path(path) # don't return whatever @discoverer.add_path returns return true end def discover - synchronized do - @discoverer.discover - end + @discoverer.discover # don't return whatever @discoverer.discover returns return true end @@ -60,6 +53,7 @@ def subscribe(observer, sincedb_collection) break if quit? sleep(@settings.stat_interval) end + sincedb_collection.write_if_requested # does nothing if no requests to write were lodged. @watched_files_collection.close_all end # def subscribe @@ -67,42 +61,28 @@ def subscribe(observer, sincedb_collection) # differently from Tail mode - see the ReadMode::Processor and TailMode::Processor def iterate_on_state return if @watched_files_collection.empty? - synchronized do - begin - # creates this snapshot of watched_file values just once - watched_files = @watched_files_collection.values - @processor.process_closed(watched_files) - return if quit? - @processor.process_ignored(watched_files) - return if quit? - @processor.process_watched(watched_files) - return if quit? - @processor.process_active(watched_files) - ensure - @watched_files_collection.delete(@processor.deletable_filepaths) - @processor.deletable_filepaths.clear - end + begin + # creates this snapshot of watched_file values just once + watched_files = @watched_files_collection.values + @processor.process_all_states(watched_files) + ensure + @watched_files_collection.delete(@processor.deletable_filepaths) + @processor.deletable_filepaths.clear end end # def each def quit - @quit_lock.synchronize do - @quit = true - end - end # def quit + @quit.make_true + end def quit? - @quit_lock.synchronize { @quit } + @quit.true? end private - def synchronized(&block) - @lock.synchronize { block.call } - end - def reset_quit - @quit_lock.synchronize { @quit = false } + @quit.make_false end end end diff --git a/lib/filewatch/watched_file.rb b/lib/filewatch/watched_file.rb index a80a93c..78873a8 100644 --- a/lib/filewatch/watched_file.rb +++ b/lib/filewatch/watched_file.rb @@ -2,31 +2,162 @@ module FileWatch class WatchedFile - include InodeMixin # see bootstrap.rb at `if LogStash::Environment.windows?` + PATH_BASED_STAT = 0 + IO_BASED_STAT = 1 - attr_reader :bytes_read, :state, :file, :buffer, :recent_states - attr_reader :path, :filestat, :accessed_at, :modified_at, :pathname - attr_reader :sdb_key_v1, :last_stat_size, :listener + attr_reader :bytes_read, :state, :file, :buffer, :recent_states, :bytes_unread + attr_reader :path, :accessed_at, :modified_at, :pathname, :filename + attr_reader :listener, :read_loop_count, :read_chunk_size, :stat, :read_bytesize_description attr_accessor :last_open_warning_at # this class represents a file that has been discovered + # path based stat is taken at discovery def initialize(pathname, stat, settings) @settings = settings @pathname = Pathname.new(pathname) # given arg pathname might be a string or a Pathname object @path = @pathname.to_path - @bytes_read = 0 - @last_stat_size = 0 - # the prepare_inode method is sourced from the mixed module above - @sdb_key_v1 = InodeStruct.new(*prepare_inode(path, stat)) + @filename = @pathname.basename.to_s + full_state_reset(stat) + watch + set_user_defined_read_loop + set_accessed_at + end + + def no_restat_reset + full_state_reset(@stat) + end + + def full_state_reset(this_stat = nil) + if this_stat.nil? + begin + this_stat = PathStatClass.new(pathname) + rescue Errno::ENOENT + delay_delete + return + end + end + @bytes_read = 0 # tracks bytes read from the open file or initialized from a matched sincedb_value off disk. + @bytes_unread = 0 # tracks bytes not yet read from the open file. So we can warn on shrink when unread bytes are seen. + file_close + set_stat(this_stat) + @listener = nil + @last_open_warning_at = nil # initial as true means we have not associated this watched_file with a previous sincedb value yet. # and we should read from the beginning if necessary @initial = true @recent_states = [] # keep last 8 states, managed in set_state - @state = :watched - set_stat(stat) # can change @last_stat_size + # the prepare_inode method is sourced from the mixed module above + watch if active? || @state.nil? + end + + def rotate_from(other) + # move all state from other to this one + set_user_defined_read_loop + file_close + @bytes_read = other.bytes_read + @bytes_unread = other.bytes_unread @listener = nil + @initial = false + @recent_states = other.recent_states + @accessed_at = other.accessed_at + if !other.delayed_delete? + # we don't know if a file exists at the other.path yet + # so no reset + other.full_state_reset + end + set_stat PathStatClass.new(pathname) + ignore + end + + def set_stat(stat) + @stat = stat + @size = @stat.size + @sdb_key_v1 = @stat.inode_struct + end + + def rotate_as_initial_file + # rotation, when no sincedb record exists for new inode - we have never seen this inode before. + rotate_as_file + @initial = true + end + + def rotate_as_file(bytes_read = 0) + # rotation, when a sincedb record exists for new inode, but no watched file to rotate from + # probably caused by a deletion detected in the middle of the rename cascade + # RARE due to delayed_delete - there would have to be a large time span between the renames. + @bytes_read = bytes_read # tracks bytes read from the open file or initialized from a matched sincedb_value off disk. + @bytes_unread = 0 # tracks bytes not yet read from the open file. So we can warn on shrink when unread bytes are seen. @last_open_warning_at = nil - set_accessed_at + # initial as true means we have not associated this watched_file with a previous sincedb value yet. + # and we should read from the beginning if necessary + @initial = false + @recent_states = [] # keep last 8 states, managed in set_state + set_stat(PathStatClass.new(pathname)) + reopen + watch + end + + def stat_sincedb_key + @stat.inode_struct + end + + def rotation_detected? + stat_sincedb_key != sincedb_key + end + + def restat + @stat.restat + if rotation_detected? + # switch to new state now + rotation_in_progress + else + @size = @stat.size + update_bytes_unread + end + end + + def modified_at + @stat.modified_at + end + + def position_for_new_sincedb_value + if @initial + # this file was found in first discovery + @settings.start_new_files_at == :beginning ? 0 : last_stat_size + else + # always start at the beginning if found after first discovery + 0 + end + end + + def last_stat_size + @stat.size + end + + def current_size + @size + end + + def shrunk? + @size < @bytes_read + end + + def grown? + @size > @bytes_read + end + + def size_changed? + # called from closed and ignored + # before the last stat was taken file should be fully read. + @size != @bytes_read + end + + def all_read? + @bytes_read >= @size + end + + def file_at_path_found_again + restore_previous_state end def set_listener(observer) @@ -61,12 +192,11 @@ def compressed? @path.end_with?('.gz','.gzip') end - def size_changed? - @last_stat_size != bytes_read - end - - def all_read? - @last_stat_size == bytes_read + def reopen + if file_open? + file_close + open + end end def open @@ -88,9 +218,9 @@ def file_seek(amount, whence = IO::SEEK_SET) @file.sysseek(amount, whence) end - def file_read(amount) + def file_read(amount = nil) set_accessed_at - @file.sysread(amount) + @file.sysread(amount || @read_chunk_size) end def file_open? @@ -101,6 +231,13 @@ def reset_buffer @buffer.flush end + def read_extract_lines + data = file_read + result = buffer_extract(data) + increment_bytes_read(data.bytesize) + result + end + def buffer_extract(data) warning, additional = "", {} lines = @buffer.extract(data) @@ -112,7 +249,7 @@ def buffer_extract(data) additional["delimiter"] = @settings.delimiter additional["read_position"] = @bytes_read additional["bytes_read_count"] = data.bytesize - additional["last_known_file_size"] = @last_stat_size + additional["last_known_file_size"] = last_stat_size additional["file_path"] = @path end BufferExtractResult.new(lines, warning, additional) @@ -121,19 +258,19 @@ def buffer_extract(data) def increment_bytes_read(delta) return if delta.nil? @bytes_read += delta + update_bytes_unread + @bytes_read end def update_bytes_read(total_bytes_read) return if total_bytes_read.nil? @bytes_read = total_bytes_read + update_bytes_unread + @bytes_read end - def update_path(_path) - @path = _path - end - - def update_stat(st) - set_stat(st) + def rotation_in_progress + set_state :rotation_in_progress end def activate @@ -142,7 +279,11 @@ def activate def ignore set_state :ignored - @bytes_read = @filestat.size + end + + def ignore_as_unread + ignore + @bytes_read = @size end def close @@ -157,10 +298,26 @@ def unwatch set_state :unwatched end + def delay_delete + set_state :delayed_delete + end + + def restore_previous_state + set_state @recent_states.pop + end + + def rotation_in_progress? + @state == :rotation_in_progress + end + def active? @state == :active end + def delayed_delete? + @state == :delayed_delete + end + def ignored? @state == :ignored end @@ -185,21 +342,26 @@ def expiry_ignore_enabled? !@settings.ignore_older.nil? end - def shrunk? - @last_stat_size < @bytes_read + def set_depth_first_read_loop + @read_loop_count = FileWatch::MAX_ITERATIONS + @read_chunk_size = FileWatch::FILE_READ_SIZE + @read_bytesize_description = "All" end - def grown? - @last_stat_size > @bytes_read + def set_user_defined_read_loop + @read_loop_count = @settings.file_chunk_count + @read_chunk_size = @settings.file_chunk_size + @read_bytesize_description = @read_loop_count == FileWatch::MAX_ITERATIONS ? "All" : (@read_loop_count * @read_chunk_size).to_s end - def restat - set_stat(pathname.stat) + def reset_bytes_unread + # called from shrink + @bytes_unread = 0 end def set_state(value) @recent_states.shift if @recent_states.size == 8 - @recent_states << @state + @recent_states << @state unless @state.nil? @state = value end @@ -216,7 +378,7 @@ def file_ignorable? # (Time.now - stat.mtime) <- in jruby, this does int and float # conversions before the subtraction and returns a float. # so use all floats upfront - (Time.now.to_f - @modified_at) > @settings.ignore_older + (Time.now.to_f - modified_at) > @settings.ignore_older end def file_can_close? @@ -224,16 +386,26 @@ def file_can_close? (Time.now.to_f - @accessed_at) > @settings.close_older end + def details + detail = "@filename='#{filename}', @state='#{state}', @recent_states='#{@recent_states.inspect}', " + detail.concat("@bytes_read='#{@bytes_read}', @bytes_unread='#{@bytes_unread}', current_size='#{current_size}', ") + detail.concat("last_stat_size='#{last_stat_size}', file_open?='#{file_open?}', @initial=#{@initial}") + "" + end + + def inspect + "\" 0 + char_pointer_to_ruby_string(out) + else + "unknown" + end + ensure + CloseHandle(handle) if close_handle + end + + def self.identifier_from_io(io) + FileWatch::FileExt.io_handle(io) do |pointer| + identifier_from_handle(pointer, false) + end + end + + def self.identifier_from_path(path) + identifier_from_handle(open_handle_from_path(path)) + end + + def self.identifier_from_path_ex(path) + identifier_from_handle_ex(open_handle_from_path(path)) + end + + def self.identifier_from_io_ex(io) + FileWatch::FileExt.io_handle(io) do |pointer| + identifier_from_handle_ex(pointer, false) + end + end + + def self.identifier_from_handle_ex(handle, close_handle = true) + fileIdInfo = Winhelper::FileIdInfo.new + success = GetFileInformationByHandleEx(handle, :FileIdInfo, fileIdInfo, fileIdInfo.size) + if success > 0 + vsn = fileIdInfo[:volumeSerialNumber] + lpfid = fileIdInfo[:fileId][:lowPart] + hpfid = fileIdInfo[:fileId][:highPart] + return "#{vsn}-#{lpfid}-#{hpfid}" + else + return 'unknown' + end + ensure + CloseHandle(handle) if close_handle + end + + def self.identifier_from_handle(handle, close_handle = true) fileInfo = Winhelper::FileInformation.new success = GetFileInformationByHandle(handle, fileInfo) - CloseHandle(handle) - if success == 1 + if success > 0 #args = [ - # fileInfo[:fileAttributes], fileInfo[:volumeSerialNumber], fileInfo[:fileSizeHigh], fileInfo[:fileSizeLow], - # fileInfo[:numberOfLinks], fileInfo[:fileIndexHigh], fileInfo[:fileIndexLow] - # ] + # fileInfo[:fileAttributes], fileInfo[:volumeSerialNumber], fileInfo[:fileSizeHigh], fileInfo[:fileSizeLow], + # fileInfo[:numberOfLinks], fileInfo[:fileIndexHigh], fileInfo[:fileIndexLow] + # ] #p "Information: %u %u %u %u %u %u %u " % args #this is only guaranteed on NTFS, for ReFS on windows 2012, GetFileInformationByHandleEx should be used with FILE_ID_INFO, which returns a 128 bit identifier return "#{fileInfo[:volumeSerialNumber]}-#{fileInfo[:fileIndexLow]}-#{fileInfo[:fileIndexHigh]}" else - #p "cannot retrieve file information, returning path" - return path; + return 'unknown' end + ensure + CloseHandle(handle) if close_handle + end + + private + + def self.open_handle_from_path(path) + CreateFileW(in_buffer(path), 0, 7, nil, 3, 128, nil) + end + + def self.in_buffer(string) + utf16le(string) + end + + def self.char_pointer_to_ruby_string(char_pointer, length = 256) + bytes = char_pointer.get_array_of_uchar(0, length) + ignore = bytes.reverse.index{|b| b != 0} - 1 + our_bytes = bytes[0, bytes.length - ignore] + our_bytes.pack("C*").force_encoding("UTF-16LE").encode("UTF-8") + end + + def self.utf16le(string) + string.encode("UTF-16LE") + end + + def self.win1252(string) + string.encode("Windows-1252") end end + #fileId = Winhelper.GetWindowsUniqueFileIdentifier('C:\inetpub\logs\LogFiles\W3SVC1\u_ex1fdsadfsadfasdf30612.log') #p "FileId: " + fileId #p "outside function, sleeping" diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 81b04b4..511e4dd 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -206,7 +206,7 @@ class File < LogStash::Inputs::Base # 1MB from each active file. See the option `max_open_files` for more info. # The default set internally is very large, 4611686018427387903. By default # the file is read to the end before moving to the next active file. - config :file_chunk_count, :validate => :number, :default => FileWatch::FIXNUM_MAX + config :file_chunk_count, :validate => :number, :default => FileWatch::MAX_ITERATIONS # Which attribute of a discovered file should be used to sort the discovered files. # Files can be sort by modified date or full path alphabetic. @@ -312,8 +312,14 @@ def register end end @codec = LogStash::Codecs::IdentityMapCodec.new(@codec) + @completely_stopped = Concurrent::AtomicBoolean.new end # def register + def completely_stopped? + # to synchronise after(:each) blocks in tests that remove the sincedb file before atomic_write completes + @completely_stopped.true? + end + def listener_for(path) # path is the identity FileListener.new(path, self) @@ -333,6 +339,7 @@ def run(queue) @watcher.subscribe(self) # halts here until quit is called # last action of the subscribe call is to write the sincedb exit_flush + @completely_stopped.make_true end # def run def post_process_this(event) @@ -354,7 +361,7 @@ def log_line_received(path, line) end def stop - if @watcher + unless @watcher.nil? @codec.close @watcher.quit end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 38e89de..a340f84 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.3' + s.version = '4.1.4' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -23,7 +23,14 @@ Gem::Specification.new do |s| s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" s.add_runtime_dependency 'logstash-codec-plain' - s.add_runtime_dependency 'addressable' + + if RUBY_VERSION.start_with?("1") + s.add_runtime_dependency 'rake', '~> 12.2.0' + s.add_runtime_dependency 'addressable', '~> 2.4.0' + else + s.add_runtime_dependency 'addressable' + end + s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0'] s.add_development_dependency 'stud', ['~> 0.0.19'] diff --git a/run_until_fail.sh b/run_until_fail.sh new file mode 100755 index 0000000..3814284 --- /dev/null +++ b/run_until_fail.sh @@ -0,0 +1,4 @@ +while true +do + LOG_AT=ERROR bundle exec rspec -fd --fail-fast --tag ~lsof ./spec || break +done diff --git a/spec/file_ext/file_ext_windows_spec.rb b/spec/file_ext/file_ext_windows_spec.rb new file mode 100644 index 0000000..df54d98 --- /dev/null +++ b/spec/file_ext/file_ext_windows_spec.rb @@ -0,0 +1,36 @@ +# encoding: utf-8 + +require_relative '../filewatch/spec_helper' + +if LogStash::Environment.windows? + describe "basic ops" do + let(:fixture_dir) { Pathname.new(FileWatch::FIXTURE_DIR).expand_path } + let(:file_path) { fixture_dir.join('uncompressed.log') } + it "path works" do + path = file_path.to_path + identifier = Winhelper.identifier_from_path(path) + STDOUT.puts("--- >>", identifier, "------") + expect(identifier.count('-')).to eq(2) + fs_name = Winhelper.file_system_type_from_path(path) + STDOUT.puts("--- >>", fs_name, "------") + expect(fs_name).to eq("NTFS") + # identifier = Winhelper.identifier_from_path_ex(path) + # STDOUT.puts("--- >>", identifier, "------") + # expect(identifier.count('-')).to eq(2) + end + + it "io works" do + file = FileWatch::FileOpener.open(file_path.to_path) + identifier = Winhelper.identifier_from_io(file) + file.close + STDOUT.puts("--- >>", identifier, "------") + expect(identifier.count('-')).to eq(2) + # fs_name = Winhelper.file_system_type_from_io(file) + # STDOUT.puts("--- >>", fs_name, "------") + # expect(fs_name).to eq("NTFS") + # identifier = Winhelper.identifier_from_path_ex(path) + # STDOUT.puts("--- >>", identifier, "------") + # expect(identifier.count('-')).to eq(2) + end + end +end diff --git a/spec/filewatch/read_mode_handlers_read_file_spec.rb b/spec/filewatch/read_mode_handlers_read_file_spec.rb index 4667b36..ab36e71 100644 --- a/spec/filewatch/read_mode_handlers_read_file_spec.rb +++ b/spec/filewatch/read_mode_handlers_read_file_spec.rb @@ -12,7 +12,7 @@ module FileWatch let(:sdb_collection) { SincedbCollection.new(settings) } let(:directory) { Pathname.new(FIXTURE_DIR) } let(:pathname) { directory.join('uncompressed.log') } - let(:watched_file) { WatchedFile.new(pathname, pathname.stat, settings) } + let(:watched_file) { WatchedFile.new(pathname, PathStatClass.new(pathname), settings) } let(:processor) { ReadMode::Processor.new(settings).add_watch(watch) } let(:file) { DummyFileReader.new(settings.file_chunk_size, 2) } diff --git a/spec/filewatch/reading_spec.rb b/spec/filewatch/reading_spec.rb index 4f1f845..53deee1 100644 --- a/spec/filewatch/reading_spec.rb +++ b/spec/filewatch/reading_spec.rb @@ -30,82 +30,115 @@ module FileWatch end let(:observer) { TestObserver.new } let(:reading) { ObservingRead.new(opts) } - let(:actions) do - RSpec::Sequencing.run_after(0.45, "quit after a short time") do - reading.quit - end - end + let(:listener1) { observer.listener_for(file_path) } after do FileUtils.rm_rf(directory) unless directory =~ /fixture/ end context "when watching a directory with files" do - let(:directory) { Stud::Temporary.directory } - let(:watch_dir) { ::File.join(directory, "*.log") } - let(:file_path) { ::File.join(directory, "1.log") } - + let(:actions) do + RSpec::Sequencing.run("quit after a short time") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then("watch") do + reading.watch_this(watch_dir) + end + .then("wait") do + wait(2).for{listener1.calls.last}.to eq(:delete) + end + .then("quit") do + reading.quit + end + end it "the file is read" do - File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } - actions.activate - reading.watch_this(watch_dir) + actions.activate_quietly reading.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete]) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(listener1.lines).to eq(["line1", "line2"]) end end context "when watching a directory with files and sincedb_path is /dev/null or NUL" do - let(:directory) { Stud::Temporary.directory } let(:sincedb_path) { File::NULL } - let(:watch_dir) { ::File.join(directory, "*.log") } - let(:file_path) { ::File.join(directory, "1.log") } - + let(:actions) do + RSpec::Sequencing.run("quit after a short time") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then("watch") do + reading.watch_this(watch_dir) + end + .then("wait") do + wait(2).for{listener1.calls.last}.to eq(:delete) + end + .then("quit") do + reading.quit + end + end it "the file is read" do - File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } - actions.activate - reading.watch_this(watch_dir) + actions.activate_quietly reading.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete]) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(listener1.lines).to eq(["line1", "line2"]) end end context "when watching a directory with files using striped reading" do - let(:directory) { Stud::Temporary.directory } - let(:watch_dir) { ::File.join(directory, "*.log") } - let(:file_path1) { ::File.join(directory, "1.log") } let(:file_path2) { ::File.join(directory, "2.log") } # use a chunk size that does not align with the line boundaries - let(:opts) { super.merge(:file_chunk_size => 10, :file_chunk_count => 1)} + let(:opts) { super.merge(:file_chunk_size => 10, :file_chunk_count => 1, :file_sort_by => "path")} let(:lines) { [] } let(:observer) { TestObserver.new(lines) } - + let(:listener2) { observer.listener_for(file_path2) } + let(:actions) do + RSpec::Sequencing.run("create file") do + File.open(file_path, "w") { |file| file.write("string1\nstring2") } + File.open(file_path2, "w") { |file| file.write("stringA\nstringB") } + end + .then("watch") do + reading.watch_this(watch_dir) + end + .then("wait") do + wait(2).for{listener1.calls.last == :delete && listener2.calls.last == :delete}.to eq(true) + end + .then("quit") do + reading.quit + end + end it "the files are read seemingly in parallel" do - File.open(file_path1, "w") { |file| file.write("string1\nstring2\n") } - File.open(file_path2, "w") { |file| file.write("stringA\nstringB\n") } - actions.activate - reading.watch_this(watch_dir) + actions.activate_quietly reading.subscribe(observer) - if lines.first == "stringA" - expect(lines).to eq(%w(stringA string1 stringB string2)) - else - expect(lines).to eq(%w(string1 stringA string2 stringB)) - end + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(listener2.calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(lines).to eq(%w(string1 stringA string2 stringB)) end end context "when a non default delimiter is specified and it is not in the content" do let(:opts) { super.merge(:delimiter => "\nø") } - + let(:actions) do + RSpec::Sequencing.run("create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2") } + end + .then("watch") do + reading.watch_this(watch_dir) + end + .then("wait") do + wait(2).for{listener1.calls.last}.to eq(:delete) + end + .then("quit") do + reading.quit + end + end it "the file is opened, data is read, but no lines are found initially, at EOF the whole file becomes the line" do - File.open(file_path, "wb") { |file| file.write("line1\nline2") } - actions.activate - reading.watch_this(watch_dir) + actions.activate_quietly reading.subscribe(observer) - listener = observer.listener_for(file_path) - expect(listener.calls).to eq([:open, :accept, :eof, :delete]) - expect(listener.lines).to eq(["line1\nline2"]) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :eof, :delete]) + expect(listener1.lines).to eq(["line1\nline2"]) sincedb_record_fields = File.read(sincedb_path).split(" ") position_field_index = 3 # tailing, no delimiter, we are expecting one, if it grows we read from the start. @@ -116,18 +149,28 @@ module FileWatch describe "reading fixtures" do let(:directory) { FIXTURE_DIR } - + let(:actions) do + RSpec::Sequencing.run("watch") do + reading.watch_this(watch_dir) + end + .then("wait") do + wait(1).for{listener1.calls.last}.to eq(:delete) + end + .then("quit") do + reading.quit + end + end context "for an uncompressed file" do let(:watch_dir) { ::File.join(directory, "unc*.log") } let(:file_path) { ::File.join(directory, 'uncompressed.log') } it "the file is read" do FileWatch.make_fixture_current(file_path) - actions.activate - reading.watch_this(watch_dir) + actions.activate_quietly reading.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete]) - expect(observer.listener_for(file_path).lines.size).to eq(2) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(listener1.lines.size).to eq(2) end end @@ -137,11 +180,11 @@ module FileWatch it "the file is read" do FileWatch.make_fixture_current(file_path) - actions.activate - reading.watch_this(watch_dir) + actions.activate_quietly reading.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete]) - expect(observer.listener_for(file_path).lines.size).to eq(2) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(listener1.lines.size).to eq(2) end end @@ -151,11 +194,11 @@ module FileWatch it "the file is read" do FileWatch.make_fixture_current(file_path) - actions.activate - reading.watch_this(watch_dir) + actions.activate_quietly reading.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete]) - expect(observer.listener_for(file_path).lines.size).to eq(2) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept, :eof, :delete]) + expect(listener1.lines.size).to eq(2) end end end diff --git a/spec/filewatch/rotate_spec.rb b/spec/filewatch/rotate_spec.rb new file mode 100644 index 0000000..e1d2b4e --- /dev/null +++ b/spec/filewatch/rotate_spec.rb @@ -0,0 +1,451 @@ +# encoding: utf-8 +require 'stud/temporary' +require_relative 'spec_helper' +require 'filewatch/observing_tail' + +# simulate size based rotation ala +# See https://docs.python.org/2/library/logging.handlers.html#rotatingfilehandler +# The specified file is opened and used as the stream for logging. +# If mode is not specified, 'a' is used. If encoding is not None, it is used to +# open the file with that encoding. If delay is true, then file opening is deferred +# until the first call to emit(). By default, the file grows indefinitely. +# You can use the maxBytes and backupCount values to allow the file to rollover +# at a predetermined size. When the size is about to be exceeded, the file is +# closed and a new file is silently opened for output. Rollover occurs whenever +# the current log file is nearly maxBytes in length; if either of maxBytes or +# backupCount is zero, rollover never occurs. If backupCount is non-zero, the +# system will save old log files by appending the extensions ‘.1’, ‘.2’ etc., +# to the filename. For example, with a backupCount of 5 and a base file name of +# app.log, you would get app.log, app.log.1, app.log.2, up to app.log.5. +# The file being written to is always app.log. When this file is filled, it is +# closed and renamed to app.log.1, and if files app.log.1, app.log.2, etc. +# exist, then they are renamed to app.log.2, app.log.3 etc. respectively. + +module FileWatch + describe Watch, :unix => true do + let(:directory) { Pathname.new(Stud::Temporary.directory) } + let(:file1_path) { file_path.to_path } + let(:max) { 4095 } + let(:stat_interval) { 0.01 } + let(:discover_interval) { 15 } + let(:start_new_files_at) { :beginning } + let(:sincedb_path) { directory.join("tailing.sdb") } + let(:opts) do + { + :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_active => max, + :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path.to_path + } + end + let(:observer) { TestObserver.new } + let(:tailing) { ObservingTail.new(opts) } + let(:line1) { "Line 1 - Lorem ipsum dolor sit amet, consectetur adipiscing elit." } + let(:line2) { "Line 2 - Proin ut orci lobortis, congue diam in, dictum est." } + let(:line3) { "Line 3 - Sed vestibulum accumsan sollicitudin." } + + before do + directory + wait(1.0).for{Dir.exist?(directory)}.to eq(true) + end + + after do + FileUtils.rm_rf(directory) + wait(1.0).for{Dir.exist?(directory)}.to eq(false) + end + + context "create + rename rotation: when a new logfile is renamed to a path we have seen before and the open file is fully read, renamed outside glob" do + let(:watch_dir) { directory.join("*A.log") } + let(:file_path) { directory.join("1A.log") } + subject { described_class.new(conf) } + let(:listener1) { observer.listener_for(file1_path) } + let(:listener2) { observer.listener_for(second_file.to_path) } + let(:actions) do + RSpec::Sequencing + .run_after(0.25, "create file") do + file_path.open("wb") { |file| file.write("#{line1}\n") } + end + .then_after(0.25, "write a 'unfinished' line") do + file_path.open("ab") { |file| file.write(line2) } + end + .then_after(0.25, "rotate once") do + tmpfile = directory.join("1.logtmp") + tmpfile.open("wb") { |file| file.write("\n#{line3}\n")} + file_path.rename(directory.join("1.log.1")) + FileUtils.mv(directory.join("1.logtmp").to_path, file1_path) + end + .then("wait for expectation") do + wait(2).for{listener1.calls}.to eq([:open, :accept, :accept, :accept]) + end + .then("quit") do + tailing.quit + end + end + + it "content from both inodes are sent via the same stream" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + lines = listener1.lines + expect(lines[0]).to eq(line1) + expect(lines[1]).to eq(line2) + expect(lines[2]).to eq(line3) + end + end + + context "create + rename rotation: a multiple file rename cascade" do + let(:watch_dir) { directory.join("*B.log") } + let(:file_path) { directory.join("1B.log") } + subject { described_class.new(conf) } + let(:second_file) { directory.join("2B.log") } + let(:third_file) { directory.join("3B.log") } + let(:listener1) { observer.listener_for(file1_path) } + let(:listener2) { observer.listener_for(second_file.to_path) } + let(:listener3) { observer.listener_for(third_file.to_path) } + let(:actions) do + RSpec::Sequencing + .run_after(0.25, "create file") do + file_path.open("wb") { |file| file.write("#{line1}\n") } + end + .then_after(0.25, "rotate 1 - line1(66) is in 2B.log, line2(61) is in 1B.log") do + file_path.rename(second_file) + file_path.open("wb") { |file| file.write("#{line2}\n") } + end + .then_after(0.25, "rotate 2 - line1(66) is in 3B.log, line2(61) is in 2B.log, line3(47) is in 1B.log") do + second_file.rename(third_file) + file_path.rename(second_file) + file_path.open("wb") { |file| file.write("#{line3}\n") } + end + .then("wait for expectations to be met") do + wait(0.75).for{listener1.lines.size == 3 && listener3.lines.empty? && listener2.lines.empty?}.to eq(true) + end + .then("quit") do + tailing.quit + end + end + + it "content from both inodes are sent via the same stream" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + expect(listener1.lines[0]).to eq(line1) + expect(listener1.lines[1]).to eq(line2) + expect(listener1.lines[2]).to eq(line3) + end + end + + context "create + rename rotation: a two file rename cascade in slow motion" do + let(:watch_dir) { directory.join("*C.log") } + let(:file_path) { directory.join("1C.log") } + let(:stat_interval) { 0.01 } + subject { described_class.new(conf) } + let(:second_file) { directory.join("2C.log") } + let(:listener1) { observer.listener_for(file1_path) } + let(:listener2) { observer.listener_for(second_file.to_path) } + let(:actions) do + RSpec::Sequencing + .run_after(0.25, "create original - write line 1, 66 bytes") do + file_path.open("wb") { |file| file.write("#{line1}\n") } + end + .then_after(0.25, "rename to 2.log") do + file_path.rename(second_file) + end + .then_after(0.25, "write line 2 to original, 61 bytes") do + file_path.open("wb") { |file| file.write("#{line2}\n") } + end + .then_after(0.25, "rename to 2.log again") do + file_path.rename(second_file) + end + .then_after(0.25, "write line 3 to original, 47 bytes") do + file_path.open("wb") { |file| file.write("#{line3}\n") } + end + .then("wait for expectations to be met") do + wait(1).for{listener1.lines.size == 3 && listener2.lines.empty?}.to eq(true) + end + .then("quit") do + tailing.quit + end + end + + it "content from both inodes are sent via the same stream AND content from the rotated file is not read again" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + expect(listener1.lines[0]).to eq(line1) + expect(listener1.lines[1]).to eq(line2) + expect(listener1.lines[2]).to eq(line3) + end + end + + context "create + rename rotation: a two file rename cascade in normal speed" do + let(:watch_dir) { directory.join("*D.log") } + let(:file_path) { directory.join("1D.log") } + subject { described_class.new(conf) } + let(:second_file) { directory.join("2D.log") } + let(:listener1) { observer.listener_for(file1_path) } + let(:listener2) { observer.listener_for(second_file.to_path) } + let(:actions) do + RSpec::Sequencing + .run_after(0.25, "create original - write line 1, 66 bytes") do + file_path.open("wb") { |file| file.write("#{line1}\n") } + end + .then_after(0.25, "rename to 2.log") do + file_path.rename(second_file) + file_path.open("wb") { |file| file.write("#{line2}\n") } + end + .then_after(0.25, "rename to 2.log again") do + file_path.rename(second_file) + file_path.open("wb") { |file| file.write("#{line3}\n") } + end + .then("wait for expectations to be met") do + wait(0.5).for{listener1.lines.size == 3 && listener2.lines.empty?}.to eq(true) + end + .then("quit") do + tailing.quit + end + end + + it "content from both inodes are sent via the same stream AND content from the rotated file is not read again" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + expect(listener1.lines[0]).to eq(line1) + expect(listener1.lines[1]).to eq(line2) + expect(listener1.lines[2]).to eq(line3) + end + end + + context "create + rename rotation: when a new logfile is renamed to a path we have seen before but not all content from the previous the file is read" do + let(:opts) { super.merge( + :file_chunk_size => line1.bytesize.succ, + :file_chunk_count => 1 + ) } + let(:watch_dir) { directory.join("*E.log") } + let(:file_path) { directory.join("1E.log") } + subject { described_class.new(conf) } + let(:listener1) { observer.listener_for(file1_path) } + let(:actions) do + RSpec::Sequencing + .run_after(0.25, "create file") do + file_path.open("wb") do |file| + 65.times{file.puts(line1)} + end + end + .then_after(0.25, "rotate") do + tmpfile = directory.join("1E.logtmp") + tmpfile.open("wb") { |file| file.puts(line1)} + file_path.rename(directory.join("1E.log.1")) + tmpfile.rename(directory.join("1E.log")) + end + .then("wait for expectations to be met") do + wait(0.5).for{listener1.lines.size}.to eq(66) + end + .then("quit") do + tailing.quit + end + end + + it "content from both inodes are sent via the same stream" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + expected_calls = ([:accept] * 66).unshift(:open) + expect(listener1.lines.uniq).to eq([line1]) + expect(listener1.calls).to eq(expected_calls) + expect(sincedb_path.readlines.size).to eq(2) + end + end + + context "copy + truncate rotation: when a logfile is copied to a new path and truncated and the open file is fully read" do + let(:watch_dir) { directory.join("*F.log") } + let(:file_path) { directory.join("1F.log") } + subject { described_class.new(conf) } + let(:listener1) { observer.listener_for(file1_path) } + let(:actions) do + RSpec::Sequencing + .run_after(0.25, "create file") do + file_path.open("wb") { |file| file.puts(line1); file.puts(line2) } + end + .then_after(0.25, "rotate") do + FileUtils.cp(file1_path, directory.join("1F.log.1").to_path) + file_path.truncate(0) + end + .then_after(0.25, "write to truncated file") do + file_path.open("wb") { |file| file.puts(line3) } + end + .then("wait for expectations to be met") do + wait(0.5).for{listener1.lines.size}.to eq(3) + end + .then("quit") do + tailing.quit + end + end + + it "content is read correctly" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + expect(listener1.lines).to eq([line1, line2, line3]) + expect(listener1.calls).to eq([:open, :accept, :accept, :accept]) + end + end + + context "copy + truncate rotation: when a logfile is copied to a new path and truncated before the open file is fully read" do + let(:opts) { super.merge( + :file_chunk_size => line1.bytesize.succ, + :file_chunk_count => 1 + ) } + let(:watch_dir) { directory.join("*G.log") } + let(:file_path) { directory.join("1G.log") } + subject { described_class.new(conf) } + let(:listener1) { observer.listener_for(file1_path) } + let(:actions) do + RSpec::Sequencing + .run_after(0.25, "create file") do + file_path.open("wb") { |file| 65.times{file.puts(line1)} } + end + .then_after(0.25, "rotate") do + FileUtils.cp(file1_path, directory.join("1G.log.1").to_path) + file_path.truncate(0) + end + .then_after(0.25, "write to truncated file") do + file_path.open("wb") { |file| file.puts(line3) } + end + .then("wait for expectations to be met") do + wait(0.5).for{listener1.lines.last}.to eq(line3) + end + .then("quit") do + tailing.quit + end + end + + it "unread content before the truncate is lost" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + expect(listener1.lines.size).to be < 66 + end + end + + context "? rotation: when an active file is renamed inside the glob and the reading does not lag" do + let(:watch_dir) { directory.join("*H.log") } + let(:file_path) { directory.join("1H.log") } + let(:file2) { directory.join("2H.log") } + subject { described_class.new(conf) } + let(:listener1) { observer.listener_for(file1_path) } + let(:listener2) { observer.listener_for(file2.to_path) } + let(:actions) do + RSpec::Sequencing + .run_after(0.25, "create file") do + file_path.open("wb") { |file| file.puts(line1); file.puts(line2) } + end + .then_after(0.25, "rename") do + FileUtils.mv(file1_path, file2.to_path) + end + .then_after(0.25, "write to renamed file") do + file2.open("ab") { |file| file.puts(line3) } + end + .then("wait for expectations to be met") do + wait(0.75).for{listener1.lines.size + listener2.lines.size}.to eq(3) + end + .then("quit") do + tailing.quit + end + end + + it "content is read correctly, the renamed file is not reread from scratch" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + expect(listener1.lines).to eq([line1, line2]) + expect(listener2.lines).to eq([line3]) + end + end + + context "? rotation: when an active file is renamed inside the glob and the reading lags behind" do + let(:opts) { super.merge( + :file_chunk_size => line1.bytesize.succ, + :file_chunk_count => 2 + ) } + let(:watch_dir) { directory.join("*I.log") } + let(:file_path) { directory.join("1I.log") } + let(:file2) { directory.join("2I.log") } + subject { described_class.new(conf) } + let(:listener1) { observer.listener_for(file1_path) } + let(:listener2) { observer.listener_for(file2.to_path) } + let(:actions) do + RSpec::Sequencing + .run_after(0.25, "create file") do + file_path.open("wb") { |file| 65.times{file.puts(line1)} } + end + .then_after(0.25, "rename") do + FileUtils.mv(file1_path, file2.to_path) + end + .then_after(0.25, "write to renamed file") do + file2.open("ab") { |file| file.puts(line3) } + end + .then("wait for expectations to be met") do + wait(1.25).for{listener1.lines.size + listener2.lines.size}.to eq(66) + end + .then("quit") do + tailing.quit + end + end + + it "content is read correctly, the renamed file is not reread from scratch" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + expect(listener2.lines.last).to eq(line3) + end + end + + context "? rotation: when a not active file is rotated outside the glob before the file is read" do + let(:opts) { super.merge( + :close_older => 3600, + :max_active => 1, + :file_sort_by => "path" + ) } + let(:watch_dir) { directory.join("*J.log") } + let(:file_path) { directory.join("1J.log") } + let(:file2) { directory.join("2J.log") } + let(:file3) { directory.join("2J.log.1") } + let(:listener1) { observer.listener_for(file1_path) } + let(:listener2) { observer.listener_for(file2.to_path) } + let(:listener3) { observer.listener_for(file3.to_path) } + subject { described_class.new(conf) } + let(:actions) do + RSpec::Sequencing + .run_after(0.25, "create file") do + file_path.open("wb") { |file| 65.times{file.puts(line1)} } + file2.open("wb") { |file| 65.times{file.puts(line1)} } + end + .then_after(0.25, "rename") do + FileUtils.mv(file2.to_path, file3.to_path) + end + .then("wait for expectations to be met") do + wait(1.25).for{listener1.lines.size}.to eq(65) + end + .then("quit") do + tailing.quit + end + end + + it "file 1 content is read correctly, the renamed file 2 is not read at all" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + expect(listener2.lines.size).to eq(0) + expect(listener3.lines.size).to eq(0) + end + end + end +end diff --git a/spec/filewatch/spec_helper.rb b/spec/filewatch/spec_helper.rb index 0b8bfb7..f074133 100644 --- a/spec/filewatch/spec_helper.rb +++ b/spec/filewatch/spec_helper.rb @@ -1,7 +1,8 @@ # encoding: utf-8 require "rspec_sequencing" -require 'rspec/wait' +# require 'rspec/wait' require "logstash/devutils/rspec/spec_helper" +require "concurrent" require "timecop" def formatted_puts(text) @@ -24,9 +25,32 @@ def formatted_puts(text) end end +require_relative "../helpers/rspec_wait_handler_helper" unless defined? RSPEC_WAIT_HANDLER_PATCHED +require_relative "../helpers/logging_level_helper" unless defined? LOG_AT_HANDLED + require 'filewatch/bootstrap' module FileWatch + class DummyIO + def stat + self + end + def ino + 23456 + end + def size + 65535 + end + def mtime + Time.now + end + def dev_major + 1 + end + def dev_minor + 5 + end + end class DummyFileReader def initialize(read_size, iterations) @@ -34,6 +58,7 @@ def initialize(read_size, iterations) @iterations = iterations @closed = false @accumulated = 0 + @io = DummyIO.new end def file_seek(*) end @@ -43,6 +68,9 @@ def close() def closed? @closed end + def to_io + @io + end def sysread(amount) @accumulated += amount if @accumulated > @read_size * @iterations @@ -67,7 +95,7 @@ def self.make_fixture_current(path, time = Time.now) class TracerBase def initialize - @tracer = [] + @tracer = Concurrent::Array.new end def trace_for(symbol) @@ -91,8 +119,8 @@ class Listener def initialize(path) @path = path - @lines = [] - @calls = [] + @lines = Concurrent::Array.new + @calls = Concurrent::Array.new end def add_lines(lines) @@ -134,7 +162,7 @@ def initialize(combined_lines = nil) else lambda{|k| Listener.new(k).add_lines(combined_lines) } end - @listeners = Hash.new {|hash, key| hash[key] = listener_proc.call(key) } + @listeners = Concurrent::Hash.new {|hash, key| hash[key] = listener_proc.call(key) } end def listener_for(path) @@ -145,8 +173,3 @@ def clear @listeners.clear; end end end - -ENV["LOG_AT"].tap do |level| - LogStash::Logging::Logger::configure_logging(level) unless level.nil? -end - diff --git a/spec/filewatch/tailing_spec.rb b/spec/filewatch/tailing_spec.rb index 5e082a6..30998ea 100644 --- a/spec/filewatch/tailing_spec.rb +++ b/spec/filewatch/tailing_spec.rb @@ -5,42 +5,45 @@ module FileWatch describe Watch do - before(:all) do - @thread_abort = Thread.abort_on_exception - Thread.abort_on_exception = true - end - - after(:all) do - Thread.abort_on_exception = @thread_abort - end - let(:directory) { Stud::Temporary.directory } - let(:watch_dir) { ::File.join(directory, "*.log") } - let(:file_path) { ::File.join(directory, "1.log") } + let(:watch_dir) { ::File.join(directory, "*#{suffix}.log") } + let(:file_path) { ::File.join(directory, "1#{suffix}.log") } + let(:file_path2) { ::File.join(directory, "2#{suffix}.log") } + let(:file_path3) { ::File.join(directory, "3#{suffix}.log") } let(:max) { 4095 } let(:stat_interval) { 0.1 } let(:discover_interval) { 4 } - let(:start_new_files_at) { :beginning } + let(:start_new_files_at) { :end } let(:sincedb_path) { ::File.join(directory, "tailing.sdb") } let(:opts) do { :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_active => max, - :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path + :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path, + :file_sort_by => "path" } end let(:observer) { TestObserver.new } + let(:listener1) { observer.listener_for(file_path) } + let(:listener2) { observer.listener_for(file_path2) } + let(:listener3) { observer.listener_for(file_path3) } let(:tailing) { ObservingTail.new(opts) } + before do + directory + wait(1.0).for{Dir.exist?(directory)}.to eq(true) + end + after do FileUtils.rm_rf(directory) + wait(1.0).for{Dir.exist?(directory)}.to eq(false) end describe "max open files (set to 1)" do let(:max) { 1 } - let(:file_path2) { File.join(directory, "2.log") } let(:wait_before_quit) { 0.15 } let(:stat_interval) { 0.01 } let(:discover_interval) { 4 } + let(:start_new_files_at) { :beginning } let(:actions) do RSpec::Sequencing .run_after(wait_before_quit, "quit after a short time") do @@ -51,94 +54,110 @@ module FileWatch before do ENV["FILEWATCH_MAX_FILES_WARN_INTERVAL"] = "0" File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } - File.open(file_path2, "wb") { |file| file.write("lineA\nlineB\n") } + File.open(file_path2, "wb") { |file| file.write("line-A\nline-B\n") } end context "when max_active is 1" do + let(:suffix) { "A" } it "without close_older set, opens only 1 file" do - actions.activate + actions.activate_quietly + # create files before first discovery, they will be read from the end tailing.watch_this(watch_dir) tailing.subscribe(observer) + actions.assert_no_errors expect(tailing.settings.max_active).to eq(max) - file1_calls = observer.listener_for(file_path).calls - file2_calls = observer.listener_for(file_path2).calls - # file glob order is OS dependent - if file1_calls.empty? - expect(observer.listener_for(file_path2).lines).to eq(["lineA", "lineB"]) - expect(file2_calls).to eq([:open, :accept, :accept]) - else - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) - expect(file1_calls).to eq([:open, :accept, :accept]) - expect(file2_calls).to be_empty - end + expect(listener1.lines).to eq(["line1", "line2"]) + expect(listener1.calls).to eq([:open, :accept, :accept]) + expect(listener2.calls).to be_empty end end context "when close_older is set" do let(:wait_before_quit) { 0.8 } - let(:opts) { super.merge(:close_older => 0.15, :max_active => 1, :stat_interval => 0.1) } + let(:opts) { super.merge(:close_older => 0.1, :max_active => 1, :stat_interval => 0.1) } + let(:suffix) { "B" } it "opens both files" do - actions.activate + actions.activate_quietly tailing.watch_this(watch_dir) tailing.subscribe(observer) + actions.assert_no_errors expect(tailing.settings.max_active).to eq(1) - filelistener_1 = observer.listener_for(file_path) - filelistener_2 = observer.listener_for(file_path2) - expect(filelistener_2.calls).to eq([:open, :accept, :accept, :timed_out]) - expect(filelistener_2.lines).to eq(["lineA", "lineB"]) - expect(filelistener_1.calls).to eq([:open, :accept, :accept, :timed_out]) - expect(filelistener_1.lines).to eq(["line1", "line2"]) + expect(listener2.calls).to eq([:open, :accept, :accept, :timed_out]) + expect(listener2.lines).to eq(["line-A", "line-B"]) + expect(listener1.calls).to eq([:open, :accept, :accept, :timed_out]) + expect(listener1.lines).to eq(["line1", "line2"]) end end end - context "when watching a directory with files" do - let(:start_new_files_at) { :beginning } + context "when watching a directory with files, exisiting content is skipped" do + let(:suffix) { "C" } let(:actions) do - RSpec::Sequencing.run_after(0.45, "quit after a short time") do - tailing.quit - end + RSpec::Sequencing + .run("create file") do + File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") } + end + .then_after(0.1, "begin watching") do + tailing.watch_this(watch_dir) + end + .then_after(0.05, "add content") do + File.open(file_path, "ab") { |file| file.write("line1\nline2\n") } + end + .then("wait") do + wait(0.75).for{listener1.lines.size}.to eq(2) + end + .then("quit") do + tailing.quit + end end - it "the file is read" do - File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } - actions.activate + it "only the new content is read" do + actions.activate_quietly tailing.watch_this(watch_dir) tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept]) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept]) + expect(listener1.lines).to eq(["line1", "line2"]) end end context "when watching a directory without files and one is added" do - let(:start_new_files_at) { :beginning } - before do - tailing.watch_this(watch_dir) + let(:suffix) { "D" } + let(:actions) do RSpec::Sequencing - .run_after(0.25, "create file") do + .run("begin watching") do + tailing.watch_this(watch_dir) + end + .then_after(0.1, "create file") do File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } end - .then_after(0.45, "quit after a short time") do + .then("wait") do + wait(0.75).for{listener1.lines.size}.to eq(2) + end + .then("quit") do tailing.quit end end - it "the file is read" do + it "the file is read from the beginning" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept]) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept]) + expect(listener1.lines).to eq(["line1", "line2"]) end end - describe "given a previously discovered file" do + context "given a previously discovered file" do # these tests rely on the fact that the 'filepath' does not exist on disk # it simulates that the user deleted the file # so when a stat is taken on the file an error is raised + let(:suffix) { "E" } let(:quit_after) { 0.2 } - let(:stat) { double("stat", :size => 100, :ctime => Time.now, :mtime => Time.now, :ino => 234567, :dev_major => 3, :dev_minor => 2) } + let(:stat) { double("stat", :size => 100, :modified_at => Time.now.to_f, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) } let(:watched_file) { WatchedFile.new(file_path, stat, tailing.settings) } - before do + allow(stat).to receive(:restat).and_raise(Errno::ENOENT) tailing.watch.watched_files_collection.add(watched_file) watched_file.initial_completed end @@ -150,7 +169,7 @@ module FileWatch RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit } tailing.subscribe(observer) expect(tailing.watch.watched_files_collection).to be_empty - expect(observer.listener_for(file_path).calls).to eq([:delete]) + expect(listener1.calls).to eq([:delete]) end end @@ -160,7 +179,7 @@ module FileWatch RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit } tailing.subscribe(observer) expect(tailing.watch.watched_files_collection).to be_empty - expect(observer.listener_for(file_path).calls).to eq([:delete]) + expect(listener1.calls).to eq([:delete]) end end @@ -170,7 +189,7 @@ module FileWatch RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit } tailing.subscribe(observer) expect(tailing.watch.watched_files_collection).to be_empty - expect(observer.listener_for(file_path).calls).to eq([:delete]) + expect(listener1.calls).to eq([:delete]) end end @@ -180,172 +199,214 @@ module FileWatch RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit } tailing.subscribe(observer) expect(tailing.watch.watched_files_collection).to be_empty - expect(observer.listener_for(file_path).calls).to eq([:delete]) + expect(listener1.calls).to eq([:delete]) end end end context "when a processed file shrinks" do - let(:discover_interval) { 100 } - before do + let(:discover_interval) { 1 } + let(:suffix) { "F" } + let(:actions) do RSpec::Sequencing - .run("create file") do + .run_after(0.1, "start watching") do + tailing.watch_this(watch_dir) + end + .then_after(0.1, "create file") do + # create file after first discovery, will be read from the start File.open(file_path, "wb") { |file| file.write("line1\nline2\nline3\nline4\n") } end - .then_after(0.25, "start watching after files are written") do - tailing.watch_this(watch_dir) + .then("wait for initial lines to be read") do + wait(0.8).for{listener1.lines.size}.to eq(4), "listener1.lines.size not eq 4" end .then_after(0.25, "truncate file and write new content") do File.truncate(file_path, 0) - File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") } + File.open(file_path, "ab") { |file| file.write("lineA\nlineB\n") } + wait(0.5).for{listener1.lines.size}.to eq(6), "listener1.lines.size not eq 6" end - .then_after(0.25, "quit after a short time") do + .then("quit") do tailing.quit end end it "new changes to the shrunk file are read from the beginning" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :accept, :accept, :accept, :accept]) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4", "lineA", "lineB"]) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept, :accept, :accept, :accept, :accept]) + expect(listener1.lines).to eq(["line1", "line2", "line3", "line4", "lineA", "lineB"]) end end - context "when watching a directory with files and a file is renamed to not match glob" do + context "when watching a directory with files and a file is renamed to not match glob", :unix => true do + let(:suffix) { "G" } let(:new_file_path) { file_path + ".old" } - before do + let(:new_file_listener) { observer.listener_for(new_file_path) } + let(:actions) do RSpec::Sequencing - .run("create file") do - File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } - end - .then_after(0.25, "start watching after files are written") do + .run("start watching") do tailing.watch_this(watch_dir) end + .then_after(0.1, "create file") do + # create file after first discovery, will be read from the beginning + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end .then_after(0.55, "rename file") do FileUtils.mv(file_path, new_file_path) end .then_after(0.55, "then write to renamed file") do File.open(new_file_path, "ab") { |file| file.write("line3\nline4\n") } + wait(0.5).for{listener1.lines.size}.to eq(2), "listener1.lines.size not eq(2)" end - .then_after(0.45, "quit after a short time") do + .then_after(0.1, "quit") do tailing.quit end end it "changes to the renamed file are not read" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :delete]) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) - expect(observer.listener_for(new_file_path).calls).to eq([]) - expect(observer.listener_for(new_file_path).lines).to eq([]) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept, :delete]) + expect(listener1.lines).to eq(["line1", "line2"]) + expect(new_file_listener.calls).to eq([]) + expect(new_file_listener.lines).to eq([]) end end - context "when watching a directory with files and a file is renamed to match glob" do - let(:new_file_path) { file_path + "2.log" } + context "when watching a directory with files and a file is renamed to match glob", :unix => true do + let(:suffix) { "H" } let(:opts) { super.merge(:close_older => 0) } - before do + let(:listener2) { observer.listener_for(file_path2) } + let(:actions) do RSpec::Sequencing - .run("create file") do + .run("file created") do + # create file before first discovery, will be read from the end File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } end .then_after(0.15, "start watching after files are written") do tailing.watch_this(watch_dir) end - .then_after(0.25, "rename file") do - FileUtils.mv(file_path, new_file_path) + .then("wait") do + wait(0.5).for{listener1.calls.last}.to eq(:timed_out) end - .then("then write to renamed file") do - File.open(new_file_path, "ab") { |file| file.write("line3\nline4\n") } + .then("rename file") do + FileUtils.mv(file_path, file_path2) + end + .then_after(0.1, "then write to renamed file") do + File.open(file_path2, "ab") { |file| file.write("line3\nline4\n") } end - .then_after(0.55, "quit after a short time") do + .then_after(0.1, "wait for lines") do + wait(0.5).for{listener2.lines.size}.to eq(2) + end + .then_after(0.1, "quit") do tailing.quit end end it "the first set of lines are not re-read" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out, :delete]) - expect(observer.listener_for(new_file_path).lines).to eq(["line3", "line4"]) - expect(observer.listener_for(new_file_path).calls).to eq([:open, :accept, :accept, :timed_out]) + actions.assert_no_errors + expect(listener1.lines).to eq([]) + expect(listener1.calls).to eq([:open, :timed_out, :delete]) + expect(listener2.lines).to eq(["line3", "line4"]) + expect(listener2.calls).to eq([:open, :accept, :accept, :timed_out]) end end context "when watching a directory with files and data is appended" do - before do + let(:suffix) { "I" } + let(:actions) do RSpec::Sequencing - .run("create file") do + .run("file created") do File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } end - .then_after(0.25, "start watching after file is written") do + .then_after(0.15, "start watching after file is written") do tailing.watch_this(watch_dir) end .then_after(0.45, "append more lines to the file") do File.open(file_path, "ab") { |file| file.write("line3\nline4\n") } + wait(0.5).for{listener1.lines.size}.to eq(2) end - .then_after(0.45, "quit after a short time") do + .then_after(0.1, "quit") do tailing.quit end end - it "appended lines are read after an EOF" do + it "appended lines are read only" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :accept, :accept]) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4"]) + actions.assert_no_errors + expect(listener1.calls).to eq([:open, :accept, :accept]) + expect(listener1.lines).to eq(["line3", "line4"]) end end context "when close older expiry is enabled" do let(:opts) { super.merge(:close_older => 1) } - before do - RSpec::Sequencing - .run("create file") do - File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } - end - .then("start watching before file ages more than close_older") do - tailing.watch_this(watch_dir) - end - .then_after(2.1, "quit after allowing time to close the file") do - tailing.quit - end + let(:suffix) { "J" } + let(:actions) do + RSpec::Sequencing.run("create file") do + File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then("watch and wait") do + tailing.watch_this(watch_dir) + wait(1.25).for{listener1.calls}.to eq([:open, :timed_out]) + end + .then("quit") do + tailing.quit + end end - it "lines are read and the file times out" do + it "existing lines are not read and the file times out" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out]) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + actions.assert_no_errors + expect(listener1.lines).to eq([]) end end context "when close older expiry is enabled and after timeout the file is appended-to" do - let(:opts) { super.merge(:close_older => 1) } - before do + let(:opts) { super.merge(:close_older => 0.5) } + let(:suffix) { "K" } + let(:actions) do RSpec::Sequencing - .run("create file") do + .run("start watching") do + tailing.watch_this(watch_dir) + end + .then("create file") do File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } end - .then("start watching before file ages more than close_older") do - tailing.watch_this(watch_dir) + .then("wait for file to be read") do + wait(0.5).for{listener1.calls}.to eq([:open, :accept, :accept]), "file is not read" end - .then_after(2.1, "append more lines to file after file ages more than close_older") do + .then("wait for file to be read and time out") do + wait(0.75).for{listener1.calls}.to eq([:open, :accept, :accept, :timed_out]), "file did not timeout the first time" + end + .then("append more lines to file after file ages more than close_older") do File.open(file_path, "ab") { |file| file.write("line3\nline4\n") } end - .then_after(2.1, "quit after allowing time to close the file") do + .then("wait for last timeout") do + wait(0.75).for{listener1.calls}.to eq([:open, :accept, :accept, :timed_out, :open, :accept, :accept, :timed_out]), "file did not timeout the second time" + end + .then("quit") do tailing.quit end end it "all lines are read" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out, :open, :accept, :accept, :timed_out]) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4"]) + actions.assert_no_errors + expect(listener1.lines).to eq(["line1", "line2", "line3", "line4"]) end end context "when ignore older expiry is enabled and all files are already expired" do let(:opts) { super.merge(:ignore_older => 1) } - before do + let(:suffix) { "L" } + let(:actions) do RSpec::Sequencing .run("create file older than ignore_older and watch") do File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } @@ -358,60 +419,112 @@ module FileWatch end it "no files are read" do + actions.activate_quietly + tailing.subscribe(observer) + expect(listener1.calls).to eq([]) + expect(listener1.lines).to eq([]) + end + end + + context "when a file is renamed before it gets activated", :unix => true do + let(:max) { 1 } + let(:opts) { super.merge(:file_chunk_count => 8, :file_chunk_size => 6, :close_older => 0.1, :discover_interval => 6) } + let(:suffix) { "M" } + let(:start_new_files_at) { :beginning } # we are creating files and sincedb record before hand + let(:actions) do + RSpec::Sequencing + .run("create files and sincedb record") do + File.open(file_path, "wb") { |file| 32.times{file.write("line1\n")} } + File.open(file_path2, "wb") { |file| file.write("line2\n") } + # synthesize a sincedb record + stat = File.stat(file_path2) + record = [stat.ino.to_s, stat.dev_major.to_s, stat.dev_minor.to_s, "0", "1526220348.083179", file_path2] + File.open(sincedb_path, "wb") { |file| file.puts(record.join(" ")) } + end + .then_after(0.2, "watch") do + tailing.watch_this(watch_dir) + end + .then_after(0.1, "rename file 2") do + FileUtils.mv(file_path2, file_path3) + end + .then("wait") do + wait(2).for do + listener1.lines.size == 32 && listener2.calls == [:delete] && listener3.calls == [:open, :accept, :timed_out] + end.to eq(true), "listener1.lines != 32 or listener2.calls != [:delete] or listener3.calls != [:open, :accept, :timed_out]" + end + .then("quit") do + tailing.quit + end + end + + it "files are read correctly" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([]) - expect(observer.listener_for(file_path).lines).to eq([]) + actions.assert_no_errors + expect(listener2.lines).to eq([]) + expect(listener3.lines).to eq(["line2"]) end end context "when ignore_older is less than close_older and all files are not expired" do - let(:opts) { super.merge(:ignore_older => 1, :close_older => 1.5) } - before do + let(:opts) { super.merge(:ignore_older => 1, :close_older => 1.1) } + let(:suffix) { "N" } + let(:start_new_files_at) { :beginning } + let(:actions) do RSpec::Sequencing - .run("create file") do + .run_after(0.1, "file created") do File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } end .then("start watching before file age reaches ignore_older") do tailing.watch_this(watch_dir) end - .then_after(1.75, "quit after allowing time to close the file") do + .then("wait for lines") do + wait(1.5).for{listener1.calls}.to eq([:open, :accept, :accept, :timed_out]) + end + .then("quit") do tailing.quit end end it "reads lines normally" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out]) - expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"]) + actions.assert_no_errors + expect(listener1.lines).to eq(["line1", "line2"]) end end context "when ignore_older is less than close_older and all files are expired" do let(:opts) { super.merge(:ignore_older => 10, :close_older => 1) } - before do + let(:suffix) { "P" } + let(:actions) do RSpec::Sequencing - .run("create file older than ignore_older and watch") do + .run("creating file") do File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } + end + .then("making it older by 15 seconds and watch") do FileWatch.make_file_older(file_path, 15) tailing.watch_this(watch_dir) end - .then_after(1.5, "quit after allowing time to check the files") do + .then_after(0.75, "quit after allowing time to check the files") do tailing.quit end end it "no files are read" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([]) - expect(observer.listener_for(file_path).lines).to eq([]) + expect(listener1.calls).to eq([]) + expect(listener1.lines).to eq([]) end end context "when ignore older and close older expiry is enabled and after timeout the file is appended-to" do - let(:opts) { super.merge(:ignore_older => 20, :close_older => 1) } - before do + let(:opts) { super.merge(:ignore_older => 20, :close_older => 0.5) } + let(:suffix) { "Q" } + let(:actions) do RSpec::Sequencing - .run("create file older than ignore_older and watch") do + .run("file older than ignore_older created and watching") do File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } FileWatch.make_file_older(file_path, 25) tailing.watch_this(watch_dir) @@ -419,37 +532,46 @@ module FileWatch .then_after(0.15, "append more lines to file after file ages more than ignore_older") do File.open(file_path, "ab") { |file| file.write("line3\nline4\n") } end - .then_after(1.25, "quit after allowing time to close the file") do + .then("wait for lines") do + wait(2).for{listener1.calls}.to eq([:open, :accept, :accept, :timed_out]) + end + .then_after(0.1, "quit after allowing time to close the file") do tailing.quit end end it "reads the added lines only" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).lines).to eq(["line3", "line4"]) - expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out]) + actions.assert_no_errors + expect(listener1.lines).to eq(["line3", "line4"]) end end context "when a non default delimiter is specified and it is not in the content" do let(:opts) { super.merge(:ignore_older => 20, :close_older => 1, :delimiter => "\nø") } - before do + let(:suffix) { "R" } + let(:actions) do RSpec::Sequencing - .run("create file") do + .run("start watching") do + tailing.watch_this(watch_dir) + end + .then("creating file") do File.open(file_path, "wb") { |file| file.write("line1\nline2") } end - .then("start watching before file ages more than close_older") do - tailing.watch_this(watch_dir) + .then("wait for :timeout") do + wait(2).for{listener1.calls}.to eq([:open, :timed_out]) end - .then_after(2.1, "quit after allowing time to close the file") do + .then_after(0.75, "quit after allowing time to close the file") do tailing.quit end end it "the file is opened, data is read, but no lines are found, the file times out" do + actions.activate_quietly tailing.subscribe(observer) - expect(observer.listener_for(file_path).calls).to eq([:open, :timed_out]) - expect(observer.listener_for(file_path).lines).to eq([]) + actions.assert_no_errors + expect(listener1.lines).to eq([]) sincedb_record_fields = File.read(sincedb_path).split(" ") position_field_index = 3 # tailing, no delimiter, we are expecting one, if it grows we read from the start. @@ -459,5 +581,3 @@ module FileWatch end end end - - diff --git a/spec/filewatch/watched_file_spec.rb b/spec/filewatch/watched_file_spec.rb index adf0a7e..a532ac1 100644 --- a/spec/filewatch/watched_file_spec.rb +++ b/spec/filewatch/watched_file_spec.rb @@ -8,9 +8,9 @@ module FileWatch context 'Given two instances of the same file' do it 'their sincedb_keys should equate' do - wf_key1 = WatchedFile.new(pathname, pathname.stat, Settings.new).sincedb_key + wf_key1 = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new).sincedb_key hash_db = { wf_key1 => 42 } - wf_key2 = WatchedFile.new(pathname, pathname.stat, Settings.new).sincedb_key + wf_key2 = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new).sincedb_key expect(wf_key1).to eq(wf_key2) expect(wf_key1).to eql(wf_key2) expect(wf_key1.hash).to eq(wf_key2.hash) @@ -20,7 +20,7 @@ module FileWatch context 'Given a barrage of state changes' do it 'only the previous N state changes are remembered' do - watched_file = WatchedFile.new(pathname, pathname.stat, Settings.new) + watched_file = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new) watched_file.ignore watched_file.watch watched_file.activate diff --git a/spec/filewatch/watched_files_collection_spec.rb b/spec/filewatch/watched_files_collection_spec.rb index 06eba28..08130d5 100644 --- a/spec/filewatch/watched_files_collection_spec.rb +++ b/spec/filewatch/watched_files_collection_spec.rb @@ -4,9 +4,9 @@ module FileWatch describe WatchedFilesCollection do let(:time) { Time.now } - let(:stat1) { double("stat1", :size => 98, :ctime => time - 30, :mtime => time - 30, :ino => 234567, :dev_major => 3, :dev_minor => 2) } - let(:stat2) { double("stat2", :size => 99, :ctime => time - 20, :mtime => time - 20, :ino => 234568, :dev_major => 3, :dev_minor => 2) } - let(:stat3) { double("stat3", :size => 100, :ctime => time, :mtime => time, :ino => 234569, :dev_major => 3, :dev_minor => 2) } + let(:stat1) { double("stat1", :size => 98, :modified_at => time - 30, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) } + let(:stat2) { double("stat2", :size => 99, :modified_at => time - 20, :identifier => nil, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) } + let(:stat3) { double("stat3", :size => 100, :modified_at => time, :identifier => nil, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) } let(:wf1) { WatchedFile.new("/var/log/z.log", stat1, Settings.new) } let(:wf2) { WatchedFile.new("/var/log/m.log", stat2, Settings.new) } let(:wf3) { WatchedFile.new("/var/log/a.log", stat3, Settings.new) } diff --git a/spec/filewatch/winhelper_spec.rb b/spec/filewatch/winhelper_spec.rb index 9fb2b78..a0d874c 100644 --- a/spec/filewatch/winhelper_spec.rb +++ b/spec/filewatch/winhelper_spec.rb @@ -3,7 +3,7 @@ require "fileutils" if Gem.win_platform? - require "lib/filewatch/winhelper" + require "filewatch/winhelper" describe Winhelper do let(:path) { Stud::Temporary.file.path } @@ -13,11 +13,10 @@ end it "return a unique file identifier" do - volume_serial, file_index_low, file_index_high = Winhelper.GetWindowsUniqueFileIdentifier(path).split("").map(&:to_i) + identifier = Winhelper.identifier_from_path(path) - expect(volume_serial).not_to eq(0) - expect(file_index_low).not_to eq(0) - expect(file_index_high).not_to eq(0) + expect(identifier).not_to eq("unknown") + expect(identifier.count("-")).to eq(2) end end end diff --git a/spec/helpers/logging_level_helper.rb b/spec/helpers/logging_level_helper.rb new file mode 100644 index 0000000..0ecc433 --- /dev/null +++ b/spec/helpers/logging_level_helper.rb @@ -0,0 +1,8 @@ +# encoding: utf-8 + +ENV["LOG_AT"].tap do |level| + if !level.nil? + LogStash::Logging::Logger::configure_logging(level) + LOG_AT_HANDLED = true + end +end diff --git a/spec/helpers/rspec_wait_handler_helper.rb b/spec/helpers/rspec_wait_handler_helper.rb new file mode 100644 index 0000000..2ad595b --- /dev/null +++ b/spec/helpers/rspec_wait_handler_helper.rb @@ -0,0 +1,38 @@ +# encoding: utf-8 + +module RSpec + module Wait + module Handler + def handle_matcher(target, *args, &block) + # there is a similar patch in the rspec-wait repo since Nov, 19 2017 + # it does not look like the author is interested in the change. + # - do not use Ruby Timeout + count = RSpec.configuration.wait_timeout.fdiv(RSpec.configuration.wait_delay).ceil + failure = nil + count.times do + begin + actual = target.respond_to?(:call) ? target.call : target + super(actual, *args, &block) + failure = nil + rescue RSpec::Expectations::ExpectationNotMetError => failure + sleep RSpec.configuration.wait_delay + end + break if failure.nil? + end + raise failure unless failure.nil? + end + end + + # From: https://github.com/rspec/rspec-expectations/blob/v3.0.0/lib/rspec/expectations/handler.rb#L44-L63 + class PositiveHandler < RSpec::Expectations::PositiveExpectationHandler + extend Handler + end + + # From: https://github.com/rspec/rspec-expectations/blob/v3.0.0/lib/rspec/expectations/handler.rb#L66-L93 + class NegativeHandler < RSpec::Expectations::NegativeExpectationHandler + extend Handler + end + end +end + +RSPEC_WAIT_HANDLER_PATCHED = true diff --git a/spec/helpers/spec_helper.rb b/spec/helpers/spec_helper.rb index dc725a6..70bfd8e 100644 --- a/spec/helpers/spec_helper.rb +++ b/spec/helpers/spec_helper.rb @@ -18,7 +18,7 @@ def self.make_fixture_current(path, time = Time.now) class TracerBase def initialize - @tracer = [] + @tracer = Concurrent::Array.new end def trace_for(symbol) @@ -54,6 +54,9 @@ def clone end end +require_relative "rspec_wait_handler_helper" unless defined? RSPEC_WAIT_HANDLER_PATCHED +require_relative "logging_level_helper" unless defined? LOG_AT_HANDLED + unless RSpec::Matchers.method_defined?(:receive_call_and_args) RSpec::Matchers.define(:receive_call_and_args) do |m, args| match do |actual| @@ -66,3 +69,6 @@ def clone end end +ENV["LOG_AT"].tap do |level| + LogStash::Logging::Logger::configure_logging(level) unless level.nil? +end diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index 67fe206..d7361bb 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -9,21 +9,21 @@ require "stud/temporary" require "logstash/codecs/multiline" -FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n" - describe LogStash::Inputs::File do describe "'read' mode testing with input(conf) do |pipeline, queue|" do it "should start at the beginning of an existing file and delete the file when done" do - tmpfile_path = Stud::Temporary.pathname - sincedb_path = Stud::Temporary.pathname + directory = Stud::Temporary.directory + tmpfile_path = ::File.join(directory, "A.log") + sincedb_path = ::File.join(directory, "readmode_A_sincedb.txt") + path_path = ::File.join(directory, "*.log") conf = <<-CONFIG input { file { - type => "blah" - path => "#{tmpfile_path}" + id => "blah" + path => "#{path_path}" sincedb_path => "#{sincedb_path}" - delimiter => "#{FILE_DELIMITER}" + delimiter => "|" mode => "read" file_completed_action => "delete" } @@ -31,17 +31,49 @@ CONFIG File.open(tmpfile_path, "a") do |fd| - fd.puts("hello") - fd.puts("world") + fd.write("hello|world") fd.fsync end events = input(conf) do |pipeline, queue| + wait(0.5).for{File.exist?(tmpfile_path)}.to be_falsey 2.times.collect { queue.pop } end expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world") - expect(File.exist?(tmpfile_path)).to be_falsey + end + + it "should start at the beginning of an existing file and log the file when done" do + directory = Stud::Temporary.directory + tmpfile_path = ::File.join(directory, "A.log") + sincedb_path = ::File.join(directory, "readmode_A_sincedb.txt") + path_path = ::File.join(directory, "*.log") + log_completed_path = ::File.join(directory, "A_completed.txt") + + conf = <<-CONFIG + input { + file { + id => "blah" + path => "#{path_path}" + sincedb_path => "#{sincedb_path}" + delimiter => "|" + mode => "read" + file_completed_action => "log" + file_completed_log_path => "#{log_completed_path}" + } + } + CONFIG + + File.open(tmpfile_path, "a") do |fd| + fd.write("hello|world") + fd.fsync + end + + events = input(conf) do |pipeline, queue| + wait(0.5).for{IO.read(log_completed_path)}.to match(/A\.log/) + 2.times.collect { queue.pop } + end + expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world") end end @@ -63,7 +95,6 @@ type => "blah" path => "#{tmpfile_path}" sincedb_path => "#{sincedb_path}" - delimiter => "#{FILE_DELIMITER}" mode => "read" file_completed_action => "log" file_completed_log_path => "#{log_completed_path}" @@ -72,12 +103,12 @@ CONFIG events = input(conf) do |pipeline, queue| + wait(0.5).for{IO.read(log_completed_path)}.to match(/#{file_path.to_s}/) 2.times.collect { queue.pop } end expect(events[0].get("message")).to start_with("2010-03-12 23:51") expect(events[1].get("message")).to start_with("2010-03-12 23:51") - expect(IO.read(log_completed_path)).to eq(file_path.to_s + "\n") end end @@ -86,10 +117,11 @@ let(:file_path) { fixture_dir.join('uncompressed.log') } it "the file is read and the path is logged to the `file_completed_log_path` file" do - tmpfile_path = fixture_dir.join("unc*.log") - sincedb_path = Stud::Temporary.pathname FileInput.make_fixture_current(file_path.to_path) - log_completed_path = Stud::Temporary.pathname + tmpfile_path = fixture_dir.join("unc*.log") + directory = Stud::Temporary.directory + sincedb_path = ::File.join(directory, "readmode_B_sincedb.txt") + log_completed_path = ::File.join(directory, "B_completed.txt") conf = <<-CONFIG input { @@ -97,7 +129,6 @@ type => "blah" path => "#{tmpfile_path}" sincedb_path => "#{sincedb_path}" - delimiter => "#{FILE_DELIMITER}" mode => "read" file_completed_action => "log" file_completed_log_path => "#{log_completed_path}" @@ -106,23 +137,25 @@ CONFIG events = input(conf) do |pipeline, queue| + wait(0.5).for{IO.read(log_completed_path)}.to match(/uncompressed\.log/) 2.times.collect { queue.pop } end expect(events[0].get("message")).to start_with("2010-03-12 23:51") expect(events[1].get("message")).to start_with("2010-03-12 23:51") - expect(IO.read(log_completed_path)).to eq(file_path.to_s + "\n") end end context "for a compressed file" do it "the file is read" do - tmpfile_path = fixture_dir.join("compressed.*.*") - sincedb_path = Stud::Temporary.pathname file_path = fixture_dir.join('compressed.log.gz') file_path2 = fixture_dir.join('compressed.log.gzip') FileInput.make_fixture_current(file_path.to_path) - log_completed_path = Stud::Temporary.pathname + FileInput.make_fixture_current(file_path2.to_path) + tmpfile_path = fixture_dir.join("compressed.*.*") + directory = Stud::Temporary.directory + sincedb_path = ::File.join(directory, "readmode_C_sincedb.txt") + log_completed_path = ::File.join(directory, "C_completed.txt") conf = <<-CONFIG input { @@ -130,7 +163,6 @@ type => "blah" path => "#{tmpfile_path}" sincedb_path => "#{sincedb_path}" - delimiter => "#{FILE_DELIMITER}" mode => "read" file_completed_action => "log" file_completed_log_path => "#{log_completed_path}" @@ -139,6 +171,7 @@ CONFIG events = input(conf) do |pipeline, queue| + wait(0.5).for{IO.read(log_completed_path).scan(/compressed\.log\.gz(ip)?/).size}.to eq(2) 4.times.collect { queue.pop } end @@ -146,9 +179,6 @@ expect(events[1].get("message")).to start_with("2010-03-12 23:51") expect(events[2].get("message")).to start_with("2010-03-12 23:51") expect(events[3].get("message")).to start_with("2010-03-12 23:51") - logged_completions = IO.read(log_completed_path).split - expect(logged_completions.first).to match(/compressed\.log\.(gzip|gz)$/) - expect(logged_completions.last).to match(/compressed\.log\.(gzip|gz)$/) end end end diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index 369906e..1250eba 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -9,7 +9,7 @@ # LogStash::Logging::Logger::configure_logging("DEBUG") -TEST_FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n" +TEST_FILE_DELIMITER = $/ describe LogStash::Inputs::File do describe "'tail' mode testing with input(conf) do |pipeline, queue|" do @@ -22,152 +22,159 @@ end end - it "should start at the beginning of an existing file" do - tmpfile_path = Stud::Temporary.pathname - sincedb_path = Stud::Temporary.pathname - - conf = <<-CONFIG - input { - file { - type => "blah" - path => "#{tmpfile_path}" - start_position => "beginning" - sincedb_path => "#{sincedb_path}" - delimiter => "#{TEST_FILE_DELIMITER}" + let(:directory) { Stud::Temporary.directory } + let(:sincedb_dir) { Stud::Temporary.directory } + let(:tmpfile_path) { ::File.join(directory, "#{name}.txt") } + let(:sincedb_path) { ::File.join(sincedb_dir, "readmode_#{name}_sincedb.txt") } + let(:path_path) { ::File.join(directory, "*.txt") } + + context "for an existing file" do + let(:name) { "A" } + it "should start at the beginning" do + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{path_path}" + start_position => "beginning" + sincedb_path => "#{sincedb_path}" + delimiter => "#{TEST_FILE_DELIMITER}" + } } - } - CONFIG + CONFIG - File.open(tmpfile_path, "a") do |fd| - fd.puts("hello") - fd.puts("world") - fd.fsync - end + File.open(tmpfile_path, "a") do |fd| + fd.puts("hello") + fd.puts("world") + fd.fsync + end - events = input(conf) do |pipeline, queue| - 2.times.collect { queue.pop } + events = input(conf) do |pipeline, queue| + 2.times.collect { queue.pop } + end + expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world") end - expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world") end - it "should restart at the sincedb value" do - tmpfile_path = Stud::Temporary.pathname - sincedb_path = Stud::Temporary.pathname - - conf = <<-CONFIG - input { - file { - type => "blah" - path => "#{tmpfile_path}" - start_position => "beginning" - sincedb_path => "#{sincedb_path}" - delimiter => "#{TEST_FILE_DELIMITER}" + context "running the input twice" do + let(:name) { "B" } + it "should restart at the sincedb value" do + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{path_path}" + start_position => "beginning" + sincedb_path => "#{sincedb_path}" + "file_sort_by" => "path" + delimiter => "#{TEST_FILE_DELIMITER}" + } } - } - CONFIG + CONFIG - File.open(tmpfile_path, "w") do |fd| - fd.puts("hello3") - fd.puts("world3") - end + File.open(tmpfile_path, "w") do |fd| + fd.puts("hello3") + fd.puts("world3") + end - events = input(conf) do |pipeline, queue| - 2.times.collect { queue.pop } - end + events = input(conf) do |pipeline, queue| + 2.times.collect { queue.pop } + end - expect(events.map{|e| e.get("message")}).to contain_exactly("hello3", "world3") + expect(events.map{|e| e.get("message")}).to contain_exactly("hello3", "world3") - File.open(tmpfile_path, "a") do |fd| - fd.puts("foo") - fd.puts("bar") - fd.puts("baz") - fd.fsync - end + File.open(tmpfile_path, "a") do |fd| + fd.puts("foo") + fd.puts("bar") + fd.puts("baz") + fd.fsync + end - events = input(conf) do |pipeline, queue| - 3.times.collect { queue.pop } + events = input(conf) do |pipeline, queue| + 3.times.collect { queue.pop } + end + messages = events.map{|e| e.get("message")} + expect(messages).to contain_exactly("foo", "bar", "baz") end - messages = events.map{|e| e.get("message")} - expect(messages).to contain_exactly("foo", "bar", "baz") end - it "should not overwrite existing path and host fields" do - tmpfile_path = Stud::Temporary.pathname - sincedb_path = Stud::Temporary.pathname - - conf = <<-CONFIG - input { - file { - type => "blah" - path => "#{tmpfile_path}" - start_position => "beginning" - sincedb_path => "#{sincedb_path}" - delimiter => "#{TEST_FILE_DELIMITER}" - codec => "json" + context "when path and host fields exist" do + let(:name) { "C" } + it "should not overwrite them" do + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{path_path}" + start_position => "beginning" + sincedb_path => "#{sincedb_path}" + delimiter => "#{TEST_FILE_DELIMITER}" + codec => "json" + } } - } - CONFIG + CONFIG - File.open(tmpfile_path, "w") do |fd| - fd.puts('{"path": "my_path", "host": "my_host"}') - fd.puts('{"my_field": "my_val"}') - fd.fsync - end + File.open(tmpfile_path, "w") do |fd| + fd.puts('{"path": "my_path", "host": "my_host"}') + fd.puts('{"my_field": "my_val"}') + fd.fsync + end - events = input(conf) do |pipeline, queue| - 2.times.collect { queue.pop } - end + events = input(conf) do |pipeline, queue| + 2.times.collect { queue.pop } + end - existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1] + existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1] - expect(events[existing_path_index].get("path")).to eq "my_path" - expect(events[existing_path_index].get("host")).to eq "my_host" - expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + expect(events[existing_path_index].get("path")).to eq "my_path" + expect(events[existing_path_index].get("host")).to eq "my_host" + expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" - expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}" - expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" - expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}" + expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + end end - it "should read old files" do - tmpfile_path = Stud::Temporary.pathname - - conf = <<-CONFIG - input { - file { - type => "blah" - path => "#{tmpfile_path}" - start_position => "beginning" - codec => "json" + context "running the input twice" do + let(:name) { "D" } + it "should read old files" do + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{path_path}" + start_position => "beginning" + codec => "json" + } } - } - CONFIG + CONFIG - File.open(tmpfile_path, "w") do |fd| - fd.puts('{"path": "my_path", "host": "my_host"}') - fd.puts('{"my_field": "my_val"}') - fd.fsync - end - # arbitrary old file (2 days) - FileInput.make_file_older(tmpfile_path, 48 * 60 * 60) + File.open(tmpfile_path, "w") do |fd| + fd.puts('{"path": "my_path", "host": "my_host"}') + fd.puts('{"my_field": "my_val"}') + fd.fsync + end + # arbitrary old file (2 days) + FileInput.make_file_older(tmpfile_path, 48 * 60 * 60) - events = input(conf) do |pipeline, queue| - 2.times.collect { queue.pop } + events = input(conf) do |pipeline, queue| + 2.times.collect { queue.pop } + end + existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1] + expect(events[existing_path_index].get("path")).to eq "my_path" + expect(events[existing_path_index].get("host")).to eq "my_host" + expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + + expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}" + expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" end - existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1] - expect(events[existing_path_index].get("path")).to eq "my_path" - expect(events[existing_path_index].get("host")).to eq "my_host" - expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" - - expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}" - expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" - expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" end - context "when sincedb_path is an existing directory" do - let(:tmpfile_path) { Stud::Temporary.pathname } - let(:sincedb_path) { Stud::Temporary.directory } - subject { LogStash::Inputs::File.new("path" => tmpfile_path, "sincedb_path" => sincedb_path) } + context "when sincedb_path is a directory" do + let(:name) { "E" } + subject { LogStash::Inputs::File.new("path" => path_path, "sincedb_path" => directory) } after :each do FileUtils.rm_rf(sincedb_path) @@ -180,16 +187,19 @@ end describe "testing with new, register, run and stop" do + let(:suffix) { "A" } let(:conf) { Hash.new } let(:mlconf) { Hash.new } let(:events) { Array.new } let(:mlcodec) { LogStash::Codecs::Multiline.new(mlconf) } - let(:codec) { FileInput::CodecTracer.new } - let(:tmpfile_path) { Stud::Temporary.pathname } - let(:sincedb_path) { Stud::Temporary.pathname } + let(:tracer_codec) { FileInput::CodecTracer.new } let(:tmpdir_path) { Stud::Temporary.directory } + let(:tmpfile_path) { ::File.join(tmpdir_path, "#{suffix}.txt") } + let(:path_path) { ::File.join(tmpdir_path, "*.txt") } + let(:sincedb_path) { ::File.join(tmpdir_path, "sincedb-#{suffix}") } after :each do + sleep(0.1) until subject.completely_stopped? FileUtils.rm_rf(sincedb_path) end @@ -204,7 +214,7 @@ end mlconf.update("pattern" => "^\s", "what" => "previous") conf.update("type" => "blah", - "path" => tmpfile_path, + "path" => path_path, "sincedb_path" => sincedb_path, "stat_interval" => 0.1, "codec" => mlcodec, @@ -213,16 +223,22 @@ it "reads the appended data only" do subject.register - RSpec::Sequencing - .run_after(0.2, "assert zero events then append two lines") do - expect(events.size).to eq(0) + actions = RSpec::Sequencing + .run_after(1, "append two lines after delay") do File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") } end - .then_after(0.4, "quit") do + .then("wait for one event") do + wait(0.75).for{events.size}.to eq(1) + end + .then("quit") do subject.stop end + .then("wait for flushed event") do + wait(0.75).for{events.size}.to eq(2) + end subject.run(events) + actions.assert_no_errors event1 = events[0] expect(event1).not_to be_nil @@ -240,218 +256,172 @@ context "when close_older config is specified" do let(:line) { "line1.1-of-a" } - + let(:suffix) { "X" } subject { described_class.new(conf) } before do conf.update( "type" => "blah", - "path" => "#{tmpdir_path}/*.log", + "path" => path_path, "sincedb_path" => sincedb_path, "stat_interval" => 0.02, - "codec" => codec, - "close_older" => 0.5, + "codec" => tracer_codec, + "close_older" => "100 ms", + "start_position" => "beginning", "delimiter" => TEST_FILE_DELIMITER) subject.register end - it "having timed_out, the identity is evicted" do - RSpec::Sequencing + it "having timed_out, the codec is auto flushed" do + actions = RSpec::Sequencing .run("create file") do - File.open("#{tmpdir_path}/a.log", "wb") { |file| file.puts(line) } + File.open(tmpfile_path, "wb") { |file| file.puts(line) } end - .then_after(0.3, "identity is mapped") do - expect(codec.trace_for(:accept)).to eq([true]) - expect(subject.codec.identity_count).to eq(1) + .then_after(0.1, "identity is mapped") do + wait(0.75).for{subject.codec.identity_map[tmpfile_path]}.not_to be_nil, "identity is not mapped" end - .then_after(0.3, "test for auto_flush") do - expect(codec.trace_for(:auto_flush)).to eq([true]) - expect(subject.codec.identity_count).to eq(0) + .then("wait for auto_flush") do + wait(0.75).for{subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)}.to eq([true]), "autoflush didn't" end - .then_after(0.1, "quit") do + .then("quit") do subject.stop end subject.run(events) + actions.assert_no_errors + expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true]) end end context "when ignore_older config is specified" do - let(:line) { "line1.1-of-a" } - let(:tmp_dir_file) { "#{tmpdir_path}/a.log" } - - subject { described_class.new(conf) } - + let(:suffix) { "Y" } before do - File.open(tmp_dir_file, "a") do |fd| - fd.puts(line) - fd.fsync - end - FileInput.make_file_older(tmp_dir_file, 2) conf.update( "type" => "blah", - "path" => "#{tmpdir_path}/*.log", + "path" => path_path, "sincedb_path" => sincedb_path, "stat_interval" => 0.02, - "codec" => codec, - "ignore_older" => 1, + "codec" => tracer_codec, + "ignore_older" => "500 ms", "delimiter" => TEST_FILE_DELIMITER) - - subject.register - Thread.new { subject.run(events) } end + subject { described_class.new(conf) } + let(:line) { "line1.1-of-a" } it "the file is not read" do - sleep 0.1 - subject.stop - expect(codec).to receive_call_and_args(:accept, false) - expect(codec).to receive_call_and_args(:auto_flush, false) - expect(subject.codec.identity_count).to eq(0) + subject.register + RSpec::Sequencing + .run("create file") do + File.open(tmp_dir_file, "a") do |fd| + fd.puts(line) + fd.fsync + end + FileInput.make_file_older(tmp_dir_file, 2) + end + .then_after(0.5, "stop") do + subject.stop + end + subject.run(events) + expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to be_falsey end end context "when wildcard path and a multiline codec is specified" do subject { described_class.new(conf) } - + let(:suffix) { "J" } + let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") } before do mlconf.update("pattern" => "^\s", "what" => "previous") conf.update( "type" => "blah", - "path" => "#{tmpdir_path}/*.log", + "path" => path_path, + "start_position" => "beginning", "sincedb_path" => sincedb_path, "stat_interval" => 0.05, "codec" => mlcodec, + "file_sort_by" => "path", "delimiter" => TEST_FILE_DELIMITER) subject.register end it "collects separate multiple line events from each file" do + subject actions = RSpec::Sequencing .run_after(0.1, "create files") do - File.open("#{tmpdir_path}/A.log", "wb") do |fd| - fd.puts("line1.1-of-a") - fd.puts(" line1.2-of-a") - fd.puts(" line1.3-of-a") + File.open(tmpfile_path, "wb") do |fd| + fd.puts("line1.1-of-J") + fd.puts(" line1.2-of-J") + fd.puts(" line1.3-of-J") end - File.open("#{tmpdir_path}/z.log", "wb") do |fd| - fd.puts("line1.1-of-z") - fd.puts(" line1.2-of-z") - fd.puts(" line1.3-of-z") + File.open(tmpfile_path2, "wb") do |fd| + fd.puts("line1.1-of-K") + fd.puts(" line1.2-of-K") + fd.puts(" line1.3-of-K") end end - .then_after(0.2, "assert both files are mapped as identities and stop") do - expect(subject.codec.identity_count).to eq(2) + .then("assert both files are mapped as identities and stop") do + wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities" end - .then_after(0.1, "stop") do + .then("stop") do subject.stop end - .then_after(0.2 , "stop flushes both events") do - expect(events.size).to eq(2) - e1, e2 = events - e1_message = e1.get("message") - e2_message = e2.get("message") - - # can't assume File A will be read first - if e1_message.start_with?('line1.1-of-z') - expect(e1.get("path")).to match(/z.log/) - expect(e2.get("path")).to match(/A.log/) - expect(e1_message).to eq("line1.1-of-z#{TEST_FILE_DELIMITER} line1.2-of-z#{TEST_FILE_DELIMITER} line1.3-of-z") - expect(e2_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") - else - expect(e1.get("path")).to match(/A.log/) - expect(e2.get("path")).to match(/z.log/) - expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") - expect(e2_message).to eq("line1.1-of-z#{TEST_FILE_DELIMITER} line1.2-of-z#{TEST_FILE_DELIMITER} line1.3-of-z") - end - end subject.run(events) # wait for actions to complete - actions.value + actions.assert_no_errors + expect(events.size).to eq(2) + e1, e2 = events + e1_message = e1.get("message") + e2_message = e2.get("message") + + expect(e1.get("path")).to match(/J.txt/) + expect(e2.get("path")).to match(/K.txt/) + expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER} line1.2-of-J#{TEST_FILE_DELIMITER} line1.3-of-J") + expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER} line1.2-of-K#{TEST_FILE_DELIMITER} line1.3-of-K") end context "if auto_flush is enabled on the multiline codec" do let(:mlconf) { { "auto_flush_interval" => 0.5 } } - + let(:suffix) { "M" } it "an event is generated via auto_flush" do actions = RSpec::Sequencing .run_after(0.1, "create files") do - File.open("#{tmpdir_path}/A.log", "wb") do |fd| + File.open(tmpfile_path, "wb") do |fd| fd.puts("line1.1-of-a") fd.puts(" line1.2-of-a") fd.puts(" line1.3-of-a") end end - .then_after(0.75, "wait for auto_flush") do - e1 = events.first - e1_message = e1.get("message") - expect(e1["path"]).to match(/a.log/) - expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") + .then("wait for auto_flush") do + wait(2).for{events.size}.to eq(1), "events size is not 1" end .then("stop") do subject.stop end subject.run(events) # wait for actions to complete - actions.value + actions.assert_no_errors + e1 = events.first + e1_message = e1.get("message") + expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") + expect(e1.get("path")).to match(/M.txt$/) end end end - context "when #run is called multiple times", :unix => true do - let(:file_path) { "#{tmpdir_path}/a.log" } - let(:buffer) { [] } - let(:run_thread_proc) do - lambda { Thread.new { subject.run(buffer) } } - end - let(:lsof_proc) do - lambda { `lsof -p #{Process.pid} | grep #{file_path}` } - end - - subject { described_class.new(conf) } - - before do - conf.update( - "path" => tmpdir_path + "/*.log", - "start_position" => "beginning", - "stat_interval" => 0.1, - "sincedb_path" => sincedb_path) - - File.open(file_path, "w") do |fd| - fd.puts('foo') - fd.puts('bar') - fd.fsync - end - end - - it "should only actually open files when content changes are detected" do - subject.register - expect(lsof_proc.call).to eq("") - # first run processes the file and records sincedb progress - run_thread_proc.call - wait(1).for{lsof_proc.call.scan(file_path).size}.to eq(1) - # second run quits the first run - # sees the file has not changed size and does not open it - run_thread_proc.call - wait(1).for{lsof_proc.call.scan(file_path).size}.to eq(0) - # truncate and write less than before - File.open(file_path, "w"){ |fd| fd.puts('baz'); fd.fsync } - # sees the file has changed size and does open it - wait(1).for{lsof_proc.call.scan(file_path).size}.to eq(1) - end - end - describe "specifying max_open_files" do + let(:suffix) { "P" } + let(:tmpfile_path2) { ::File.join(tmpdir_path, "Q.txt") } subject { described_class.new(conf) } before do - File.open("#{tmpdir_path}/a.log", "w") do |fd| - fd.puts("line1-of-a") - fd.puts("line2-of-a") + File.open(tmpfile_path, "w") do |fd| + fd.puts("line1-of-P") + fd.puts("line2-of-P") fd.fsync end - File.open("#{tmpdir_path}/z.log", "w") do |fd| - fd.puts("line1-of-z") - fd.puts("line2-of-z") + File.open(tmpfile_path2, "w") do |fd| + fd.puts("line1-of-Q") + fd.puts("line2-of-Q") fd.fsync end end @@ -461,37 +431,34 @@ conf.clear conf.update( "type" => "blah", - "path" => "#{tmpdir_path}/*.log", + "path" => path_path, "sincedb_path" => sincedb_path, "stat_interval" => 0.1, "max_open_files" => 1, "start_position" => "beginning", + "file_sort_by" => "path", "delimiter" => TEST_FILE_DELIMITER) subject.register end it "collects line events from only one file" do actions = RSpec::Sequencing - .run_after(0.2, "assert one identity is mapped") do - expect(subject.codec.identity_count).to eq(1) + .run("assert one identity is mapped") do + wait(0.4).for{subject.codec.identity_count}.to be > 0, "no identity is mapped" end - .then_after(0.1, "stop") do + .then("stop") do subject.stop end - .then_after(0.1, "stop flushes last event") do - expect(events.size).to eq(2) - e1, e2 = events - if Dir.glob("#{tmpdir_path}/*.log").first =~ %r{a\.log} - #linux and OSX have different retrieval order - expect(e1.get("message")).to eq("line1-of-a") - expect(e2.get("message")).to eq("line2-of-a") - else - expect(e1.get("message")).to eq("line1-of-z") - expect(e2.get("message")).to eq("line2-of-z") - end + .then("stop flushes last event") do + wait(0.4).for{events.size}.to eq(4), "events size does not equal 4" end subject.run(events) # wait for actions future value - actions.value + actions.assert_no_errors + e1, e2, e3, e4 = events + expect(e1.get("message")).to eq("line1-of-P") + expect(e2.get("message")).to eq("line2-of-P") + expect(e3.get("message")).to eq("line1-of-Q") + expect(e4.get("message")).to eq("line2-of-Q") end end @@ -499,41 +466,36 @@ before do conf.update( "type" => "blah", - "path" => "#{tmpdir_path}/*.log", + "path" => path_path, "sincedb_path" => sincedb_path, "stat_interval" => 0.1, "max_open_files" => 1, "close_older" => 0.5, "start_position" => "beginning", + "file_sort_by" => "path", "delimiter" => TEST_FILE_DELIMITER) subject.register end it "collects line events from both files" do actions = RSpec::Sequencing - .run_after(0.2, "assert both identities are mapped and the first two events are built") do - expect(subject.codec.identity_count).to eq(2) - expect(events.size).to eq(2) + .run("assert both identities are mapped and the first two events are built") do + wait(0.2).for{subject.codec.identity_count == 2 && events.size > 1}.to eq(true), "both identities are not mapped and the first two events are not built" end - .then_after(0.8, "wait for close to flush last event of each identity") do - expect(events.size).to eq(4) - if Dir.glob("#{tmpdir_path}/*.log").first =~ %r{a\.log} - #linux and OSX have different retrieval order - e1, e2, e3, e4 = events - else - e3, e4, e1, e2 = events - end - expect(e1.get("message")).to eq("line1-of-a") - expect(e2.get("message")).to eq("line2-of-a") - expect(e3.get("message")).to eq("line1-of-z") - expect(e4.get("message")).to eq("line2-of-z") + .then("wait for close to flush last event of each identity") do + wait(0.8).for{events.size}.to eq(4), "close does not flush last event of each identity" end .then_after(0.1, "stop") do subject.stop end subject.run(events) # wait for actions future value - actions.value + actions.assert_no_errors + e1, e2, e3, e4 = events + expect(e1.get("message")).to eq("line1-of-P") + expect(e2.get("message")).to eq("line2-of-P") + expect(e3.get("message")).to eq("line1-of-Q") + expect(e4.get("message")).to eq("line2-of-Q") end end end diff --git a/src/main/java/jnr/posix/windows/WindowsFileInformationByHandle.java b/src/main/java/jnr/posix/windows/WindowsFileInformationByHandle.java new file mode 100644 index 0000000..7e36502 --- /dev/null +++ b/src/main/java/jnr/posix/windows/WindowsFileInformationByHandle.java @@ -0,0 +1,24 @@ +package jnr.posix.windows; +/* +This, sadly can't be used. +See JrubyFileWatchLibrary class +The jnr jar is loaded by a different class loader than our jar (in rspec anyway) +Even though the package is the same, Java restricts access to `dwVolumeSerialNumber` in the super class +We have to continue to use FFI in Ruby. +*/ + +public class WindowsFileInformationByHandle extends WindowsByHandleFileInformation { + public WindowsFileInformationByHandle(jnr.ffi.Runtime runtime) { + super(runtime); + } + + public java.lang.String getIdentifier() { + StringBuilder builder = new StringBuilder(); + builder.append(dwVolumeSerialNumber.intValue()); + builder.append("-"); + builder.append(nFileIndexHigh.intValue()); + builder.append("-"); + builder.append(nFileIndexLow.intValue()); + return builder.toString(); + } +} diff --git a/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java b/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java index 7a859a1..e37d186 100644 --- a/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java +++ b/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java @@ -12,28 +12,40 @@ * fnv code extracted and modified from https://github.com/jakedouglas/fnv-java */ +import jnr.ffi.Runtime; +import jnr.posix.HANDLE; +import jnr.posix.JavaLibCHelper; +import jnr.posix.POSIX; +import jnr.posix.WindowsLibC; +import jnr.posix.WindowsPOSIX; +import jnr.posix.util.WindowsHelpers; +import jnr.posix.windows.WindowsFileInformationByHandle; import org.jruby.Ruby; import org.jruby.RubyBignum; import org.jruby.RubyClass; import org.jruby.RubyFixnum; import org.jruby.RubyIO; -import org.jruby.RubyInteger; import org.jruby.RubyModule; +import org.jruby.RubyNumeric; import org.jruby.RubyObject; import org.jruby.RubyString; import org.jruby.anno.JRubyClass; import org.jruby.anno.JRubyMethod; +import org.jruby.ext.ffi.Factory; +import org.jruby.ext.ffi.MemoryIO; +import org.jruby.ext.ffi.Pointer; import org.jruby.runtime.Arity; +import org.jruby.runtime.Block; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; import org.jruby.runtime.load.Library; +import org.jruby.util.io.OpenFile; import java.io.IOException; import java.math.BigInteger; import java.nio.channels.Channel; import java.nio.channels.FileChannel; import java.nio.file.FileSystems; -import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -47,6 +59,21 @@ public class JrubyFileWatchLibrary implements Library { private static final BigInteger MOD32 = new BigInteger("2").pow(32); private static final BigInteger MOD64 = new BigInteger("2").pow(64); + // private static final int GENERIC_ALL = 268435456; + private static final int GENERIC_READ = -2147483648; + // private static final int GENERIC_WRITE = 1073741824; + // private static final int GENERIC_EXECUTE = 33554432; + // private static final int FILE_SHARE_DELETE = 4; + private static final int FILE_SHARE_READ = 1; + private static final int FILE_SHARE_WRITE = 2; + // private static final int CREATE_ALWAYS = 2; + // private static final int CREATE_NEW = 1; + // private static final int OPEN_ALWAYS = 4; + private static final int OPEN_EXISTING = 3; + // private static final int TRUNCATE_EXISTING = 5; + private static final int FILE_FLAG_BACKUP_SEMANTICS = 33554432; + // private static final int FILE_ATTRIBUTE_READONLY = 1; + @Override public final void load(final Ruby runtime, final boolean wrap) { final RubyModule module = runtime.defineModule("FileWatch"); @@ -59,30 +86,119 @@ public final void load(final Ruby runtime, final boolean wrap) { } - @JRubyClass(name = "FileExt", parent = "Object") + @JRubyClass(name = "FileExt") public static class RubyFileExt extends RubyObject { - public RubyFileExt(final Ruby runtime, final RubyClass metaClass) { - super(runtime, metaClass); + public RubyFileExt(final Ruby runtime, final RubyClass meta) { + super(runtime, meta); } - public RubyFileExt(final RubyClass metaClass) { - super(metaClass); + public RubyFileExt(final RubyClass meta) { + super(meta); } @JRubyMethod(name = "open", required = 1, meta = true) - public static RubyIO open(final ThreadContext context, final IRubyObject self, final RubyString path) throws IOException { + public static IRubyObject open(final ThreadContext context, final IRubyObject self, final RubyString path) throws IOException { final Path javapath = FileSystems.getDefault().getPath(path.asJavaString()); - final OpenOption[] options = new OpenOption[1]; - options[0] = StandardOpenOption.READ; - final Channel channel = FileChannel.open(javapath, options); - return new RubyIO(Ruby.getGlobalRuntime(), channel); + final Channel channel = FileChannel.open(javapath, StandardOpenOption.READ); + final RubyIO irubyobject = new RubyWinIO(context.runtime, channel); + return irubyobject; + } + + @JRubyMethod(name = "io_handle", required = 1, meta = true) + public static IRubyObject ioHandle(final ThreadContext context, final IRubyObject self, final IRubyObject object, Block block) { + final Ruby runtime = context.runtime; + if (!block.isGiven()) { + throw runtime.newArgumentError(0, 1); + } + if (object instanceof RubyWinIO) { + final RubyWinIO rubyWinIO = (RubyWinIO) object; + final OpenFile fptr = rubyWinIO.getOpenFileChecked(); + final boolean locked = fptr.lock(); + try { + fptr.checkClosed(); + if (rubyWinIO.isDirect()) { + final MemoryIO memoryio = Factory.getInstance().wrapDirectMemory(runtime, rubyWinIO.getAddress()); + final Pointer pointer = new Pointer(runtime, memoryio); + return block.yield(context, pointer); + } + } finally { + if (locked) { + fptr.unlock(); + } + } + } else { + System.out.println("Required argument is not a WinIO instance"); + } + return runtime.newString(); + } + + //@JRubyMethod(name = "io_inode", required = 1, meta = true) + public static RubyString ioInode(final ThreadContext context, final IRubyObject self, final IRubyObject object) { + final Ruby runtime = context.runtime; + if (!(object instanceof RubyIO)) { + System.out.println("Required argument is not an IO instance"); + return runtime.newString(); + } + final RubyIO rubyIO = (RubyIO) object; + final OpenFile fptr = rubyIO.getOpenFileChecked(); + final boolean locked = fptr.lock(); + String inode = ""; + try { + fptr.checkClosed(); + final POSIX posix = runtime.getPosix(); + final int realFileno = fptr.fd().realFileno; + if (posix.isNative() && posix instanceof WindowsPOSIX && realFileno != -1) { + final WindowsPOSIX winposix = (WindowsPOSIX) posix; + final WindowsLibC wlibc = (WindowsLibC) winposix.libc(); + final WindowsFileInformationByHandle info = new WindowsFileInformationByHandle(Runtime.getRuntime(runtime.getPosix().libc())); + final HANDLE handle = JavaLibCHelper.gethandle(JavaLibCHelper.getDescriptorFromChannel(fptr.fd().chFile)); + if (handle.isValid()) { + if (wlibc.GetFileInformationByHandle(handle, info) > 0) { + inode = info.getIdentifier(); + } else { + System.out.println("Could not 'GetFileInformationByHandle' from handle"); + } + } else { + System.out.println("Could not derive 'HANDLE' from Ruby IO instance via io.getOpenFileChecked().fd().chFile"); + } + } + } finally { + if (locked) { + fptr.unlock(); + } + } + return runtime.newString(inode); + } + + //@JRubyMethod(name = "path_inode", required = 1, meta = true) + public static RubyString pathInode(final ThreadContext context, final IRubyObject self, final RubyString path) { + final Ruby runtime = context.runtime; + final POSIX posix = runtime.getPosix(); + String inode = ""; + if (posix.isNative() && posix.libc() instanceof WindowsLibC) { + final WindowsLibC wlibc = (WindowsLibC) posix.libc(); + final byte[] wpath = WindowsHelpers.toWPath(path.toString()); + final HANDLE handle = wlibc.CreateFileW(wpath, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, null, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, 0); + if (handle.isValid()) { + final WindowsFileInformationByHandle info = new WindowsFileInformationByHandle(Runtime.getRuntime(runtime.getPosix().libc())); + if (wlibc.GetFileInformationByHandle(handle, info) > 0) { + inode = info.getIdentifier(); + } else { + System.out.println("Could not 'GetFileInformationByHandle' from handle"); + } + wlibc.CloseHandle(handle); + } else { + System.out.printf("Could not open file via 'CreateFileW' on path: %s", path.toString()); + } + } + return runtime.newString(inode); } } // This class may be used by fingerprinting in the future @SuppressWarnings({"NewMethodNamingConvention", "ChainOfInstanceofChecks"}) - @JRubyClass(name = "Fnv", parent = "Object") + @JRubyClass(name = "Fnv") public static class Fnv extends RubyObject { private byte[] bytes; @@ -98,12 +214,12 @@ public Fnv(final RubyClass metaClass) { } @JRubyMethod(name = "coerce_bignum", meta = true, required = 1) - public static IRubyObject coerceBignum(final ThreadContext ctx, final IRubyObject recv, final IRubyObject i) { - if (i instanceof RubyBignum) { - return i; + public static IRubyObject coerceBignum(final ThreadContext ctx, final IRubyObject recv, final IRubyObject rubyObject) { + if (rubyObject instanceof RubyBignum) { + return rubyObject; } - if (i instanceof RubyFixnum) { - return RubyBignum.newBignum(ctx.runtime, ((RubyFixnum)i).getBigIntegerValue()); + if (rubyObject instanceof RubyFixnum) { + return RubyBignum.newBignum(ctx.runtime, ((RubyNumeric) rubyObject).getBigIntegerValue()); } throw ctx.runtime.newRaiseException(ctx.runtime.getClass("StandardError"), "Can't coerce"); } @@ -141,7 +257,7 @@ public IRubyObject closed_p(final ThreadContext ctx) { } @JRubyMethod(name = "fnv1a32", optional = 1) - public IRubyObject fnv1a_32(final ThreadContext ctx, IRubyObject[] args) { + public IRubyObject fnv1a_32(final ThreadContext ctx, final IRubyObject[] args) { IRubyObject[] args1 = args; if(open) { args1 = Arity.scanArgs(ctx.runtime, args1, 0, 1); @@ -151,7 +267,7 @@ public IRubyObject fnv1a_32(final ThreadContext ctx, IRubyObject[] args) { } @JRubyMethod(name = "fnv1a64", optional = 1) - public IRubyObject fnv1a_64(final ThreadContext ctx, IRubyObject[] args) { + public IRubyObject fnv1a_64(final ThreadContext ctx, final IRubyObject[] args) { IRubyObject[] args1 = args; if(open) { args1 = Arity.scanArgs(ctx.runtime, args1, 0, 1); @@ -161,13 +277,9 @@ public IRubyObject fnv1a_64(final ThreadContext ctx, IRubyObject[] args) { } private long convertLong(final IRubyObject obj) { - if(obj instanceof RubyInteger) { - return ((RubyInteger)obj).getLongValue(); + if(obj instanceof RubyNumeric) { + return ((RubyNumeric) obj).getLongValue(); } - if(obj instanceof RubyFixnum) { - return ((RubyFixnum)obj).getLongValue(); - } - return size; } diff --git a/src/main/java/org/logstash/filewatch/RubyWinIO.java b/src/main/java/org/logstash/filewatch/RubyWinIO.java new file mode 100644 index 0000000..c6cfbe4 --- /dev/null +++ b/src/main/java/org/logstash/filewatch/RubyWinIO.java @@ -0,0 +1,65 @@ +package org.logstash.filewatch; + +import jnr.posix.HANDLE; +import jnr.posix.JavaLibCHelper; +import org.jruby.Ruby; +import org.jruby.RubyBoolean; +import org.jruby.RubyIO; +import org.jruby.anno.JRubyClass; +import org.jruby.anno.JRubyMethod; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.util.io.OpenFile; + +import java.nio.channels.Channel; + +@JRubyClass(name = "WinIO") +public class RubyWinIO extends RubyIO { + private boolean valid; + private boolean direct; + private long address; + + public RubyWinIO(Ruby runtime, Channel channel) { + super(runtime, channel); + final OpenFile fptr = getOpenFileChecked(); + final boolean locked = fptr.lock(); + try { + fptr.checkClosed(); + final HANDLE handle = JavaLibCHelper.gethandle(JavaLibCHelper.getDescriptorFromChannel(fptr.fd().chFile)); + if (handle.isValid()) { + direct = handle.toPointer().isDirect(); + address = handle.toPointer().address(); + valid = true; + } else { + direct = false; + address = 0L; + valid = false; + } + } finally { + if (locked) { + fptr.unlock(); + } + } + } + + @JRubyMethod(name = "valid?") + public RubyBoolean valid_p(ThreadContext context) { + return context.runtime.newBoolean(valid); + } + + @Override + @JRubyMethod + public IRubyObject close() { + direct = false; + address = 0L; + return super.close(); + } + + final public boolean isDirect() { + return direct; + } + + final public long getAddress() { + return address; + } +} From 92d86729c608430550f3d772e7ce9e137b8f5e4a Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Wed, 11 Jul 2018 12:40:28 +0100 Subject: [PATCH 35/91] WIP: Better support for NFS, step 1 (#199) * switch to loop controlled reading * fix test Fixes #189 --- lib/filewatch/bootstrap.rb | 1 + lib/filewatch/read_mode/handlers/read_file.rb | 77 +++++++++++-------- lib/filewatch/tail_mode/handlers/base.rb | 8 +- lib/filewatch/tail_mode/handlers/grow.rb | 6 +- lib/filewatch/tail_mode/handlers/shrink.rb | 9 ++- lib/filewatch/tail_mode/processor.rb | 4 +- lib/filewatch/watched_file.rb | 50 ++++++++---- .../read_mode_handlers_read_file_spec.rb | 2 +- 8 files changed, 99 insertions(+), 58 deletions(-) diff --git a/lib/filewatch/bootstrap.rb b/lib/filewatch/bootstrap.rb index 51e9b37..d756d32 100644 --- a/lib/filewatch/bootstrap.rb +++ b/lib/filewatch/bootstrap.rb @@ -42,6 +42,7 @@ def to_s end BufferExtractResult = Struct.new(:lines, :warning, :additional) + LoopControlResult = Struct.new(:count, :size, :more) class NoSinceDBPathGiven < StandardError; end diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index 9219792..1f73692 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -5,42 +5,51 @@ class ReadFile < Base def handle_specifically(watched_file) if open_file(watched_file) add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) - changed = false - logger.trace("reading...", "amount" => watched_file.read_bytesize_description, "filename" => watched_file.filename) - watched_file.read_loop_count.times do + loop do break if quit? - begin - # expect BufferExtractResult - result = watched_file.read_extract_lines - # read_extract_lines will increment bytes_read - logger.trace(result.warning, result.additional) unless result.warning.empty? - changed = true - result.lines.each do |line| - watched_file.listener.accept(line) - # sincedb position is independent from the watched_file bytes_read - sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size) - end - sincedb_collection.request_disk_flush - rescue EOFError - # flush the buffer now in case there is no final delimiter - line = watched_file.buffer.flush - watched_file.listener.accept(line) unless line.empty? - watched_file.listener.eof - watched_file.file_close - key = watched_file.sincedb_key - sincedb_collection.reading_completed(key) - sincedb_collection.clear_watched_file(key) - watched_file.listener.deleted - watched_file.unwatch - break - rescue Errno::EWOULDBLOCK, Errno::EINTR - watched_file.listener.error - break - rescue => e - logger.error("read_to_eof: general error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) - watched_file.listener.error - break + loop_control = watched_file.loop_control_adjusted_for_stat_size + controlled_read(watched_file, loop_control) + sincedb_collection.request_disk_flush + break unless loop_control.more + end + if watched_file.all_read? + # flush the buffer now in case there is no final delimiter + line = watched_file.buffer.flush + watched_file.listener.accept(line) unless line.empty? + watched_file.listener.eof + watched_file.file_close + key = watched_file.sincedb_key + sincedb_collection.reading_completed(key) + sincedb_collection.clear_watched_file(key) + watched_file.listener.deleted + watched_file.unwatch + end + end + end + + def controlled_read(watched_file, loop_control) + logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename) + loop_control.count.times do + begin + result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult + logger.info(result.warning, result.additional) unless result.warning.empty? + result.lines.each do |line| + watched_file.listener.accept(line) + # sincedb position is independent from the watched_file bytes_read + delta = line.bytesize + @settings.delimiter_byte_size + sincedb_collection.increment(watched_file.sincedb_key, delta) end + rescue EOFError + logger.error("controlled_read: eof error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) + break + rescue Errno::EWOULDBLOCK, Errno::EINTR + logger.error("controlled_read: block or interrupt error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) + watched_file.listener.error + break + rescue => e + logger.error("controlled_read: general error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) + watched_file.listener.error + break end end end diff --git a/lib/filewatch/tail_mode/handlers/base.rb b/lib/filewatch/tail_mode/handlers/base.rb index 783eb4e..a8ee33a 100644 --- a/lib/filewatch/tail_mode/handlers/base.rb +++ b/lib/filewatch/tail_mode/handlers/base.rb @@ -30,9 +30,9 @@ def update_existing_specifically(watched_file, sincedb_value) private - def read_to_eof(watched_file) - logger.trace("reading...", "amount" => watched_file.read_bytesize_description, "filename" => watched_file.filename) + def controlled_read(watched_file, loop_control) changed = false + logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename) # from a real config (has 102 file inputs) # -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously # -- If we put all log files in one file input glob we will have indexing delay, because Logstash waits until the first file becomes EOF @@ -40,9 +40,9 @@ def read_to_eof(watched_file) # we enable the pseudo parallel processing of each file. # user also has the option to specify a low `stat_interval` and a very high `discover_interval`to respond # quicker to changing files and not allowing too much content to build up before reading it. - watched_file.read_loop_count.times do + loop_control.count.times do begin - result = watched_file.read_extract_lines # expect BufferExtractResult + result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult logger.trace(result.warning, result.additional) unless result.warning.empty? changed = true result.lines.each do |line| diff --git a/lib/filewatch/tail_mode/handlers/grow.rb b/lib/filewatch/tail_mode/handlers/grow.rb index 4c93fe9..7d967e9 100644 --- a/lib/filewatch/tail_mode/handlers/grow.rb +++ b/lib/filewatch/tail_mode/handlers/grow.rb @@ -4,7 +4,11 @@ module FileWatch module TailMode module Handlers class Grow < Base def handle_specifically(watched_file) watched_file.file_seek(watched_file.bytes_read) - read_to_eof(watched_file) + loop do + loop_control = watched_file.loop_control_adjusted_for_stat_size + controlled_read(watched_file, loop_control) + break unless loop_control.more + end end end end end end diff --git a/lib/filewatch/tail_mode/handlers/shrink.rb b/lib/filewatch/tail_mode/handlers/shrink.rb index 54fc3ed..659dd47 100644 --- a/lib/filewatch/tail_mode/handlers/shrink.rb +++ b/lib/filewatch/tail_mode/handlers/shrink.rb @@ -3,10 +3,13 @@ module FileWatch module TailMode module Handlers class Shrink < Base def handle_specifically(watched_file) - sdbv = add_or_update_sincedb_collection(watched_file) + add_or_update_sincedb_collection(watched_file) watched_file.file_seek(watched_file.bytes_read) - read_to_eof(watched_file) - logger.trace("handle_specifically: after read_to_eof", "watched file" => watched_file.details, "sincedb value" => sdbv) + loop do + loop_control = watched_file.loop_control_adjusted_for_stat_size + controlled_read(watched_file, loop_control) + break unless loop_control.more + end end def update_existing_specifically(watched_file, sincedb_value) diff --git a/lib/filewatch/tail_mode/processor.rb b/lib/filewatch/tail_mode/processor.rb index ed990c7..cad1208 100644 --- a/lib/filewatch/tail_mode/processor.rb +++ b/lib/filewatch/tail_mode/processor.rb @@ -155,9 +155,9 @@ def process_rotation_in_progress(watched_files) # we need to keep reading the open file, if we close it we lose it because the path is now pointing at a different file. logger.trace(">>> Rotation In Progress - inode change detected and original content is not fully read, reading all", "watched_file details" => watched_file.details) # need to fully read open file while we can - watched_file.set_depth_first_read_loop + watched_file.set_maximum_read_loop grow(watched_file) - watched_file.set_user_defined_read_loop + watched_file.set_standard_read_loop else logger.warn(">>> Rotation In Progress - inode change detected and original content is not fully read, file is closed and path points to new content", "watched_file details" => watched_file.details) end diff --git a/lib/filewatch/watched_file.rb b/lib/filewatch/watched_file.rb index 78873a8..77fd4e8 100644 --- a/lib/filewatch/watched_file.rb +++ b/lib/filewatch/watched_file.rb @@ -7,7 +7,8 @@ class WatchedFile attr_reader :bytes_read, :state, :file, :buffer, :recent_states, :bytes_unread attr_reader :path, :accessed_at, :modified_at, :pathname, :filename - attr_reader :listener, :read_loop_count, :read_chunk_size, :stat, :read_bytesize_description + attr_reader :listener, :read_loop_count, :read_chunk_size, :stat + attr_reader :loop_count_type, :loop_count_mode attr_accessor :last_open_warning_at # this class represents a file that has been discovered @@ -19,7 +20,7 @@ def initialize(pathname, stat, settings) @filename = @pathname.basename.to_s full_state_reset(stat) watch - set_user_defined_read_loop + set_standard_read_loop set_accessed_at end @@ -52,7 +53,7 @@ def full_state_reset(this_stat = nil) def rotate_from(other) # move all state from other to this one - set_user_defined_read_loop + set_standard_read_loop file_close @bytes_read = other.bytes_read @bytes_unread = other.bytes_unread @@ -231,8 +232,8 @@ def reset_buffer @buffer.flush end - def read_extract_lines - data = file_read + def read_extract_lines(amount) + data = file_read(amount) result = buffer_extract(data) increment_bytes_read(data.bytesize) result @@ -342,16 +343,39 @@ def expiry_ignore_enabled? !@settings.ignore_older.nil? end - def set_depth_first_read_loop - @read_loop_count = FileWatch::MAX_ITERATIONS - @read_chunk_size = FileWatch::FILE_READ_SIZE - @read_bytesize_description = "All" - end - - def set_user_defined_read_loop + def set_standard_read_loop @read_loop_count = @settings.file_chunk_count @read_chunk_size = @settings.file_chunk_size - @read_bytesize_description = @read_loop_count == FileWatch::MAX_ITERATIONS ? "All" : (@read_loop_count * @read_chunk_size).to_s + # e.g. 1 * 10 bytes -> 10 or 256 * 65536 -> 1677716 or 140737488355327 * 32768 -> 4611686018427355136 + @standard_loop_max_bytes = @read_loop_count * @read_chunk_size + end + + def set_maximum_read_loop + # used to quickly fully read an open file when rotation is detected + @read_loop_count = FileWatch::MAX_ITERATIONS + @read_chunk_size = FileWatch::FILE_READ_SIZE + @standard_loop_max_bytes = @read_loop_count * @read_chunk_size + end + + def loop_control_adjusted_for_stat_size + more = false + to_read = current_size - @bytes_read + return LoopControlResult.new(0, 0, more) if to_read < 1 + return LoopControlResult.new(1, to_read, more) if to_read < @read_chunk_size + # set as if to_read is greater than or equal to max_bytes + # use the ones from settings and don't indicate more + count = @read_loop_count + if to_read < @standard_loop_max_bytes + # if the defaults are used then this branch will be taken + # e.g. to_read is 100 and max_bytes is 4 * 30 -> 120 + # will overrun and trigger EOF, build less iterations + # will generate 3 * 30 -> 90 this time and we indicate more + # a 2GB file in read mode will get one loop of 64666 x 32768 (2119006656 / 32768) + # and a second loop with 1 x 31168 + count = to_read / @read_chunk_size + more = true + end + LoopControlResult.new(count, @read_chunk_size, more) end def reset_bytes_unread diff --git a/spec/filewatch/read_mode_handlers_read_file_spec.rb b/spec/filewatch/read_mode_handlers_read_file_spec.rb index ab36e71..f1a6ff5 100644 --- a/spec/filewatch/read_mode_handlers_read_file_spec.rb +++ b/spec/filewatch/read_mode_handlers_read_file_spec.rb @@ -20,7 +20,7 @@ module FileWatch let(:watch) { double("watch", :quit? => false) } it "calls 'sincedb_write' exactly 2 times" do allow(FileOpener).to receive(:open).with(watched_file.path).and_return(file) - expect(sdb_collection).to receive(:sincedb_write).exactly(2).times + expect(sdb_collection).to receive(:sincedb_write).exactly(1).times watched_file.activate processor.initialize_handlers(sdb_collection, TestObserver.new) processor.read_file(watched_file) From 7a62d01338194db9b614d6cba6b24664f89197fe Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Thu, 12 Jul 2018 17:58:01 +0100 Subject: [PATCH 36/91] Update docs and changelog, make ready for release of 4.1.4. (#201) * update docs and changelog, make ready for release. * Use max_open_files instead of max_active when transferring plugin config to the Settings. * give this test more time on travis --- CHANGELOG.md | 10 ++++++++ docs/index.asciidoc | 42 +++++++++++++++++++++------------- lib/filewatch/settings.rb | 4 ++-- spec/filewatch/rotate_spec.rb | 4 ++-- spec/filewatch/tailing_spec.rb | 6 ++--- spec/inputs/file_tail_spec.rb | 8 +++---- 6 files changed, 46 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cc7611..6ff46d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## 4.1.4 + - Fixed a regression where files discovered after first discovery were not + always read from the beginning. Applies to tail mode only. + [#198](https://github.com/logstash-plugins/logstash-input-file/issues/198) + - Added much better support for file rotation schemes of copy/truncate and + rename cascading. Applies to tail mode only. + - Added support for processing files over remote mounts e.g. NFS. Before, it + was possible to read into memory allocated but not filled with data resulting + in ASCII NUL (0) bytes in the message field. Now, files are read up to the + size as given by the remote filesystem client. Applies to tail and read modes. ## 4.1.3 - Fixed `read` mode of regular files sincedb write is requested in each read loop iteration rather than waiting for the end-of-file to be reached. Note: for gz files, diff --git a/docs/index.asciidoc b/docs/index.asciidoc index aab3dab..cfb40a3 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -78,12 +78,6 @@ Read mode also allows for an action to take place after processing the file comp In the past attempts to simulate a Read mode while still assuming infinite streams was not ideal and a dedicated Read mode is an improvement. -==== Reading from remote network volumes - -The file input is not tested on remote filesystems such as NFS, Samba, s3fs-fuse, etc. These -remote filesystems typically have behaviors that are very different from local filesystems and -are therefore unlikely to work correctly when used with the file input. - ==== Tracking of current position in watched files The plugin keeps track of the current position in each file by @@ -103,6 +97,10 @@ A different `sincedb_path` must be used for each input. Using the same path will cause issues. The read checkpoints for each input must be stored in a different path so the information does not override. +Files are tracked via an identifier. This identifier is made up of the +inode, major device number and minor device number. In windows, a different +identifier is taken from a `kernel32` API call. + Sincedb records can now be expired meaning that read positions of older files will not be remembered after a certain time period. File systems may need to reuse inodes for new content. Ideally, we would not use the read position of old content, @@ -123,6 +121,19 @@ old sincedb records converted to the new format, this is blank. On non-Windows systems you can obtain the inode number of a file with e.g. `ls -li`. +==== Reading from remote network volumes + +The file input is not thoroughly tested on remote filesystems such as NFS, +Samba, s3fs-fuse, etc, however NFS is occasionally tested. The file size as given by +the remote FS client is used to govern how much data to read at any given time to +prevent reading into allocated but yet unfilled memory. +As we use the device major and minor in the identifier to track "last read" +positions of files and on remount the device major and minor can change, the +sincedb records may not match across remounts. +Read mode might not be suitable for remote filesystems as the file size at +discovery on the client side may not be the same as the file size on the remote side +due to latency in the remote to client copy process. + ==== File rotation in Tail mode File rotation is detected and handled by this input, regardless of @@ -130,16 +141,15 @@ whether the file is rotated via a rename or a copy operation. To support programs that write to the rotated file for some time after the rotation has taken place, include both the original filename and the rotated filename (e.g. /var/log/syslog and /var/log/syslog.1) in -the filename patterns to watch (the `path` option). Note that the -rotated filename will be treated as a new file so if -`start_position` is set to 'beginning' the rotated file will be -reprocessed. - -With the default value of `start_position` ('end') any messages -written to the end of the file between the last read operation prior -to the rotation and its reopening under the new name (an interval -determined by the `stat_interval` and `discover_interval` options) -will not get picked up. +the filename patterns to watch (the `path` option). +For a rename, the inode will be detected as having moved from +`/var/log/syslog` to `/var/log/syslog.1` and so the "state" is moved +internally too, the old content will not be reread but any new content +on the renamed file will be read. +For copy/truncate the copied content into a new file path, if discovered, will +be treated as a new discovery and be read from the beginning. The copied file +paths should therefore not be in the filename patterns to watch (the `path` option). +The truncation will be detected and the "last read" position updated to zero. [id="plugins-{type}s-{plugin}-options"] ==== File Input Configuration Options diff --git a/lib/filewatch/settings.rb b/lib/filewatch/settings.rb index 1ec08e9..e970435 100644 --- a/lib/filewatch/settings.rb +++ b/lib/filewatch/settings.rb @@ -21,7 +21,7 @@ def initialize defaults = { :delimiter => "\n", :file_chunk_size => FILE_READ_SIZE, - :max_active => 4095, + :max_open_files => 4095, :file_chunk_count => MAX_ITERATIONS, :sincedb_clean_after => 14, :exclude => [], @@ -37,7 +37,7 @@ def initialize def add_options(opts) @opts.update(opts) - self.max_open_files = @opts[:max_active] + self.max_open_files = @opts[:max_open_files] @delimiter = @opts[:delimiter] @delimiter_byte_size = @delimiter.bytesize @file_chunk_size = @opts[:file_chunk_size] diff --git a/spec/filewatch/rotate_spec.rb b/spec/filewatch/rotate_spec.rb index e1d2b4e..66d9ae4 100644 --- a/spec/filewatch/rotate_spec.rb +++ b/spec/filewatch/rotate_spec.rb @@ -32,7 +32,7 @@ module FileWatch let(:sincedb_path) { directory.join("tailing.sdb") } let(:opts) do { - :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_active => max, + :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_open_files => max, :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path.to_path } end @@ -410,7 +410,7 @@ module FileWatch context "? rotation: when a not active file is rotated outside the glob before the file is read" do let(:opts) { super.merge( :close_older => 3600, - :max_active => 1, + :max_open_files => 1, :file_sort_by => "path" ) } let(:watch_dir) { directory.join("*J.log") } diff --git a/spec/filewatch/tailing_spec.rb b/spec/filewatch/tailing_spec.rb index 30998ea..5b71ca4 100644 --- a/spec/filewatch/tailing_spec.rb +++ b/spec/filewatch/tailing_spec.rb @@ -17,7 +17,7 @@ module FileWatch let(:sincedb_path) { ::File.join(directory, "tailing.sdb") } let(:opts) do { - :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_active => max, + :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_open_files => max, :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path, :file_sort_by => "path" } @@ -74,7 +74,7 @@ module FileWatch context "when close_older is set" do let(:wait_before_quit) { 0.8 } - let(:opts) { super.merge(:close_older => 0.1, :max_active => 1, :stat_interval => 0.1) } + let(:opts) { super.merge(:close_older => 0.1, :max_open_files => 1, :stat_interval => 0.1) } let(:suffix) { "B" } it "opens both files" do actions.activate_quietly @@ -100,7 +100,7 @@ module FileWatch .then_after(0.1, "begin watching") do tailing.watch_this(watch_dir) end - .then_after(0.05, "add content") do + .then_after(0.2, "add content") do File.open(file_path, "ab") { |file| file.write("line1\nline2\n") } end .then("wait") do diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index 1250eba..7cca515 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -449,16 +449,14 @@ subject.stop end .then("stop flushes last event") do - wait(0.4).for{events.size}.to eq(4), "events size does not equal 4" + wait(0.4).for{events.size}.to eq(2), "events size does not equal 2" end subject.run(events) # wait for actions future value actions.assert_no_errors - e1, e2, e3, e4 = events + e1, e2 = events expect(e1.get("message")).to eq("line1-of-P") expect(e2.get("message")).to eq("line2-of-P") - expect(e3.get("message")).to eq("line1-of-Q") - expect(e4.get("message")).to eq("line2-of-Q") end end @@ -480,7 +478,7 @@ it "collects line events from both files" do actions = RSpec::Sequencing .run("assert both identities are mapped and the first two events are built") do - wait(0.2).for{subject.codec.identity_count == 2 && events.size > 1}.to eq(true), "both identities are not mapped and the first two events are not built" + wait(0.4).for{subject.codec.identity_count == 1 && events.size == 2}.to eq(true), "both identities are not mapped and the first two events are not built" end .then("wait for close to flush last event of each identity") do wait(0.8).for{events.size}.to eq(4), "close does not flush last event of each identity" From 5aa6e8c8e23134fb95ab935b8eb6d6f1be8b67a4 Mon Sep 17 00:00:00 2001 From: thbay <41009724+thbay@users.noreply.github.com> Date: Mon, 16 Jul 2018 11:54:39 +0200 Subject: [PATCH 37/91] Update for faster shutdown in tail mode (#200) * Make tail mode shutdown input faster When input is shutdown tail aborts its read-for-eof loop faster. Construct is almost the same as in read mode. --- lib/filewatch/read_mode/handlers/read_file.rb | 1 + lib/filewatch/tail_mode/handlers/base.rb | 10 +++++++++- lib/filewatch/tail_mode/handlers/grow.rb | 1 + lib/filewatch/tail_mode/handlers/shrink.rb | 1 + lib/filewatch/tail_mode/processor.rb | 14 +++++++------- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index 1f73692..2a01cb5 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -30,6 +30,7 @@ def handle_specifically(watched_file) def controlled_read(watched_file, loop_control) logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename) loop_control.count.times do + break if quit? begin result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult logger.info(result.warning, result.additional) unless result.warning.empty? diff --git a/lib/filewatch/tail_mode/handlers/base.rb b/lib/filewatch/tail_mode/handlers/base.rb index a8ee33a..2da9495 100644 --- a/lib/filewatch/tail_mode/handlers/base.rb +++ b/lib/filewatch/tail_mode/handlers/base.rb @@ -6,12 +6,17 @@ class Base include LogStash::Util::Loggable attr_reader :sincedb_collection - def initialize(sincedb_collection, observer, settings) + def initialize(processor, sincedb_collection, observer, settings) @settings = settings + @processor = processor @sincedb_collection = sincedb_collection @observer = observer end + def quit? + @processor.watch.quit? + end + def handle(watched_file) logger.trace("handling: #{watched_file.filename}") unless watched_file.has_listener? @@ -41,7 +46,9 @@ def controlled_read(watched_file, loop_control) # user also has the option to specify a low `stat_interval` and a very high `discover_interval`to respond # quicker to changing files and not allowing too much content to build up before reading it. loop_control.count.times do + break if quit? begin + logger.debug("read_to_eof: get chunk") result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult logger.trace(result.warning, result.additional) unless result.warning.empty? changed = true @@ -62,6 +69,7 @@ def controlled_read(watched_file, loop_control) break end end + logger.debug("read_to_eof: exit due to quit") if quit? sincedb_collection.request_disk_flush if changed end diff --git a/lib/filewatch/tail_mode/handlers/grow.rb b/lib/filewatch/tail_mode/handlers/grow.rb index 7d967e9..755529f 100644 --- a/lib/filewatch/tail_mode/handlers/grow.rb +++ b/lib/filewatch/tail_mode/handlers/grow.rb @@ -5,6 +5,7 @@ class Grow < Base def handle_specifically(watched_file) watched_file.file_seek(watched_file.bytes_read) loop do + break if quit? loop_control = watched_file.loop_control_adjusted_for_stat_size controlled_read(watched_file, loop_control) break unless loop_control.more diff --git a/lib/filewatch/tail_mode/handlers/shrink.rb b/lib/filewatch/tail_mode/handlers/shrink.rb index 659dd47..ac4a00a 100644 --- a/lib/filewatch/tail_mode/handlers/shrink.rb +++ b/lib/filewatch/tail_mode/handlers/shrink.rb @@ -6,6 +6,7 @@ def handle_specifically(watched_file) add_or_update_sincedb_collection(watched_file) watched_file.file_seek(watched_file.bytes_read) loop do + break if quit? loop_control = watched_file.loop_control_adjusted_for_stat_size controlled_read(watched_file, loop_control) break unless loop_control.more diff --git a/lib/filewatch/tail_mode/processor.rb b/lib/filewatch/tail_mode/processor.rb index cad1208..84a6301 100644 --- a/lib/filewatch/tail_mode/processor.rb +++ b/lib/filewatch/tail_mode/processor.rb @@ -35,13 +35,13 @@ def add_watch(watch) def initialize_handlers(sincedb_collection, observer) @sincedb_collection = sincedb_collection - @create_initial = Handlers::CreateInitial.new(sincedb_collection, observer, @settings) - @create = Handlers::Create.new(sincedb_collection, observer, @settings) - @grow = Handlers::Grow.new(sincedb_collection, observer, @settings) - @shrink = Handlers::Shrink.new(sincedb_collection, observer, @settings) - @delete = Handlers::Delete.new(sincedb_collection, observer, @settings) - @timeout = Handlers::Timeout.new(sincedb_collection, observer, @settings) - @unignore = Handlers::Unignore.new(sincedb_collection, observer, @settings) + @create_initial = Handlers::CreateInitial.new(self, sincedb_collection, observer, @settings) + @create = Handlers::Create.new(self, sincedb_collection, observer, @settings) + @grow = Handlers::Grow.new(self, sincedb_collection, observer, @settings) + @shrink = Handlers::Shrink.new(self, sincedb_collection, observer, @settings) + @delete = Handlers::Delete.new(self, sincedb_collection, observer, @settings) + @timeout = Handlers::Timeout.new(self, sincedb_collection, observer, @settings) + @unignore = Handlers::Unignore.new(self, sincedb_collection, observer, @settings) end def create(watched_file) From 06ad65c6918dcb6fc75187ef36d1e05c8f7ecc97 Mon Sep 17 00:00:00 2001 From: Karen Metts Date: Wed, 18 Jul 2018 09:46:51 -0400 Subject: [PATCH 38/91] Fixed string_duration anchor to use asciidoc ref Fixes #203 --- CHANGELOG.md | 5 +++++ docs/index.asciidoc | 26 +++++++++++++------------- logstash-input-file.gemspec | 2 +- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ff46d1..b1134dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 4.1.5 + - Fixed text anchor by changing it from hardcoded to asciidoc reference to + work in versioned plugin reference + ## 4.1.4 - Fixed a regression where files discovered after first discovery were not always read from the beginning. Applies to tail mode only. @@ -8,6 +12,7 @@ was possible to read into memory allocated but not filled with data resulting in ASCII NUL (0) bytes in the message field. Now, files are read up to the size as given by the remote filesystem client. Applies to tail and read modes. + ## 4.1.3 - Fixed `read` mode of regular files sincedb write is requested in each read loop iteration rather than waiting for the end-of-file to be reached. Note: for gz files, diff --git a/docs/index.asciidoc b/docs/index.asciidoc index cfb40a3..0d1a4be 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -159,12 +159,12 @@ This plugin supports the following configuration options plus the <> for the details. +see <> for the details. [cols="<,<,<",options="header",] |======================================================================= |Setting |Input type|Required -| <> |<> or <>|No +| <> |<> or <>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -174,15 +174,15 @@ see <> for the details. | <> |<>|No | <> |<>, one of `["last_modified", "path"]`|No | <> |<>, one of `["asc", "desc"]`|No -| <> |<> or <>|No +| <> |<> or <>|No | <> |<>|No | <> |<>, one of `["tail", "read"]`|No | <> |<>|Yes -| <> |<> or <>|No +| <> |<> or <>|No | <> |<>|No -| <> |<> or <>|No +| <> |<> or <>|No | <> |<>, one of `["beginning", "end"]`|No -| <> |<> or <>|No +| <> |<> or <>|No |======================================================================= Also see <> for a list of options supported by all @@ -193,7 +193,7 @@ input plugins. [id="plugins-{type}s-{plugin}-close_older"] ===== `close_older` - * Value type is <> or <> + * Value type is <> or <> * Default value is `"1 hour"` The file input closes any files that were last read the specified @@ -311,7 +311,7 @@ If you use special naming conventions for the file full paths then perhaps [id="plugins-{type}s-{plugin}-ignore_older"] ===== `ignore_older` - * Value type is <> or <> + * Value type is <> or <> * There is no default value for this setting. When the file input discovers a file that was last modified @@ -371,7 +371,7 @@ on the {logstash-ref}/configuration-file-structure.html#array[Logstash configura [id="plugins-{type}s-{plugin}-sincedb_clean_after"] ===== `sincedb_clean_after` - * Value type is <> or <> + * Value type is <> or <> * The default value for this setting is "2 weeks". * If a number is specified then it is interpreted as *days* and can be decimal e.g. 0.5 is 12 hours. @@ -395,7 +395,7 @@ NOTE: it must be a file path and not a directory path [id="plugins-{type}s-{plugin}-sincedb_write_interval"] ===== `sincedb_write_interval` - * Value type is <> or <> + * Value type is <> or <> * Default value is `"15 seconds"` How often (in seconds) to write a since database with the current position of @@ -421,7 +421,7 @@ position recorded in the sincedb file will be used. [id="plugins-{type}s-{plugin}-stat_interval"] ===== `stat_interval` - * Value type is <> or <> + * Value type is <> or <> * Default value is `"1 second"` How often (in seconds) we stat files to see if they have been modified. @@ -438,9 +438,8 @@ the pipeline is congested. So the overall loop time is a combination of the [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] -:default_codec!: -[id="string_duration"] +[id="plugins-{type}s-{plugin}-string_duration"] // Move this to the includes when we make string durations available generally. ==== String Durations @@ -474,3 +473,4 @@ Supported values: `us` `usec` `usecs`, e.g. "600 us", "800 usec", "900 usecs" [NOTE] `micro` `micros` and `microseconds` are not supported +:default_codec!: diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index a340f84..7c76a3e 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.4' + s.version = '4.1.5' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 84a46e31feb7d8da0dc0bacf238146bcd33d0f36 Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Mon, 10 Sep 2018 19:22:43 +0100 Subject: [PATCH 39/91] Rescue ENOENT when discoverer stats a discovered file and its missing (#208) This is hard to create a meaningful test for this. The async `rm` call will have to occur at exactly the right point between a file being discovered and it being deleted. Fixes #204 --- CHANGELOG.md | 3 +++ lib/filewatch/discoverer.rb | 7 ++++++- logstash-input-file.gemspec | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1134dc..91a9d5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.1.6 + - Fixed Errno::ENOENT exception in Discoverer. [Issue #204](https://github.com/logstash-plugins/logstash-input-file/issues/204) + ## 4.1.5 - Fixed text anchor by changing it from hardcoded to asciidoc reference to work in versioned plugin reference diff --git a/lib/filewatch/discoverer.rb b/lib/filewatch/discoverer.rb index f73b1a3..81b31ae 100644 --- a/lib/filewatch/discoverer.rb +++ b/lib/filewatch/discoverer.rb @@ -63,8 +63,13 @@ def discover_any_files(path, ongoing) new_discovery = false watched_file = @watched_files_collection.watched_file_by_path(file) if watched_file.nil? + begin + path_stat = PathStatClass.new(pathname) + rescue Errno::ENOENT + next + end + watched_file = WatchedFile.new(pathname, path_stat, @settings) new_discovery = true - watched_file = WatchedFile.new(pathname, PathStatClass.new(pathname), @settings) end # if it already unwatched or its excluded then we can skip next if watched_file.unwatched? || can_exclude?(watched_file, new_discovery) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 7c76a3e..b625469 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.5' + s.version = '4.1.6' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 6438462e985566dadbcdfeea58c71505611f92bd Mon Sep 17 00:00:00 2001 From: Shashank Sahni Date: Tue, 16 Oct 2018 08:14:38 -0700 Subject: [PATCH 40/91] Allowing discovery of symlinks (#215) --- lib/filewatch/discoverer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/filewatch/discoverer.rb b/lib/filewatch/discoverer.rb index 81b31ae..c101559 100644 --- a/lib/filewatch/discoverer.rb +++ b/lib/filewatch/discoverer.rb @@ -56,7 +56,7 @@ def discover_files_ongoing(path) end def discover_any_files(path, ongoing) - fileset = Dir.glob(path).select{|f| File.file?(f) && !File.symlink?(f)} + fileset = Dir.glob(path).select{|f| File.file?(f)} logger.trace("discover_files", "count" => fileset.size) fileset.each do |file| pathname = Pathname.new(file) From 9d698b1d766655e868aa788b5b104b7a3e91b27d Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Mon, 29 Oct 2018 14:19:50 +0100 Subject: [PATCH 41/91] Force all files under rotation to start at 0 or at the sincedb record. (#217) * Force all files under rotation to start at 0 or at the sincedb record. * Update travis.yml to update versions. Fixes #214 --- .travis.yml | 5 ++- CHANGELOG.md | 7 +++- build.gradle | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- lib/filewatch/read_mode/processor.rb | 2 ++ lib/filewatch/tail_mode/processor.rb | 10 +++--- lib/filewatch/watched_file.rb | 6 ---- logstash-input-file.gemspec | 2 +- spec/filewatch/rotate_spec.rb | 46 +++++++++++++++++++++++- 9 files changed, 64 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index b5cfb80..57bc1a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,8 @@ addons: - docker-ce matrix: include: - - env: ELASTIC_STACK_VERSION=5.6.10 - - env: ELASTIC_STACK_VERSION=6.3.0 - - env: ELASTIC_STACK_VERSION=6.4.0-SNAPSHOT + - env: ELASTIC_STACK_VERSION=5.6.12 + - env: ELASTIC_STACK_VERSION=6.4.2 - env: ELASTIC_STACK_VERSION=7.0.0-alpha1-SNAPSHOT fast_finish: true install: ci/unit/docker-setup.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 91a9d5a..426c89f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 4.1.7 + - Fixed problem in rotation handling where the target file being rotated was + subjected to the start_position setting when it must always start from the beginning. + [Issue #214](https://github.com/logstash-plugins/logstash-input-file/issues/214) + ## 4.1.6 - Fixed Errno::ENOENT exception in Discoverer. [Issue #204](https://github.com/logstash-plugins/logstash-input-file/issues/204) @@ -15,7 +20,7 @@ was possible to read into memory allocated but not filled with data resulting in ASCII NUL (0) bytes in the message field. Now, files are read up to the size as given by the remote filesystem client. Applies to tail and read modes. - + ## 4.1.3 - Fixed `read` mode of regular files sincedb write is requested in each read loop iteration rather than waiting for the end-of-file to be reached. Note: for gz files, diff --git a/build.gradle b/build.gradle index f2fd43f..28974c4 100644 --- a/build.gradle +++ b/build.gradle @@ -64,5 +64,5 @@ jar.finalizedBy(copyGemjar) // See http://www.gradle.org/docs/current/userguide/gradle_wrapper.html task wrapper(type: Wrapper) { description = 'Install Gradle wrapper' - gradleVersion = '4.5.1' + gradleVersion = '4.9' } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 568c50b..949819d 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.9-bin.zip diff --git a/lib/filewatch/read_mode/processor.rb b/lib/filewatch/read_mode/processor.rb index 88e8505..17e9092 100644 --- a/lib/filewatch/read_mode/processor.rb +++ b/lib/filewatch/read_mode/processor.rb @@ -80,6 +80,8 @@ def process_watched(watched_files) end end + ## TODO add process_rotation_in_progress + def process_active(watched_files) logger.trace("Active processing") # Handles watched_files in the active state. diff --git a/lib/filewatch/tail_mode/processor.rb b/lib/filewatch/tail_mode/processor.rb index 84a6301..d363806 100644 --- a/lib/filewatch/tail_mode/processor.rb +++ b/lib/filewatch/tail_mode/processor.rb @@ -168,14 +168,16 @@ def process_rotation_in_progress(watched_files) potential_sdb_value = @sincedb_collection.get(potential_key) logger.trace(">>> Rotation In Progress", "watched_file" => watched_file.details, "found_sdb_value" => sdb_value, "potential_key" => potential_key, "potential_sdb_value" => potential_sdb_value) if potential_sdb_value.nil? + logger.trace("---------- >>>> Rotation In Progress: rotating as existing file") + watched_file.rotate_as_file + trace_message = "---------- >>>> Rotation In Progress: no potential sincedb value " if sdb_value.nil? - logger.trace("---------- >>> Rotation In Progress: rotating as initial file, no potential sincedb value AND no found sincedb value") - watched_file.rotate_as_initial_file + trace_message.concat("AND no found sincedb value") else - logger.trace("---------- >>>> Rotation In Progress: rotating as existing file, no potential sincedb value BUT found sincedb value") - watched_file.rotate_as_file + trace_message.concat("BUT found sincedb value") sdb_value.clear_watched_file end + logger.trace(trace_message) new_sdb_value = SincedbValue.new(0) new_sdb_value.set_watched_file(watched_file) @sincedb_collection.set(potential_key, new_sdb_value) diff --git a/lib/filewatch/watched_file.rb b/lib/filewatch/watched_file.rb index 77fd4e8..13a6933 100644 --- a/lib/filewatch/watched_file.rb +++ b/lib/filewatch/watched_file.rb @@ -76,12 +76,6 @@ def set_stat(stat) @sdb_key_v1 = @stat.inode_struct end - def rotate_as_initial_file - # rotation, when no sincedb record exists for new inode - we have never seen this inode before. - rotate_as_file - @initial = true - end - def rotate_as_file(bytes_read = 0) # rotation, when a sincedb record exists for new inode, but no watched file to rotate from # probably caused by a deletion detected in the middle of the rename cascade diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index b625469..1c27e36 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.6' + s.version = '4.1.7' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filewatch/rotate_spec.rb b/spec/filewatch/rotate_spec.rb index 66d9ae4..cf85704 100644 --- a/spec/filewatch/rotate_spec.rb +++ b/spec/filewatch/rotate_spec.rb @@ -28,7 +28,7 @@ module FileWatch let(:max) { 4095 } let(:stat_interval) { 0.01 } let(:discover_interval) { 15 } - let(:start_new_files_at) { :beginning } + let(:start_new_files_at) { :end } let(:sincedb_path) { directory.join("tailing.sdb") } let(:opts) do { @@ -447,5 +447,49 @@ module FileWatch expect(listener3.lines.size).to eq(0) end end + + context "? rotation: when an active file is renamed inside the glob - issue 214" do + let(:watch_dir) { directory.join("*L.log") } + let(:file_path) { directory.join("1L.log") } + let(:second_file) { directory.join("2L.log") } + subject { described_class.new(conf) } + let(:listener1) { observer.listener_for(file1_path) } + let(:listener2) { observer.listener_for(second_file.to_path) } + let(:stat_interval) { 0.25 } + let(:discover_interval) { 1 } + let(:line4) { "Line 4 - Some other non lorem ipsum content" } + let(:actions) do + RSpec::Sequencing + .run_after(0.75, "create file") do + file_path.open("wb") { |file| file.puts(line1); file.puts(line2) } + end + .then_after(0.5, "rename") do + file_path.rename(second_file) + file_path.open("wb") { |file| file.puts("#{line3}") } + end + .then("wait for expectations to be met") do + wait(2.0).for{listener1.lines.size + listener2.lines.size}.to eq(3) + end + .then_after(0.5, "rename again") do + file_path.rename(second_file) + file_path.open("wb") { |file| file.puts("#{line4}") } + end + .then("wait for expectations to be met") do + wait(2.0).for{listener1.lines.size + listener2.lines.size}.to eq(4) + end + .then("quit") do + tailing.quit + end + end + + it "content is read correctly, the renamed file is not reread from scratch" do + actions.activate_quietly + tailing.watch_this(watch_dir.to_path) + tailing.subscribe(observer) + actions.assert_no_errors + expect(listener1.lines).to eq([line1, line2, line3, line4]) + expect(listener2.lines).to eq([]) + end + end end end From f4c8c663ea7352c8d59676912bd69cb524f53d88 Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Tue, 27 Nov 2018 10:52:08 +0000 Subject: [PATCH 42/91] Read + Tail modes, fix endless outer loop when inner loop gets an error. (#221) * Fix endless outer loop when inner loop gets an error. The processing is lagging behind, i.e. bytes read is less that file size. 1. The grow handler begins to a read of the remaining bytes from an offset - as a loop B inside loop A. 2. While the code is in loop B, the file is truncated and sysread raises an EOF error that breaks out of loop B but the watched_file thinks it still has more to read (the truncation has not been detected at this time), loop A retries loop B again without success. Fixes #205 * change to flag_read_error --- CHANGELOG.md | 5 +++++ lib/filewatch/bootstrap.rb | 18 +++++++++++++++++- lib/filewatch/read_mode/handlers/read_file.rb | 5 ++++- lib/filewatch/tail_mode/handlers/base.rb | 3 +++ lib/filewatch/tail_mode/handlers/grow.rb | 2 +- lib/filewatch/tail_mode/handlers/shrink.rb | 2 +- logstash-input-file.gemspec | 2 +- 7 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 426c89f..3b497cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 4.1.8 + - Fixed problem in tail and read modes where the read loop could get stuck if an IO error occurs in the loop. + The file appears to be being read but it is not, suspected with file truncation schemes. + [Issue #205](https://github.com/logstash-plugins/logstash-input-file/issues/205) + ## 4.1.7 - Fixed problem in rotation handling where the target file being rotated was subjected to the start_position setting when it must always start from the beginning. diff --git a/lib/filewatch/bootstrap.rb b/lib/filewatch/bootstrap.rb index d756d32..d4779ae 100644 --- a/lib/filewatch/bootstrap.rb +++ b/lib/filewatch/bootstrap.rb @@ -42,7 +42,23 @@ def to_s end BufferExtractResult = Struct.new(:lines, :warning, :additional) - LoopControlResult = Struct.new(:count, :size, :more) + + class LoopControlResult + attr_reader :count, :size, :more + + def initialize(count, size, more) + @count, @size, @more = count, size, more + @read_error_detected = false + end + + def flag_read_error + @read_error_detected = true + end + + def keep_looping? + !@read_error_detected && @more + end + end class NoSinceDBPathGiven < StandardError; end diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index 2a01cb5..c85f6a3 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -10,7 +10,7 @@ def handle_specifically(watched_file) loop_control = watched_file.loop_control_adjusted_for_stat_size controlled_read(watched_file, loop_control) sincedb_collection.request_disk_flush - break unless loop_control.more + break unless loop_control.keep_looping? end if watched_file.all_read? # flush the buffer now in case there is no final delimiter @@ -42,14 +42,17 @@ def controlled_read(watched_file, loop_control) end rescue EOFError logger.error("controlled_read: eof error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) + loop_control.flag_read_error break rescue Errno::EWOULDBLOCK, Errno::EINTR logger.error("controlled_read: block or interrupt error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) watched_file.listener.error + loop_control.flag_read_error break rescue => e logger.error("controlled_read: general error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) watched_file.listener.error + loop_control.flag_read_error break end end diff --git a/lib/filewatch/tail_mode/handlers/base.rb b/lib/filewatch/tail_mode/handlers/base.rb index 2da9495..9d4ebe0 100644 --- a/lib/filewatch/tail_mode/handlers/base.rb +++ b/lib/filewatch/tail_mode/handlers/base.rb @@ -59,13 +59,16 @@ def controlled_read(watched_file, loop_control) end rescue EOFError # it only makes sense to signal EOF in "read" mode not "tail" + loop_control.flag_read_error break rescue Errno::EWOULDBLOCK, Errno::EINTR watched_file.listener.error + loop_control.flag_read_error break rescue => e logger.error("read_to_eof: general error reading #{watched_file.path}", "error" => e.inspect, "backtrace" => e.backtrace.take(4)) watched_file.listener.error + loop_control.flag_read_error break end end diff --git a/lib/filewatch/tail_mode/handlers/grow.rb b/lib/filewatch/tail_mode/handlers/grow.rb index 755529f..7cc2d26 100644 --- a/lib/filewatch/tail_mode/handlers/grow.rb +++ b/lib/filewatch/tail_mode/handlers/grow.rb @@ -8,7 +8,7 @@ def handle_specifically(watched_file) break if quit? loop_control = watched_file.loop_control_adjusted_for_stat_size controlled_read(watched_file, loop_control) - break unless loop_control.more + break unless loop_control.keep_looping? end end end diff --git a/lib/filewatch/tail_mode/handlers/shrink.rb b/lib/filewatch/tail_mode/handlers/shrink.rb index ac4a00a..1cd5a56 100644 --- a/lib/filewatch/tail_mode/handlers/shrink.rb +++ b/lib/filewatch/tail_mode/handlers/shrink.rb @@ -9,7 +9,7 @@ def handle_specifically(watched_file) break if quit? loop_control = watched_file.loop_control_adjusted_for_stat_size controlled_read(watched_file, loop_control) - break unless loop_control.more + break unless loop_control.keep_looping? end end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 1c27e36..56707ef 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.7' + s.version = '4.1.8' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From a74b21aa2ae74e21f277e9b959abaf0c3952cc34 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 5 Dec 2018 06:03:39 -0500 Subject: [PATCH 43/91] Reduce log level of 'OPEN_WARN_INTERVAL' log message (#224) This was causing frequent log messages, negating the logic below to reduce the frequency of log messages. --- lib/filewatch/read_mode/handlers/base.rb | 2 +- lib/filewatch/tail_mode/handlers/base.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/filewatch/read_mode/handlers/base.rb b/lib/filewatch/read_mode/handlers/base.rb index 6a086d8..ceac0b7 100644 --- a/lib/filewatch/read_mode/handlers/base.rb +++ b/lib/filewatch/read_mode/handlers/base.rb @@ -41,7 +41,7 @@ def open_file(watched_file) # don't emit this message too often. if a file that we can't # read is changing a lot, we'll try to open it more often, and spam the logs. now = Time.now.to_i - logger.warn("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") + logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}") watched_file.last_open_warning_at = now diff --git a/lib/filewatch/tail_mode/handlers/base.rb b/lib/filewatch/tail_mode/handlers/base.rb index 9d4ebe0..5763fb9 100644 --- a/lib/filewatch/tail_mode/handlers/base.rb +++ b/lib/filewatch/tail_mode/handlers/base.rb @@ -85,7 +85,7 @@ def open_file(watched_file) # don't emit this message too often. if a file that we can't # read is changing a lot, we'll try to open it more often, and spam the logs. now = Time.now.to_i - logger.warn("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") + logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}") watched_file.last_open_warning_at = now From 6c979e5c4681ab8051c1716dcee891bb0d55bbc1 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 19 Dec 2018 09:39:16 -0500 Subject: [PATCH 44/91] Version bump and changelog entry Fixes #227 --- CHANGELOG.md | 3 +++ logstash-input-file.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b497cf..0715143 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.1.9 + - Fixed issue where logs were being spammed with needless error messages [#224](https://github.com/logstash-plugins/logstash-input-file/pull/224) + ## 4.1.8 - Fixed problem in tail and read modes where the read loop could get stuck if an IO error occurs in the loop. The file appears to be being read but it is not, suspected with file truncation schemes. diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 56707ef..b04b88c 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.8' + s.version = '4.1.9' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From e68f57d3a069c38fe26e0361a6654e19f8a8b360 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 19 Dec 2018 11:01:53 -0500 Subject: [PATCH 45/91] Fix .travis.yml ELASTIC_STACK_VERSION for 7.0.0-alpha1 Replace 7.0.0-alpha1-SNAPSHOT with 7.0.0-alpha1 as alpha1-SNAPSHOT does not exist on the snapshot website Fixes #227 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 57bc1a3..6c4f115 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ matrix: include: - env: ELASTIC_STACK_VERSION=5.6.12 - env: ELASTIC_STACK_VERSION=6.4.2 - - env: ELASTIC_STACK_VERSION=7.0.0-alpha1-SNAPSHOT + - env: ELASTIC_STACK_VERSION=7.0.0-alpha1 fast_finish: true install: ci/unit/docker-setup.sh script: ci/unit/docker-run.sh From d53079985e693fea45c62205960fa7fe4b79bba5 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Thu, 3 Jan 2019 15:26:11 -0500 Subject: [PATCH 46/91] pin bundler version to < 2 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6c4f115..7dcd01a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ services: docker addons: apt: packages: - - docker-ce + - docker-ce matrix: include: - env: ELASTIC_STACK_VERSION=5.6.12 @@ -13,3 +13,4 @@ matrix: fast_finish: true install: ci/unit/docker-setup.sh script: ci/unit/docker-run.sh +before_install: gem install bundler -v '< 2' From 95ef6f33d81bddf59168d66e46d49ce51dbf7753 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Fri, 4 Jan 2019 12:08:51 -0500 Subject: [PATCH 47/91] Update dockerfile to use compatible version of Bundler Bundler 2.0 introduced requirements that are incompatible with the version of Ruby shipped with Logstash 5.6. This commit installs a pre 2.0 version of Bundler. It also removes an irrelevant step from the travis yml Fixes #229 --- .travis.yml | 1 - ci/unit/Dockerfile | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7dcd01a..c77beb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,4 +13,3 @@ matrix: fast_finish: true install: ci/unit/docker-setup.sh script: ci/unit/docker-run.sh -before_install: gem install bundler -v '< 2' diff --git a/ci/unit/Dockerfile b/ci/unit/Dockerfile index b2fc1c6..da8f046 100644 --- a/ci/unit/Dockerfile +++ b/ci/unit/Dockerfile @@ -6,6 +6,6 @@ COPY --chown=logstash:logstash . /usr/share/plugins/this WORKDIR /usr/share/plugins/this ENV PATH=/usr/share/logstash/vendor/jruby/bin:${PATH} ENV LOGSTASH_SOURCE 1 -RUN jruby -S gem install bundler +RUN jruby -S gem install bundler -v '< 2' RUN jruby -S bundle install --jobs=3 --retry=3 RUN jruby -S bundle exec rake vendor From 082897a66c8410b6b343083146307275d530c88b Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Tue, 12 Mar 2019 21:50:31 +0000 Subject: [PATCH 48/91] Fix for Windows "unknown 0 0" identifier. (#233) * Fix for Windows "unknown 0 0" identifier. Make ruby string look like a C String. * make into module call + bump gemspec Fixes #223 --- .travis.yml | 6 +++--- CHANGELOG.md | 3 +++ lib/filewatch/winhelper.rb | 12 ++++++------ logstash-input-file.gemspec | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index c77beb8..cfccb62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,9 @@ addons: - docker-ce matrix: include: - - env: ELASTIC_STACK_VERSION=5.6.12 - - env: ELASTIC_STACK_VERSION=6.4.2 - - env: ELASTIC_STACK_VERSION=7.0.0-alpha1 + - env: ELASTIC_STACK_VERSION=5.6.15 + - env: ELASTIC_STACK_VERSION=6.6.1 + - env: ELASTIC_STACK_VERSION=7.0.0-beta1 fast_finish: true install: ci/unit/docker-setup.sh script: ci/unit/docker-run.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 0715143..e3a30eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.1.10 + - Fixed problem in Windows where some paths would fail to return an identifier ("inode"). Make path into a C style String before encoding to UTF-16LE. [#232](https://github.com/logstash-plugins/logstash-input-file/issues/232) + ## 4.1.9 - Fixed issue where logs were being spammed with needless error messages [#224](https://github.com/logstash-plugins/logstash-input-file/pull/224) diff --git a/lib/filewatch/winhelper.rb b/lib/filewatch/winhelper.rb index 0d5e859..31c5279 100644 --- a/lib/filewatch/winhelper.rb +++ b/lib/filewatch/winhelper.rb @@ -177,11 +177,7 @@ def self.identifier_from_handle(handle, close_handle = true) private def self.open_handle_from_path(path) - CreateFileW(in_buffer(path), 0, 7, nil, 3, 128, nil) - end - - def self.in_buffer(string) - utf16le(string) + CreateFileW(utf16le(path), 0, 7, nil, 3, 128, nil) end def self.char_pointer_to_ruby_string(char_pointer, length = 256) @@ -192,7 +188,11 @@ def self.char_pointer_to_ruby_string(char_pointer, length = 256) end def self.utf16le(string) - string.encode("UTF-16LE") + to_cstring(string).encode("UTF-16LE") + end + + def self.to_cstring(rubystring) + rubystring + 0.chr end def self.win1252(string) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index b04b88c..ad92eab 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.9' + s.version = '4.1.10' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 0363b94e183efb4d63466ec9805326d1f5d651ca Mon Sep 17 00:00:00 2001 From: Karen Metts Date: Mon, 23 Sep 2019 15:22:07 -0400 Subject: [PATCH 49/91] Update link to FAQ Fixes #247 --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 0d1a4be..0e1c5fd 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -379,7 +379,7 @@ The sincedb record now has a last active timestamp associated with it. If no changes are detected in a tracked file in the last N days its sincedb tracking record expires and will not be persisted. This option helps protect against the inode recycling problem. -Filebeat has a {filebeat-ref}/faq.html#inode-reuse-issue[FAQ about inode recycling]. +Filebeat has a {filebeat-ref}/inode-reuse-issue.html[FAQ about inode recycling]. [id="plugins-{type}s-{plugin}-sincedb_path"] ===== `sincedb_path` From f588d11e048cba5dc4bf2f1f410a35683331d03f Mon Sep 17 00:00:00 2001 From: Karen Metts Date: Mon, 23 Sep 2019 15:25:45 -0400 Subject: [PATCH 50/91] Bump to v. 4.1.11 Fixes #247 --- CHANGELOG.md | 3 +++ logstash-input-file.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3a30eb..d66e1ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.1.11 + - Fixed link to FAQ [#tbd](https://github.com/logstash-plugins/logstash-input-file/pull/tbd) + ## 4.1.10 - Fixed problem in Windows where some paths would fail to return an identifier ("inode"). Make path into a C style String before encoding to UTF-16LE. [#232](https://github.com/logstash-plugins/logstash-input-file/issues/232) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index ad92eab..24b8312 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.10' + s.version = '4.1.11' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 1393071acf072ebe90427cb3a43b1babefd450aa Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Mon, 23 Sep 2019 15:33:36 -0400 Subject: [PATCH 51/91] Update changelog with PR number Fixes #247 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d66e1ac..4b1b82e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 4.1.11 - - Fixed link to FAQ [#tbd](https://github.com/logstash-plugins/logstash-input-file/pull/tbd) + - Fixed link to FAQ [#247](https://github.com/logstash-plugins/logstash-input-file/pull/247) ## 4.1.10 - Fixed problem in Windows where some paths would fail to return an identifier ("inode"). Make path into a C style String before encoding to UTF-16LE. [#232](https://github.com/logstash-plugins/logstash-input-file/issues/232) From c6a8f2811ac5d2c9ed30ab94687a2eb21ccb3bb7 Mon Sep 17 00:00:00 2001 From: David Allouche Date: Fri, 3 Jan 2020 23:52:16 +0100 Subject: [PATCH 52/91] discoverer.rb: Exclude patterns match filename The documentation says: Exclusions (matched against the filename, not full path). Filename patterns are valid here, too. https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html#plugins-inputs-file-exclude Fix can_exclude? method so it matches the filename and not the full pathname. --- lib/filewatch/discoverer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/filewatch/discoverer.rb b/lib/filewatch/discoverer.rb index c101559..0b9d889 100644 --- a/lib/filewatch/discoverer.rb +++ b/lib/filewatch/discoverer.rb @@ -35,7 +35,7 @@ def discover def can_exclude?(watched_file, new_discovery) @exclude.each do |pattern| - if watched_file.pathname.fnmatch?(pattern) + if watched_file.pathname.basename.fnmatch?(pattern) if new_discovery logger.trace("Discoverer can_exclude?: #{watched_file.path}: skipping " + "because it matches exclude #{pattern}") From 2251e43ce672fb3de96840332f2edccb0d97da76 Mon Sep 17 00:00:00 2001 From: David Allouche Date: Mon, 6 Jan 2020 16:53:29 +0100 Subject: [PATCH 53/91] CHANGELOG, gemspec: Bump to 4.1.12 (fix regression in exclude) --- CHANGELOG.md | 4 ++++ logstash-input-file.gemspec | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1b82e..075cb21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 4.1.12 + - Fix regression in `exclude` handling. Patterns are matched against the filename, not full path. + [Issue #237](https://github.com/logstash-plugins/logstash-input-file/issues/237) + ## 4.1.11 - Fixed link to FAQ [#247](https://github.com/logstash-plugins/logstash-input-file/pull/247) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 24b8312..80742b3 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.11' + s.version = '4.1.12' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 2c167da02fe48cfc70c3fa87d0c6f8ec2855f806 Mon Sep 17 00:00:00 2001 From: Joe Marrama Date: Tue, 1 Oct 2019 17:02:53 -0700 Subject: [PATCH 54/91] Fix #248, sinceDB now can handle pathnames with spaces This also fixes a couple of tests that previously didn't test anything due to the input IO iterator not being rewound before being passed to the code under test. Also, this updates the README to specify a missing build step. Bumped version 4.1.13 Fixes #249 --- CHANGELOG.md | 3 ++ README.md | 5 +++ lib/filewatch/sincedb_record_serializer.rb | 2 +- logstash-input-file.gemspec | 2 +- .../sincedb_record_serializer_spec.rb | 35 ++++++++++++++++--- 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 075cb21..91b1431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.1.13 + - Fixed sinceDB to work spaces filename [#249](https://github.com/logstash-plugins/logstash-input-file/pull/249) + ## 4.1.12 - Fix regression in `exclude` handling. Patterns are matched against the filename, not full path. [Issue #237](https://github.com/logstash-plugins/logstash-input-file/issues/237) diff --git a/README.md b/README.md index 5153942..0f4e337 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,11 @@ bundle install ```sh bundle install +``` + + - Build the jar library used for watching files +```bash +./gradlew build ``` - Run tests diff --git a/lib/filewatch/sincedb_record_serializer.rb b/lib/filewatch/sincedb_record_serializer.rb index 81e8a34..208a0a8 100644 --- a/lib/filewatch/sincedb_record_serializer.rb +++ b/lib/filewatch/sincedb_record_serializer.rb @@ -51,7 +51,7 @@ def parse_line_v2(parts) inode_struct = prepare_inode_struct(parts) pos = parts.shift.to_i expires_at = Float(parts.shift) # this is like Time.now.to_f - path_in_sincedb = parts.shift + path_in_sincedb = parts.join(" ") value = SincedbValue.new(pos, expires_at).add_path_in_sincedb(path_in_sincedb) [inode_struct, value] end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 80742b3..e662133 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.12' + s.version = '4.1.13' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filewatch/sincedb_record_serializer_spec.rb b/spec/filewatch/sincedb_record_serializer_spec.rb index 7beaba9..77985be 100644 --- a/spec/filewatch/sincedb_record_serializer_spec.rb +++ b/spec/filewatch/sincedb_record_serializer_spec.rb @@ -9,30 +9,55 @@ module FileWatch let(:io) { StringIO.new } let(:db) { Hash.new } - subject { described_class.new(Settings.days_to_seconds(14)) } + subject { SincedbRecordSerializer.new(Settings.days_to_seconds(14)) } context "deserialize from IO" do it 'reads V1 records' do - io.write("5391299 1 4 12\n") + io.write("5391297 1 4 12\n") + io.rewind + rows = 0 subject.deserialize(io) do |inode_struct, sincedb_value| - expect(inode_struct.inode).to eq("5391299") + expect(inode_struct.inode).to eq("5391297") expect(inode_struct.maj).to eq(1) expect(inode_struct.min).to eq(4) expect(sincedb_value.position).to eq(12) + rows += 1 end + expect(rows).to be > 0 end it 'reads V2 records from an IO object' do now = Time.now.to_f - io.write("5391299 1 4 12 #{now} /a/path/to/1.log\n") + io.write("5391298 1 4 12 #{now} /a/path/to/1.log\n") + io.rewind + rows = 0 subject.deserialize(io) do |inode_struct, sincedb_value| - expect(inode_struct.inode).to eq("5391299") + expect(inode_struct.inode).to eq("5391298") expect(inode_struct.maj).to eq(1) expect(inode_struct.min).to eq(4) expect(sincedb_value.position).to eq(12) expect(sincedb_value.last_changed_at).to eq(now) expect(sincedb_value.path_in_sincedb).to eq("/a/path/to/1.log") + rows += 1 end + expect(rows).to be > 0 + end + + it 'properly handles spaces in a filename' do + now = Time.now.to_f + io.write("53912987 1 4 12 #{now} /a/path/to/log log.log\n") + io.rewind + rows = 0 + subject.deserialize(io) do |inode_struct, sincedb_value| + expect(inode_struct.inode).to eq("53912987") + expect(inode_struct.maj).to eq(1) + expect(inode_struct.min).to eq(4) + expect(sincedb_value.position).to eq(12) + expect(sincedb_value.last_changed_at).to eq(now) + expect(sincedb_value.path_in_sincedb).to eq("/a/path/to/log log.log") + rows += 1 + end + expect(rows).to be > 0 end end From d90a498d01f7c9ca763523d52c98771e02445e8b Mon Sep 17 00:00:00 2001 From: andsel Date: Mon, 20 Jan 2020 12:50:19 +0100 Subject: [PATCH 55/91] Increased wait period to let the watcher to process previous contents Fixes #253 --- spec/filewatch/tailing_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/filewatch/tailing_spec.rb b/spec/filewatch/tailing_spec.rb index 5b71ca4..26e9d74 100644 --- a/spec/filewatch/tailing_spec.rb +++ b/spec/filewatch/tailing_spec.rb @@ -90,7 +90,7 @@ module FileWatch end end - context "when watching a directory with files, exisiting content is skipped" do + context "when watching a directory with files, existing content is skipped" do let(:suffix) { "C" } let(:actions) do RSpec::Sequencing @@ -100,11 +100,11 @@ module FileWatch .then_after(0.1, "begin watching") do tailing.watch_this(watch_dir) end - .then_after(0.2, "add content") do + .then_after(2, "add content") do File.open(file_path, "ab") { |file| file.write("line1\nline2\n") } end .then("wait") do - wait(0.75).for{listener1.lines.size}.to eq(2) + wait(0.75).for{listener1.lines}.to eq(["line1", "line2"]) end .then("quit") do tailing.quit From 88b46801651f22c5489981c4b471e89b5c0adbe1 Mon Sep 17 00:00:00 2001 From: BigYellowHammer Date: Fri, 24 Jan 2020 20:59:57 +0100 Subject: [PATCH 56/91] Fixing delete method to work with multiple files Fixes #254 --- lib/filewatch/watched_files_collection.rb | 1 + .../watched_files_collection_spec.rb | 27 ++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/filewatch/watched_files_collection.rb b/lib/filewatch/watched_files_collection.rb index 3d88a84..475d836 100644 --- a/lib/filewatch/watched_files_collection.rb +++ b/lib/filewatch/watched_files_collection.rb @@ -19,6 +19,7 @@ def delete(paths) Array(paths).each do |f| index = @pointers.delete(f) @files.delete_at(index) + refresh_pointers end @sort_method.call end diff --git a/spec/filewatch/watched_files_collection_spec.rb b/spec/filewatch/watched_files_collection_spec.rb index 08130d5..484aa89 100644 --- a/spec/filewatch/watched_files_collection_spec.rb +++ b/spec/filewatch/watched_files_collection_spec.rb @@ -4,12 +4,15 @@ module FileWatch describe WatchedFilesCollection do let(:time) { Time.now } + let(:filepath1){"/var/log/z.log"} + let(:filepath2){"/var/log/m.log"} + let(:filepath3){"/var/log/a.log"} let(:stat1) { double("stat1", :size => 98, :modified_at => time - 30, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) } let(:stat2) { double("stat2", :size => 99, :modified_at => time - 20, :identifier => nil, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) } let(:stat3) { double("stat3", :size => 100, :modified_at => time, :identifier => nil, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) } - let(:wf1) { WatchedFile.new("/var/log/z.log", stat1, Settings.new) } - let(:wf2) { WatchedFile.new("/var/log/m.log", stat2, Settings.new) } - let(:wf3) { WatchedFile.new("/var/log/a.log", stat3, Settings.new) } + let(:wf1) { WatchedFile.new(filepath1, stat1, Settings.new) } + let(:wf2) { WatchedFile.new(filepath2, stat2, Settings.new) } + let(:wf3) { WatchedFile.new(filepath3, stat3, Settings.new) } context "sort by last_modified in ascending order" do let(:sort_by) { "last_modified" } @@ -70,5 +73,23 @@ module FileWatch expect(collection.values).to eq([wf1, wf2, wf3]) end end + + context "when delete called" do + let(:sort_by) { "path" } + let(:sort_direction) { "desc" } + + it "is able to delete multiple files at once" do + collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction)) + collection.add(wf1) + collection.add(wf2) + collection.add(wf3) + expect(collection.keys).to eq([filepath1, filepath2, filepath3]) + + collection.delete([filepath2,filepath3]) + expect(collection.keys).to eq([filepath1]) + + end + end + end end From 42e2ef2b094741ff5744a4eafeb82f2d70305177 Mon Sep 17 00:00:00 2001 From: andsel Date: Mon, 27 Jan 2020 16:55:02 +0100 Subject: [PATCH 57/91] Bump version 4.1.14 Fixes #255 --- CHANGELOG.md | 3 +++ logstash-input-file.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91b1431..b542d69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.1.14 + - Fixed bug in delete of multiple watched files [#254](https://github.com/logstash-plugins/logstash-input-file/pull/254) + ## 4.1.13 - Fixed sinceDB to work spaces filename [#249](https://github.com/logstash-plugins/logstash-input-file/pull/249) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index e662133..ecb110d 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.13' + s.version = '4.1.14' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 0247b00fddef4f3f335937afc88ec11c981c4ec1 Mon Sep 17 00:00:00 2001 From: andsel Date: Fri, 31 Jan 2020 15:01:00 +0100 Subject: [PATCH 58/91] Bugfixed double conversion to seconds of sincedb_clean_after Fixes #257 --- CHANGELOG.md | 3 +++ lib/filewatch/settings.rb | 6 +----- lib/filewatch/sincedb_record_serializer.rb | 6 +++++- logstash-input-file.gemspec | 2 +- spec/filewatch/settings_spec.rb | 11 +++++++++++ spec/filewatch/sincedb_record_serializer_spec.rb | 2 +- 6 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 spec/filewatch/settings_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index b542d69..aa5a3d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.1.15 + - Fixed bug in conversion of sincedb_clean_after setting [#257](https://github.com/logstash-plugins/logstash-input-file/pull/257) + ## 4.1.14 - Fixed bug in delete of multiple watched files [#254](https://github.com/logstash-plugins/logstash-input-file/pull/254) diff --git a/lib/filewatch/settings.rb b/lib/filewatch/settings.rb index e970435..0c4186f 100644 --- a/lib/filewatch/settings.rb +++ b/lib/filewatch/settings.rb @@ -13,10 +13,6 @@ def self.from_options(opts) new.add_options(opts) end - def self.days_to_seconds(days) - (24 * 3600) * days.to_f - end - def initialize defaults = { :delimiter => "\n", @@ -51,7 +47,7 @@ def add_options(opts) @file_chunk_count = @opts[:file_chunk_count] @sincedb_path = @opts[:sincedb_path] @sincedb_write_interval = @opts[:sincedb_write_interval] - @sincedb_expiry_duration = self.class.days_to_seconds(@opts.fetch(:sincedb_clean_after, 14)) + @sincedb_expiry_duration = @opts.fetch(:sincedb_clean_after) @file_sort_by = @opts[:file_sort_by] @file_sort_direction = @opts[:file_sort_direction] self diff --git a/lib/filewatch/sincedb_record_serializer.rb b/lib/filewatch/sincedb_record_serializer.rb index 208a0a8..5d377b6 100644 --- a/lib/filewatch/sincedb_record_serializer.rb +++ b/lib/filewatch/sincedb_record_serializer.rb @@ -5,13 +5,17 @@ class SincedbRecordSerializer attr_reader :expired_keys + def self.days_to_seconds(days) + (24 * 3600) * days.to_f + end + def initialize(sincedb_value_expiry) @sincedb_value_expiry = sincedb_value_expiry @expired_keys = [] end def update_sincedb_value_expiry_from_days(days) - @sincedb_value_expiry = Settings.days_to_seconds(days) + @sincedb_value_expiry = SincedbRecordSerializer.days_to_seconds(days) end def serialize(db, io, as_of = Time.now.to_f) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index ecb110d..b234257 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.14' + s.version = '4.1.15' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filewatch/settings_spec.rb b/spec/filewatch/settings_spec.rb new file mode 100644 index 0000000..7b08002 --- /dev/null +++ b/spec/filewatch/settings_spec.rb @@ -0,0 +1,11 @@ +describe FileWatch::Settings do + + context "when create from options" do + it "doesn't convert sincedb_clean_after to seconds" do + res = FileWatch::Settings.from_options({:sincedb_clean_after => LogStash::Inputs::FriendlyDurations.call(1, "days").value}) + + expect(res.sincedb_expiry_duration).to eq 1 * 24 * 3600 + end + end + +end diff --git a/spec/filewatch/sincedb_record_serializer_spec.rb b/spec/filewatch/sincedb_record_serializer_spec.rb index 77985be..a751165 100644 --- a/spec/filewatch/sincedb_record_serializer_spec.rb +++ b/spec/filewatch/sincedb_record_serializer_spec.rb @@ -9,7 +9,7 @@ module FileWatch let(:io) { StringIO.new } let(:db) { Hash.new } - subject { SincedbRecordSerializer.new(Settings.days_to_seconds(14)) } + subject { SincedbRecordSerializer.new(SincedbRecordSerializer.days_to_seconds(14)) } context "deserialize from IO" do it 'reads V1 records' do From fcea343e31dcda4eff7fc4b5a9365d7b756638d5 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 19 Feb 2020 11:24:41 +0100 Subject: [PATCH 59/91] Force logstash-devutils to 1.3 (avoid 2.0) to make the tests green Fixes #258 --- logstash-input-file.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index b234257..df5ac25 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -34,7 +34,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0'] s.add_development_dependency 'stud', ['~> 0.0.19'] - s.add_development_dependency 'logstash-devutils' + s.add_development_dependency 'logstash-devutils', '~> 1.3' s.add_development_dependency 'logstash-codec-json' s.add_development_dependency 'rspec-sequencing' s.add_development_dependency "rspec-wait" From cc2fbc07d4c85e96d8494e23d38d7c87874e1474 Mon Sep 17 00:00:00 2001 From: Dawid Trznadel Date: Wed, 22 May 2019 18:13:58 +0200 Subject: [PATCH 60/91] Added settings flag `exit_after_read` that let to terminate the pipeline once it red all files. Closes #212 Fixes #240 --- CHANGELOG.md | 4 ++ docs/index.asciidoc | 14 +++++++ lib/filewatch/read_mode/processor.rb | 11 ++++++ lib/filewatch/settings.rb | 2 + lib/filewatch/watch.rb | 8 +++- lib/logstash/inputs/file.rb | 11 +++++- lib/logstash/inputs/file_listener.rb | 3 ++ logstash-input-file.gemspec | 2 +- spec/filewatch/reading_spec.rb | 59 ++++++++++++++++++++++++++++ spec/filewatch/spec_helper.rb | 4 ++ spec/filewatch/tailing_spec.rb | 2 +- spec/inputs/file_read_spec.rb | 34 ++++++++++++++++ spec/inputs/file_tail_spec.rb | 11 +++++- 13 files changed, 159 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa5a3d6..88c8078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 4.1.16 + - Added configuration setting exit_after_read to read to EOF and terminate + the input [#240](https://github.com/logstash-plugins/logstash-input-file/pull/240) + ## 4.1.15 - Fixed bug in conversion of sincedb_clean_after setting [#257](https://github.com/logstash-plugins/logstash-input-file/pull/257) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 0e1c5fd..f4be83c 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -168,6 +168,7 @@ see <> for the details | <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |<>|No | <> |<>|No | <> |<>, one of `["delete", "log", "log_and_delete"]`|No @@ -241,6 +242,19 @@ In Tail mode, you might want to exclude gzipped files: [source,ruby] exclude => "*.gz" +[id="plugins-{type}s-{plugin}-exit_after_read"] +===== `exit_after_read` + + * Value type is <> + * Default value is `false` + +This option can be used in `read` mode to enforce closing all watchers when file gets read. +Can be used in situation when content of the file is static and won't change during execution. +When set to `true` it also disables active discovery of the files - only files that were in +the directories when process was started will be read. +It supports `sincedb` entries. When file was processed once, then modified - next run will only +read newly added entries. + [id="plugins-{type}s-{plugin}-file_chunk_count"] ===== `file_chunk_count` diff --git a/lib/filewatch/read_mode/processor.rb b/lib/filewatch/read_mode/processor.rb index 17e9092..cdda265 100644 --- a/lib/filewatch/read_mode/processor.rb +++ b/lib/filewatch/read_mode/processor.rb @@ -103,10 +103,21 @@ def process_active(watched_files) else read_file(watched_file) end + + if @settings.exit_after_read + common_detach_when_allread(watched_file) + end # handlers take care of closing and unwatching end end + def common_detach_when_allread(watched_file) + watched_file.unwatch + watched_file.listener.reading_completed + deletable_filepaths << watched_file.path + logger.trace("Whole file read: #{watched_file.path}, removing from collection") + end + def common_deleted_reaction(watched_file, action) # file has gone away or we can't read it anymore. watched_file.unwatch diff --git a/lib/filewatch/settings.rb b/lib/filewatch/settings.rb index 0c4186f..311c112 100644 --- a/lib/filewatch/settings.rb +++ b/lib/filewatch/settings.rb @@ -8,6 +8,7 @@ class Settings attr_reader :exclude, :start_new_files_at, :file_chunk_count, :file_chunk_size attr_reader :sincedb_path, :sincedb_write_interval, :sincedb_expiry_duration attr_reader :file_sort_by, :file_sort_direction + attr_reader :exit_after_read def self.from_options(opts) new.add_options(opts) @@ -50,6 +51,7 @@ def add_options(opts) @sincedb_expiry_duration = @opts.fetch(:sincedb_clean_after) @file_sort_by = @opts[:file_sort_by] @file_sort_direction = @opts[:file_sort_direction] + @exit_after_read = @opts[:exit_after_read] self end diff --git a/lib/filewatch/watch.rb b/lib/filewatch/watch.rb index ea5b9f7..3c88dd1 100644 --- a/lib/filewatch/watch.rb +++ b/lib/filewatch/watch.rb @@ -43,10 +43,11 @@ def subscribe(observer, sincedb_collection) reset_quit until quit? iterate_on_state + # Don't discover new files when files to read are known at the beginning break if quit? sincedb_collection.write_if_requested glob += 1 - if glob == interval + if glob == interval && !@settings.exit_after_read discover glob = 0 end @@ -76,7 +77,10 @@ def quit end def quit? - @quit.true? + if @settings.exit_after_read + @exit = @watched_files_collection.empty? + end + @quit.true? || @exit end private diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 511e4dd..abcd146 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -222,6 +222,11 @@ class File < LogStash::Inputs::Base # perhaps path + asc will help to achieve the goal of controlling the order of file ingestion config :file_sort_direction, :validate => ["asc", "desc"], :default => "asc" + # When in 'read' mode - this option is closing all file watchers when EOF is hit + # This option also disables discovery of new/changes files. It works only on files found at the beginning + # Sincedb still works, if you run LS once again after doing some changes - only new values will be read + config :exit_after_read, :validate => :boolean, :default => false + public class << self @@ -260,6 +265,7 @@ def register :file_chunk_size => @file_chunk_size, :file_sort_by => @file_sort_by, :file_sort_direction => @file_sort_direction, + :exit_after_read => @exit_after_read, } @completed_file_handlers = [] @@ -280,7 +286,7 @@ def register raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"") end end - + @filewatch_config[:sincedb_path] = @sincedb_path @filewatch_config[:start_new_files_at] = @start_position.to_sym @@ -301,6 +307,9 @@ def register end if tail_mode? + if @exit_after_read + raise ArgumentError.new('The "exit_after_read" setting only works when the "mode" is set to "read"') + end @watcher_class = FileWatch::ObservingTail else @watcher_class = FileWatch::ObservingRead diff --git a/lib/logstash/inputs/file_listener.rb b/lib/logstash/inputs/file_listener.rb index 71ddd42..3b6d15a 100644 --- a/lib/logstash/inputs/file_listener.rb +++ b/lib/logstash/inputs/file_listener.rb @@ -21,6 +21,9 @@ def eof def error end + def reading_completed + end + def timed_out input.codec.evict(path) end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index df5ac25..75a824c 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.15' + s.version = '4.1.16' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filewatch/reading_spec.rb b/spec/filewatch/reading_spec.rb index 53deee1..e59b480 100644 --- a/spec/filewatch/reading_spec.rb +++ b/spec/filewatch/reading_spec.rb @@ -147,6 +147,65 @@ module FileWatch end end + context "when watching a directory with files using exit_after_read" do + let(:opts) { super.merge(:exit_after_read => true, :max_open_files => 2) } + let(:file_path3) { ::File.join(directory, "3.log") } + let(:file_path4) { ::File.join(directory, "4.log") } + let(:file_path5) { ::File.join(directory, "5.log") } + let(:lines) { [] } + let(:observer) { TestObserver.new(lines) } + let(:listener3) { observer.listener_for(file_path3) } + let(:file_path6) { ::File.join(directory, "6.log") } + let(:listener6) { observer.listener_for(file_path6) } + + it "the file is read" do + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + reading.watch_this(watch_dir) + reading.subscribe(observer) + expect(listener3.lines).to eq(["line1", "line2"]) + end + it "multiple files are read" do + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + File.open(file_path4, "w") { |file| file.write("line3\nline4\n") } + reading.watch_this(watch_dir) + reading.subscribe(observer) + expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4"]) + end + it "multiple files are read even if max_open_files is smaller then number of files" do + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + File.open(file_path4, "w") { |file| file.write("line3\nline4\n") } + File.open(file_path5, "w") { |file| file.write("line5\nline6\n") } + reading.watch_this(watch_dir) + reading.subscribe(observer) + expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4", "line5", "line6"]) + end + it "file as marked as reading_completed" do + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + reading.watch_this(watch_dir) + reading.subscribe(observer) + expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed]) + end + it "sincedb works correctly" do + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + reading.watch_this(watch_dir) + reading.subscribe(observer) + sincedb_record_fields = File.read(sincedb_path).split(" ") + position_field_index = 3 + expect(sincedb_record_fields[position_field_index]).to eq("12") + end + it "does not include new files added after start" do + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + reading.watch_this(watch_dir) + reading.subscribe(observer) + File.open(file_path6, "w") { |file| file.write("foob\nbar\n") } + expect(listener3.lines).to eq(["line1", "line2"]) + expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed]) + expect(listener6.calls).to eq([]) + + end + + end + describe "reading fixtures" do let(:directory) { FIXTURE_DIR } let(:actions) do diff --git a/spec/filewatch/spec_helper.rb b/spec/filewatch/spec_helper.rb index f074133..0490536 100644 --- a/spec/filewatch/spec_helper.rb +++ b/spec/filewatch/spec_helper.rb @@ -152,6 +152,10 @@ def eof def timed_out @calls << :timed_out end + + def reading_completed + @calls << :reading_completed + end end attr_reader :listeners diff --git a/spec/filewatch/tailing_spec.rb b/spec/filewatch/tailing_spec.rb index 26e9d74..bb0ff62 100644 --- a/spec/filewatch/tailing_spec.rb +++ b/spec/filewatch/tailing_spec.rb @@ -448,7 +448,7 @@ module FileWatch FileUtils.mv(file_path2, file_path3) end .then("wait") do - wait(2).for do + wait(4).for do listener1.lines.size == 32 && listener2.calls == [:delete] && listener3.calls == [:open, :accept, :timed_out] end.to eq(true), "listener1.lines != 32 or listener2.calls != [:delete] or listener3.calls != [:open, :accept, :timed_out]" end diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index d7361bb..9b38dc6 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -75,6 +75,40 @@ end expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world") end + + it "should read whole file when exit_after_read is set to true" do + directory = Stud::Temporary.directory + tmpfile_path = ::File.join(directory, "B.log") + sincedb_path = ::File.join(directory, "readmode_B_sincedb.txt") + path_path = ::File.join(directory, "*.log") + + conf = <<-CONFIG + input { + file { + id => "foo" + path => "#{path_path}" + sincedb_path => "#{sincedb_path}" + delimiter => "|" + mode => "read" + file_completed_action => "delete" + exit_after_read => true + } + } + CONFIG + + File.open(tmpfile_path, "a") do |fd| + fd.write("exit|after|end") + fd.fsync + end + + events = input(conf) do |pipeline, queue| + wait(0.5).for{File.exist?(tmpfile_path)}.to be_falsey + 3.times.collect { queue.pop } + end + + expect(events.map{|e| e.get("message")}).to contain_exactly("exit", "after", "end") + end + end describe "reading fixtures" do diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index 7cca515..ac47c9a 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -175,7 +175,7 @@ context "when sincedb_path is a directory" do let(:name) { "E" } subject { LogStash::Inputs::File.new("path" => path_path, "sincedb_path" => directory) } - + after :each do FileUtils.rm_rf(sincedb_path) end @@ -184,6 +184,15 @@ expect { subject.register }.to raise_error(ArgumentError) end end + + context "when mode it set to tail and exit_after_read equals true" do + subject { LogStash::Inputs::File.new("path" => path_path, "exit_after_read" => true, "mode" => "tail") } + + it "should raise exception" do + expect { subject.register }.to raise_error(ArgumentError) + end + end + end describe "testing with new, register, run and stop" do From deaee4e025e4c0e6b65b816fe6973ddbceb573aa Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Mon, 9 Mar 2020 11:21:48 +0100 Subject: [PATCH 61/91] Test: adjust for devutils 2.0 compat --- logstash-input-file.gemspec | 2 +- spec/inputs/file_tail_spec.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 75a824c..2b64bff 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -34,7 +34,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0'] s.add_development_dependency 'stud', ['~> 0.0.19'] - s.add_development_dependency 'logstash-devutils', '~> 1.3' + s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'logstash-codec-json' s.add_development_dependency 'rspec-sequencing' s.add_development_dependency "rspec-wait" diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index ac47c9a..367ce07 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -1,6 +1,7 @@ # encoding: utf-8 require "helpers/spec_helper" +require "logstash/devutils/rspec/shared_examples" require "logstash/inputs/file" require "tempfile" From e9ed605691828210580895533fca23b943ba90b2 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 17 Mar 2020 15:41:55 +0000 Subject: [PATCH 62/91] [skip ci] updated apache license --- LICENSE | 209 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 199 insertions(+), 10 deletions(-) diff --git a/LICENSE b/LICENSE index 2162c9b..a80a3fd 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,202 @@ -Copyright (c) 2012-2018 Elasticsearch -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Elastic and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 29cc105e0930baa332076120307aa6a0e0db7934 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 20 Mar 2020 16:46:16 +0000 Subject: [PATCH 63/91] update to centralized travis configuration --- .travis.yml | 17 ++----------- build.gradle | 7 ----- ci/unit/Dockerfile | 11 -------- ci/unit/docker-compose.yml | 17 ------------- ci/unit/docker-run.sh | 7 ----- ci/unit/docker-setup.sh | 31 ----------------------- ci/unit/run.sh | 6 ----- gradle/wrapper/gradle-wrapper.jar | Bin 54333 -> 54413 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- 9 files changed, 3 insertions(+), 95 deletions(-) delete mode 100644 ci/unit/Dockerfile delete mode 100644 ci/unit/docker-compose.yml delete mode 100755 ci/unit/docker-run.sh delete mode 100755 ci/unit/docker-setup.sh delete mode 100755 ci/unit/run.sh diff --git a/.travis.yml b/.travis.yml index cfccb62..a50fc73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,2 @@ ---- -sudo: required -services: docker -addons: - apt: - packages: - - docker-ce -matrix: - include: - - env: ELASTIC_STACK_VERSION=5.6.15 - - env: ELASTIC_STACK_VERSION=6.6.1 - - env: ELASTIC_STACK_VERSION=7.0.0-beta1 - fast_finish: true -install: ci/unit/docker-setup.sh -script: ci/unit/docker-run.sh +import: +- logstash-plugins/.ci:travis/travis.yml@1.x \ No newline at end of file diff --git a/build.gradle b/build.gradle index 28974c4..47ff4f5 100644 --- a/build.gradle +++ b/build.gradle @@ -59,10 +59,3 @@ task cleanGemjar { clean.dependsOn(cleanGemjar) jar.finalizedBy(copyGemjar) - - -// See http://www.gradle.org/docs/current/userguide/gradle_wrapper.html -task wrapper(type: Wrapper) { - description = 'Install Gradle wrapper' - gradleVersion = '4.9' -} diff --git a/ci/unit/Dockerfile b/ci/unit/Dockerfile deleted file mode 100644 index da8f046..0000000 --- a/ci/unit/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -ARG ELASTIC_STACK_VERSION -FROM docker.elastic.co/logstash/logstash:$ELASTIC_STACK_VERSION -WORKDIR /usr/share/logstash/logstash-core -RUN cp versions-gem-copy.yml ../logstash-core-plugin-api/versions-gem-copy.yml -COPY --chown=logstash:logstash . /usr/share/plugins/this -WORKDIR /usr/share/plugins/this -ENV PATH=/usr/share/logstash/vendor/jruby/bin:${PATH} -ENV LOGSTASH_SOURCE 1 -RUN jruby -S gem install bundler -v '< 2' -RUN jruby -S bundle install --jobs=3 --retry=3 -RUN jruby -S bundle exec rake vendor diff --git a/ci/unit/docker-compose.yml b/ci/unit/docker-compose.yml deleted file mode 100644 index 1dd26fd..0000000 --- a/ci/unit/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '3' - -# run tests: docker-compose -f ci/unit/docker-compose.yml up --build --force-recreate -# only set up: docker-compose -f ci/unit/docker-compose.yml up --build --no-start --force-recreate -# start manually: docker-compose -f ci/unit/docker-compose.yml run logstash -services: - logstash: - build: - context: ../../ - dockerfile: ci/unit/Dockerfile - args: - - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION - command: /usr/share/plugins/this/ci/unit/run.sh - environment: - LS_JAVA_OPTS: "-Xmx256m -Xms256m" - OSS: "true" - tty: true diff --git a/ci/unit/docker-run.sh b/ci/unit/docker-run.sh deleted file mode 100755 index 4007c55..0000000 --- a/ci/unit/docker-run.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh` -# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable. -set -e - -docker-compose -f ci/unit/docker-compose.yml run logstash diff --git a/ci/unit/docker-setup.sh b/ci/unit/docker-setup.sh deleted file mode 100755 index 37bbcd0..0000000 --- a/ci/unit/docker-setup.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh` -# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable. -set -e - -if [ "$ELASTIC_STACK_VERSION" ]; then - echo "Testing against version: $ELASTIC_STACK_VERSION" - - if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then - cd /tmp - wget https://snapshots.elastic.co/docker/logstash-"$ELASTIC_STACK_VERSION".tar.gz - tar xfvz logstash-"$ELASTIC_STACK_VERSION".tar.gz repositories - echo "Loading docker image: " - cat repositories - docker load < logstash-"$ELASTIC_STACK_VERSION".tar.gz - rm logstash-"$ELASTIC_STACK_VERSION".tar.gz - cd - - fi - - if [ -f Gemfile.lock ]; then - rm Gemfile.lock - fi - - docker-compose -f ci/unit/docker-compose.yml down - docker-compose -f ci/unit/docker-compose.yml up --no-start --build --force-recreate logstash -else - echo "Please set the ELASTIC_STACK_VERSION environment variable" - echo "For example: export ELASTIC_STACK_VERSION=6.2.4" - exit 1 -fi diff --git a/ci/unit/run.sh b/ci/unit/run.sh deleted file mode 100755 index 91e54bb..0000000 --- a/ci/unit/run.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# This is intended to be run inside the docker container as the command of the docker-compose. -set -ex - -bundle exec rspec -fd --pattern spec/**/*_spec.rb,spec/**/*_specs.rb diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index c44b679acd3f794ddbb3aa5e919244914911014a..0d4a9516871afd710a9d84d89e31ba77745607bd 100644 GIT binary patch delta 7645 zcmY+J1yEGq+sBth=@jYiT)J5r36YX+0f_|`q+M$1kY;J5J0yO9G$PHy(nv@P(v8yi z`uhIH|NFl)cjozg&v%}C?lWiRp8K56a`d%}x*SpWc_mn5f)X?*5>>W&6T zq^kdD1;Vz74z*Fl1ON^Z47@yWA3a0T2hHzpk;9J_Ov-ihHK1X`6R#PbCY}&wO^AA9 z2{S5jIJ(;LB5=hn<9&`!%3VKk9#Ou1fAIPK_d$-A$)FBk%Go0)*cuLn{ z$a3t|e|w6ibCr)r@6sQ6vTor2h(7hZjex&dUqat0w!dJ!O8PD-cATPiM#g;H+nv^Xy%D?E~{COY!*h`l+B-O5lb8FrhN zdL64ywU8||p>E%IILNQ=y_ zfgY)rY+9`uPL2KyuDq&Ac*Ewi?yF}wVXbJE=AnvoSSU_j>zI z@LSbYR{Zi?L_BD|(k5Hjwa=S2V7|bH z-FtUFw07E{lbd$6JDb$Sv3TB`b!03vH{Nx_RI)4Xn7>#0NR4*ewgYo@_Eg&6a*(hX z_VTrCUxF|kT=ET=YY=tu6mNX)H1@?W3i92iG%c!QqL^taq$IDE@}MoaYW-b&-ddSt zZtuBdU__qNRd_u4i8}tLsnf#wYmxXQEc8<7jdfU<%`5X`9@IU?r`eO__|5RvP?~qB_^DB)kA|N(gO*anC+1HSj z3qmR&H^cjZ(?kJ?=u|+B^E+r{S%cI+(ht~n-&m5e@H|X*r4zox#uyl~LospRD1W&a^_tsb`;0eJ_Pg#Brv< zNDpzY9$@$lM2^CaK}j@bKZ2eMf0+>Nd@ST5V~M@rA*|;#W(`p_o29UEZXX>(+C@u) zFBPvSh^X{)CjnR&gNc1OSV%@cVYVmz7`{Q;Y1t1=nl5_+ONEbKoj$| z>d=gc=gR_dX?mB!3KdFksn+$F1mje>h;x*My|5RjXPo6_zjDl>Qwww()cV)<$}Hi+ zmQDi;$v!mS;S<-_ShR=CC(S%-OsR)&;%un!7Kl*n+VVgx2TEhN(yu%Vh4lvBaaJ#D zE8jjf(8*V#c(KGEO3n9(`)ghhnM}eP2lByn%4!-Dp>RaZjj_mn8Q*)XbhqgYqrA`i zm6eCsT%PG}La9h2+7~s%=g}v&)%c9W1l7xe2B9LMrng)T9gVk4(L=UNrTpP=^Rpkm z{p3p3e2Z(F-jZrct?2-bSsf%m}6$vQ#>n035PY&VF zDI5kAP`IuV;VI6p)1K3O0qv71dCm(}B}UD>2@`LP*CF(0WfD@IJ{YK1+61QT8TLA+ ze411s)f6{EeI!?N_mFDMWMqaXB*jSxx`mj5RoG-Sc`aB-ELq4c8C4#}Qh)UXnZyJ* zN*@_;E>+#Q6%L$B14(n(%4CBl_y<94YJ+&n*>HfhbN<5n|e^dpKfY`?|D&rwqXOVFF zD^9EqUr4hajBb>+ptdL_j1!g*<)ei2yaG?93_+s!$+LNR+|8KuIU4NdyP67riFR~O z$zl}=ipBFBtsA~K=`6pyC)ZX-fAkapz{2|d>6Hg$q>&n9lK=qKoB#j=0Fg96iqI4x zhU;xL$+8P&N>TEfGIZC)tFrjbCj~jEhA1h@0sG`gUF=+vaU1%id>G`bzJwSZS}&V4 z@_H$9>(?jeyH(*f)j*rFo0?*dJ7sPcx4+)3?C*2m=lR*QAvT#9K3R6%qMG0TfRFif zCCokiyzo#G&Vy6u(~YGMu*WgQS!RU*;lgMT(V295R5} zlE|(qB+&f~%qiD=`WjNoL4!A;l*_eW#tB{?7XpG2_M9|06M8bt-3`|-C^MNg>U-=T zJCG$1oS5-qYlG5}j3pt#-`=*pWPrpYM~~(}ZS%W07IXs*xl7L!m&Pqx$FhwctO>>&oGLC+FdT zF3G`7Xp#xpCC21yKm3z=sGqtd9Ss0P&Po9v)=@ak7!Dt_EqlptTQvfmlS&oVKcG3T zk8}8|1&Iz{P63+E9AsS)$KDV)7qHr`J+C(~KB~?a&F4*7`OKERVwaRsat@9PdU0>8 zgLRSk7&fGhw2Mbhs>OmzmyC_{<5SJY9E?uiI@CoE$UuI;r4hf)ZMt1^UV??96Vexb z#i9e>+KkC(lhve$pH{mXnYUL!zJ0tINQ}(Zm~g4?e@TsZ}4o6YF#dYhL~wyOX zkuyBZ#FiF&;>)UfF&cfO>r1fGDP6Y5aZImVsSY`>JBMQh(-F!%oJjMj zsS=P;M=>j@4HZ46s`X)lY<~+pMuA@A+?ig^={n!4 zIbaOHx^uFE9Ry!kVASM8o;k$&l<`c=Hc{9mF%He;?(GnjZ$r()9h{ECFGbXCxB$8; zz*0`;ViVhAVBBs_n?7}d^B0YxBC$CG>L=K*99H=6l`F-7k_e9g(^~f4(#gY?QEGt~ zD{TX4q^UBLJr8T_}YA|WmAIDcF0Or zBGO0^Ogh~Vy=W7&6}RJV!r>Hs@|%f+>Cy= z1pji%duW~jRELEd|g*11a`|MN^EVgkt6xN(8K zMJ7#Yf~%#uU);*7=#(^338pnM#Sr65Ly_n7e4j>D!K869TX~3iN3^~!w@xNAHeEC} zrJ_LIl0HPWrE~hjf~gjT>H0UM0r^wJd%E!5vJ0V9j2+5U8I`$mPg3&_Q$-gU&u_0C z`DH%f=!yntmIQcyc%E0*iDOxGwhgOA-d>r6UydZ~>pcK1YtQigrQ^3rxJB%{h;9p! zmK%JE;b!>6D3FsQw@xv~9cilTW-U{t2l-|nSJcD^&T8=)%0C9#dk79RZr$z-ia*8>b5-pJF_+pmh)n-HhlVS(hATL@ zxVgJ91mWiH3XD@7h*wEi{W3X^xMceXt31g+ORjFH=*c{) zdXsK$M`MbDaC}c4*3a+5J_dH?@JG zzS|P-5Xxw1Z;D6Q*P+I#Ra$HY%2qeIhyQu1vu8x!#Fh%8zY8m;O)eC9%>uWg(e(_o zVfu%)IFksJg9;+fHuv=_?|spyU@|1At5%<`E2kfAU9j|AC@iazP47KVy#185DpEev zJ)(h(u*|8Q>Z=X|No5mmBQXgV+;rlq9GP(X4Ts*~8r$v8=Sq!cHWbqEMYzdgIERbA zfWB&J@Scl4bw&4tvXfhtpb<`e>iG!AKX+i)zYu7Tbs*T}6WGF?CP%zs-1fE}iSGYF z8>YNve=QN7XL$V>?HkMGBCeQM`t)q3?sfZJ;e@YRR(j-4>IalwV0djmF`q#SmUx=6 zXyd@kYrlV-zq6+sXVM6=uojc@RjC&j5=(jN_%X2v_S={H)!pEEv5dmA5{qMg|MYhu zYfnSPBj;4SK%5LX!=#2l0=j47l(eGV-cD}n3ZQTiV2!7^v!hZ9cl2w0n(%C7?Kw5h zr;0PvyN^iOHF|58X5DWUcg%q*TRxOsJnkT8y_-aM;fa?1LuSjv_TBHD88Z%lDjW^- z>=qci+Z*uFMb%c6RSdrRT{F5pkaOiW^*Q|{{nH1zEz9#+u3r(ZiW^|J9u=!>%A~i- zkmj|E+HOj~yymDI2*-+3PMwN+OX~8e>n#Vi8=U4j+)QovWd9+>D#o36f<)b{cC-Ki z8E8E8%oUTFHI-lIuRnB>H|YEvf)wF)pimOiWdfW2GCz#6rtuBxL}#J%F*660xN?oETm zSKnYpTb4Q#X9mv;09T0D*SL(_8!#~sW)w6@r3fbD*Sp}R%UVX}bRJS8vGz<=+6MLW zn8b!RCj}q}!PwGo7O-qyFCR|xDdK1n6eU}+U)pYG9Vo$}euBgr#%+Dm6C`!ygEDtJ zpVgB6S}h477<`o^W)W8LF4v@~D$JdObgoF4Vh7FsHT?*m{=u;%qt~JZ^ON_`J7eTbun#JWOJ=*;EFiZogWerPC~^{ zht7_9itunBnJDYxJi2sFvg%pWyXjX}zJCJN_}MV}Zn|eOYLIr=WuX~?)K&8T{+a!4 zk5l8)FE~k#=1-%%8pX^pN!FVYA#o3&>dR)ZEYxxM;quv7pSu*MFsnDSTj2!7W>Wh= zYGeqG!m;t3TgV&7M*6Pg_*?T;cnE8&GVRUP%T3AP#J#CxCz8??qFI}vQNU<<%Zg12 zoIsYpL7=KTc8ofW@8dMg9-jbc(V133MN^0P=irxMCqLSFgNUbV*JqN zB;L|ZRWW>7VD^5JFU7R+zW;viVM8$X_)+z;)?C3oFwh(y$HIR*<%h2P z0^so?j!Ima;jNmqLeU5^A@QYwTbRFXs*1y?Rd(+Whq7eCL+1m$NhL`V!O?|)Q{xHp(OQn$gR(-ItP7wh2Evl2q1KeQ$|y=wuqLsXt1I+oh7Z_@ojKY=ppNl$#rE}xKl7= zRb@K;ZA^y`?XMQ5jAPD_WFZ2&Uv-5Ec3dBP&(ySPS*+X74Rm_{D>`;pMTJWsmzs+ z>mU$-IAZ<8kt4D%(?+PQ(D!QY{sImtrciI+k(b13AbXii*F(VPptNgsLpIv>#Ef2k zJo5Otq|$0cwmbeEhjbdos89xRxk&ownrl?5)}%4s>I=&JR%kfuihV^RN_WydCFtmE zTfj6}&)vtj&Hbuzo3j#GBpd{m04&CR#1y;)Hm0M?7;4i3p+q|1~olEAIFqQl{BL^zAMa93QUbL1RJ%l0uI;0`^ z;ymbwZyU-0!kUqXFMrDj50o+BRn)Vvs#-|YZwSu)$0PpY(Jz?yF+@t!J(Vgbl%^yw zstf>eH0w~;Bq zK-0K8jJX_mkOARD)<-l) zXT&P`#dP@n3`^!KIH-EgtSAchrD^smVRAG7Wy?yi`^4&$3V9Qgt7^BagN!PI<{#=w z+AU8pF?55HpeCYK$>kdtX%d;S)*NipXwNwMb4=qr0 zpbzq*4|_(}Q*g#_09L>y{vZ-Hryn(GptS2f@5p}OL(20JP6%iv;P^y0(MEiqjaF@n zRxAZTw4DUzcLM7#^Zkl!@{4_zcW2iXOM0> znJQ{L#9mxcft%>Uto-Q1&yULdj~T}coQ+5q!4@nqRPH017ii>@uaXrRQKlsV7bH@z z&o(oqP}^v#oGqp%E;gMwc|#`SNJ>)G6Z@2fiHeMR!2B+IEe(OVJ2>*EFAG7P=R-b` zrt>EZ^iBc;SHO;I&udpk>CU*<4;Y{zN2VF3I_`l=c;2)hksn?@-|%9nwEZOCPJroV ziLnQ$#B%XF5+QbT5&eZg6L^x1p_<-M-7BwOR=mQi)`=mB!CdSuuDt>aVvAuTJ!8f( zzu~|L>k^W4>z0}neO;5+cDpM`b0lAjmnm;4%ldfli#*j~&AwLEGbkWttm&o*$-?k%5uUR|)wS zeiJuXk<0XQyUZ;|mU8=ALpi^2p2HPVPwJtQ_;rIcZ5e@a_t*0xxWa*$&GEejr;PXL z&bX$wAYT#=sli^23oTJV$>N(YkBMkLmBAy(;F)-fA=e4rtHb$XuU>S&1h2#(3tBs# zkSczCa!e~+S~yih?oA7RQ0cZ7vtFRGsJzEqlvOjUy9wIQH0z9iwjQ9pVpII#=kbhYJ5pXoGiE+Wq_FO#LA=d~tkIh*=a6aV2u6GC zdsBE;fx@rckI_JZt)bCe28M^c3(MVC6$6F_Tk5{<>>ng?X{P4y?02Nx8;)m0e-Ylh z`g&+c(q0`f^#(MP9CcM2$f$5nT;s+bDh&GV2=CsdKFC79|EP;+p_Lttt(^0iPfga% zx@t(d%inyz5IDae(Qtp!TwpJHMPu*oWuMC}m96EyeJFAC$RI;v97S=UdHZM}A54}D zJ1CL%#u!uV@pxDscy=MO@d>>Ft}5(H((nkNt&WNHmYLh;cQ*|{{QE}8_tGFXd$|8| z(1oDr)kVufIQI&mbs=(l#euv3*Ks@S6{b{~wp-~|!`iT%meN;dLj6c#CHWnhO zPY@`H`4_z~DG|4QT)=GXeb(? zME;m^2gQJss(->dRlMI1CPH8hQ2}48|B>>B@DbYT6bOqUE})CXzmhaCOYctz)1yTY z4RZl~jsKvrF%`n;HyW7z!RldIG#kXhur^u`LK+E1J4N^+Er9+Gh;~~_grx%uq6SHX zU>~6ZGC~o-E;ID8 zoXr1{*Cyo9yb+9(mS`wM@T4;kxA5O#3`9{s8G?Q4_gZN{#I!RZY^V5vpWA8 zc{L=r^T=>;0#GUrHi$ogE@_b3HP4uJFpj>ff$fa^A{!=pkveJ!8mf~_h9tQ(qJ>w} zJ4fWVyhnf7GODj4TU+dFbjs|zL2*7+DU1Y8M|HpM<6eTl-s|30o(MMQtU9Kn?1Haz zh1UWyZUWZctqFr4)*5<{P062IFbq!HFsM^EWOzt;R7R52=pb>S$h`I)}Y4A|Zi&EMm4x<$bX$z^(Y?FDtdRKCGUxs=X`^{pl5r9X;o@Rk1GZlWjIr zo`5IaF6A`95^Fdx&wIabizd^?tUbL1hu_My5lmjcKK5*s7wCkz`E{i$& zmxMfvG}4G;u1qJP!9OH~xpbJD&oJ%drRv6dqg33AKM?X3cE7N6i#vZpj?xTG!_08+ zntNjxKdm^88!D}6tac>pK&wgj@tM)_)B{TX@=~l-Vc-~df3u|#zPwcin)EcHsLFso&oy0z;BD!W58-)O9r>mGo0e9D2 zIdL_o4@9L|oyvPkq{V@NU>=pH)G?PY%7C=^!5L_i&+XI{?c151yI0AIBKGr&aDi>* z=1w#4gj`<*nb_H~!StOghf6wx^~ZkQ_K17uuv+ST5vyIs*(UCqeUhNTlAnQbgxQ-( zy?3#DSqTM{=Ffa z=RK8~411Fv_hV7MbNiZbW15j56%f{wEpu@DmI|b|tW@$uIL48gpAhFOHQajYq~jLn zuRrW#DHcua6+pZ;cNRz*MAMXUDuXF8d&a{79B4_!nEH8U0e-@NOw~BLD!stmJhLGS zs)uY5-KJVqh|q(@m7Pc^6>KJd9Y*9swNv# zjlM%wW6+2>o@Kvfz9^c{iV#-suGriuLM(*!kNvXzFM(h=6JrK;N>lL6Wg-cHl_6>9 zs*>BHWwv$ZaG#Pg#YYcKYa&}L)d8l)hv?h9eB9TJ=%lO z?Kp>}(|b@bk>r^V-(vg1I?KqH^}+Qx{lOu}iO=23hTP^;Ps#f!(i=8`zRSMrm1T$@ z&melAQgeuRPN9>@kjSqVCf!MPqA(Z-vrJtSn|9@RSPzd zPkW=l6kGuMydKWG$_;N8f$-ihYq}-Yx;Ja?WM!n0-CpyetaOg+f&633lR5*P9ovnF zLM{yxqC8<)71mrA>loJgwOt(wx8ws<`X@VcxSo<*UEIxJA{LJmHW8k{=4gne=FCS~ z-l|15^DIyR(b>4RZG(C<0=8@X998^_af07z7w!WWMtj(qVlh#-k7z`uuP&#F!lv&o znn<;tH|5Bd?hI-DB}=+b!@Bz5Xie9XI$4$Z54qYkL%AEd+9Jsqu7J;eSgl4_?42gT zMBU-Gme@aMgjd8nj_Fi6No^n#0l6WzEc%1LP|w7peC9!&TA=-!O)M5|H=A4cyf!tt z2D3oKD%<_P^)Vu~!Yg7pT|0N8ILd?d8+ozB31a7$91(&^B+;#i?NL&iZpVzWTIi{< zQ$;qLvk>D3aDw*PF-uLK@8j7{Igy}Uc9iXahL1~IJnX(sm}Xjqk{9CBmJFtdMsl<8 zSR6N8dzTz*gtbWfBap&D_?rHM>R-A7cgtA@Fo-6qe5819n@S$b$SiHGba3r6^~*P$ z=LoYeRBz_r8Y}Vm>S1maR{;wm|w~F$$1d9WzZzhHRN_U8%pr%vn@s+pVsdDhWJ8EPf{ye=YcpBTt4% zHkcVTl!iL5lZ%F>Bauy#8wJy=icik*E)#wfj(2Vd;qHIM9uB)h{7v1JGw>!8?G&Ot zIK_ASz^Sf`fJgubhl29&HI#zOLvNR4z=wmo2ExJ7{BvB~ZJ;&$v=G9bFA_{QyOR?W zjY!gMwe)x+RNj2dWGOVCX=%*3X*L;rQsP$_C-ao?j&!JX^#2xnwG#SNmT80V_*h!J ze*V(0Ygt&ZaR~fd6WHK#;NRc^Jb_p{J3FUHecR4(vF_QaeevkI2n1ibsEH}dqTiL^ zT)9b-$z>PvfoH?dry<+QC8K+7IBDm;5lpwyYe5{cwE4^y&{81EnPooR2ear3ca3Q0 zExXxp!hEV?ps3956v$U6$tKDd%K@zbulPVzaG>H`^Z=N?%NMa1Ws8-F?S`5WS(!ez z-E5eRX2){T9$Q0bjI$kq#+gDIC&O2eq4+XO!wKDU$G!0+g}kGel{3WOCOjAAO`O_H zqZQAYLfxp-+0drd$1*0VftDL*BNsL5Gv|_T^Uuyzo`TUJ9N#}k8M9JO(27L;eBUfS zuy=tRVIU2Q9DV~X5fyx|MY=iBCAWc2;GFi}Kn9l38c~_W3G(%i)(>~#H6hrhJa_xd zmxgvkYri890a=!TeOeBsKyAX^#!7DGP3&AGsN+CAR|U(&73anPQEJHm5Fj&cW?nMzrGXh8wI|(eIt&hcRn7 zs+pkjDL})u#tv_-xgpA{PweD^eD%LzpWTcu60W|GLw<4*@);GM%+{3nbv1Vkwv7Ak zp_RLN#rV!HY+YN6WruR;aw8STOse*qXV0WIiqyPg7sKPP_GjzvTyCt+$(5?bkjE1{ zvz1Qdnz+th)GMGO=2-1m)kep)aMCCe{prwq|3_GlZ*|`(u{>p$!{Q4mzy0#^CrkG_ zs-Kig)sSvxJ_--`??{t;Tjkv;2gNCBGe>j$!aa*+Ia!ZhG10wzMI-U#6IFUSW=`|m z%v6f7qnU{gt5KhhITxR-(-jr9$;}*Fg4w8F>@uyN{CMqn^ERq_iEdD%yfgEF%B>T! z0diGGp1d0D0=T*`n?&BUd@i6Z0lfd1qw>hjTL|kY-)}~0!9Rx|$jP|NxcrrrGkRcA6ki!xS0l*4|U_)3LR_yCLfoc{5rr;fA4$K50>;mk#Bw!G!Z&Bzp*ci9tFe z;WJ<*wD%fBuseb0>pY^!Qf54}b;RSBwK!lxFS@jmG!;Y3VBJ0kmG>Xz`P*~&m3)?7e}Gck~=OuPE8pDv;b0Q!G*=k`&-nm4FrEH`$^c-O?iNYtrgfF3RP_Xqqh` zU#&gKGM%|(xA^7kyLCU$yZa`mFSN;Ukwo#cF13CZg<7e9m;LCMoU>Nz#~yL#LPhXb z2}u%l1hW$IwG^a#8vIeNe^s{IBy6{6Ek_lA-P|&q7(B@Xsq(6F5L2`HjwvFvoBvBI;@(6 zYk@fV(<;5~wibczw(w<}QxhR)dW7Yi3q5XiWyegU<*ZPeRSp;ceA^?~>l)(x&Fb znpjJp3Fm)K{Tppa?`#lAY9i!4G;6bK7&W(Lt2zg{Qh=OAdj+jXk=V>?Mp{^AwUVd@v=zj$V9H()_7-9SVk4fV zjGHuQG=z0O z`f-VBM{Vs^(F=G*+F%}73l^s_OWc%1$RJaWu7@8nw#Ub}hB~>h&gK%U==pQ#f(}uu z(7}paS9ZQG;~dB@-&p+s0dnu3c5f$HRxrm2^2^TGI3T-!msVfSS$T;fRy|AAgB-6` z-Ipp?cqZ4OTDIz0*Vn1tlm4~x+^b7rgL7m!L%S#6Mg5L70HI~if|cK;2iliIiO@z@JtZr#}a;%Iic)6QiO}n9Z`?=&ho8y8sudCU9x_c19HTP z=3UMsd)movH_KZ8jgm*f=uI3lcI6{YcbJi>?^z^hMZiM-tpTr5Kz&J4x)HPOMI7|v zi(;Rm{aJWj(Id=@xBlQvT&1~j=`BN!wjJg{o^CU~{+p&Y>q~QJ!nGYb@5Oz!x&_9C z$xm<84(*`mo0#6~8pxy5?=jd@rgSNosh6&3P&@~bMPhtS!;W*cfQ@SZiL%Z}2ieWrIEN>2;|2wky`C(ATkuVSV8TLiG80QMdGp=_&A&8MB?G zV4ea9@)X_&JNi*+(RLly234htW9)_1vfS*Qv-9d&KWqm_HgDDS=SwM4X23+3S;F)yB*}RT&*?*l6tu;^CjuYaM%ALXu-znD4mV&Q9X|RMY^$cd*{4jyzJRhw zFuH`Dl0K;JUZbd(>-svGtqAX(m_}%{3=rdw0&f0mIvUX794N97gws_wZ z@G&qALSxECM($u2s~=~EU?%e=cR|c>+>R2pMjpEVP}Db7^9VShj{U{0O)>dM^#FTU zejfLjqIlKB*AU;RUOuXIr~Nz3T%Jrf3u ze?Q^RCrB!n4F|= z)w;i}HX@VnwUV*JJ3p#flFlbE>hX+fj8bSnfa}5V6vJ3n;(}QIDLd6`C|`3ET)mJ` zK{}TJZHK>8`H6~DGT^O>MS2ySiF6F%`{E3BB}`I7$z|IN}O z`m)ydTer>*tiJc*32j@hg1XE?TUs1EuBMUq2|;Jx`(LNGOpX0=bCO1=4$u!!5~EGs zm#b&xZBq{u_FGaV_Fc`$qD$pYGg3z}M@o~kM%o$2Tk#ztIhE0r$;8fxzPXjn9lLKz z`o=)=c07ekN}c~ajz1|4e)hV`rh?(ixW`j}{4E3IUBWqPaTCYz6tY_q9*8gMajNwf zy#ijQs2npwcdfXJusO&r;^R9JhRPIsM>If@izCUu#NKz#m@KEO$ZECFp7FUu|HxRu zN!&NeSr)wfl*!&`{$rWwoLOSfQ|#Vgd6^AKh%!pt8halIcITuI_O2IG8a8a(&KbEc z3dN+`#}&Gm-&*qt8I;`zRSe_1rMheAR#h=l3BsxNARe+0rz}NM^xS6hm1_ySU4?zV z5*g0M51pDuQohX`))97LLex=ZwjNa1&}kt9e-)Z|aOKqhKt-iwg8$w{z3=l#JGuzu zqBKc(SN{FE)o!1B)9}4#)u`t$)GW(EoXHiqlzDbRP>w#&%f@#yy^ZbQa#?IVCbUmx zR7)PBVCG7i_}^ZX!zdN}ItSr3kDMk_~AZmEn6X?BDQ%%>jeGIdts#B2*(F7#Fqz zgj;|4I^gN8I6mB?1*Uxm<7X{`@vGc(1(V03emBhgoS~nkD8~W%*0Y^dpgPQ`ICA)h z4sBU!evw|04 z_`<4b?f?pt{2ZLl1$GUuM8a0%7SMj$f14VMPc`b?O_S?@Q6STs3PAN#P+Gq^vNgHxEV< ztk+g%n!Fip{CoCLVR$orVP!OZL2{9^dVX76UKvqM`6l@z?B9i0{G#4G0sHlSsCX1_+b(*s zpgCmQjtvU~&FdRCx(81hk4=PmsyeKbRy$o0)`lLT9j*)AHuP8pldU|qklkU{zq?Tt zAw*RrErsxN=~(oFpDu;n_^6)d=hYl;W)7w7oheJb>UbbeaCs7vg}@5$a#f^&{_ zy)DXmrY6`NXNAr0lW|mrgDulj^zYyd>Y)|tIkjn@?_mGAkn3U6DTs=Mm#9S6@4geG zgh+_X>@pV2W1EMy+s%0}<6DqlZ$iK4nMLW1mVp*n%3ngxovP6A1%53=WuH(;GyzUx zY6ev3M5azprs4%G1=sN9|Mu!*QE(oc3!n;54V~E8Q56MknAKj@L}zl6W3e2&vyM&@jTC9WUToAb1|YePo8~3p!AWL_mO8 zqOWK@*yf@?g7k!&PWC7IJTp9z^IBa`zf0M|D9JN-7`&FjkxAHP935EvGhQVv-+j36 zsXm1NHnSDS7SWL$Yu=Z!98Z)yu$aCVW8B9sy!4+pdqL4JD6GH3XATSvxK~S%6Z}SX z(YP9<_%+0-)gtGKUFVHt>l@Q&l9mWa9S>1=%rk9w#0p|I;2Zhb&Lleg3IvWA@^-An z{4sHT$9m~#hRGC|zpdakV`v;xUF?|7B7VAj?ZfB-8tp5Iv}VMsJgmv-M|H9Iib2d1<7U6qgD4V5$&u9J?4S zT->OslTgPlWSELb5z0#unf)%}nSVmP2%@|!8xq}AXYEaj^Md}QAfi`&minImmi|2x3U3L|HRWaLb~>kgFGd*+bNhoIj<@&C6%8e2dxIKApX#53NeUN%g5$}~ zrWMp$SFDGw_}<^JL^F(0GY$4AkAn+UwkZyptk*5@VU^ges1`0(Bpm~_#>zrG7(g4fCWGW z58W6dga!@B0`!nyaU2x|nuv@7J^0rv`sV*Ek2j=Hq9HG6*6wrn-DR?XB*_aMs(B;)obH(^KBMvlVoD4AT_0OV*ejDcow0XZG zWC97Q>O%4gntNx zW=%-~w37d!Fw|ud2Rf39gYn Date: Mon, 23 Mar 2020 09:57:26 +0100 Subject: [PATCH 64/91] CI: reduce test flakiness on travis (#263) * Test: fix flaky test caused by LS bug identified and expected to be fixed in LS 7.6.2 ... for reference see elastic/logstash#11694 * Test: wait longer (due CI being a bit slow) * Refactor: actually require rspec/wait * Test: sleep a bit to avoid delete to happen * Chore: let's not leave gradle daemon around --- Rakefile | 2 +- spec/filewatch/rotate_spec.rb | 3 ++- spec/filewatch/spec_helper.rb | 2 +- spec/inputs/file_read_spec.rb | 8 ++++---- spec/inputs/file_tail_spec.rb | 4 ++-- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Rakefile b/Rakefile index e1595e1..ca92ecf 100644 --- a/Rakefile +++ b/Rakefile @@ -8,6 +8,6 @@ require "logstash/devutils/rake" desc "Compile and put filewatch jar into lib/jars" task :vendor do - exit(1) unless system './gradlew clean jar' + exit(1) unless system './gradlew --no-daemon clean jar' puts "-------------------> built filewatch jar via rake" end diff --git a/spec/filewatch/rotate_spec.rb b/spec/filewatch/rotate_spec.rb index cf85704..f4e5979 100644 --- a/spec/filewatch/rotate_spec.rb +++ b/spec/filewatch/rotate_spec.rb @@ -73,7 +73,8 @@ module FileWatch FileUtils.mv(directory.join("1.logtmp").to_path, file1_path) end .then("wait for expectation") do - wait(2).for{listener1.calls}.to eq([:open, :accept, :accept, :accept]) + sleep(0.25) # if ENV['CI'] + wait(2).for { listener1.calls }.to eq([:open, :accept, :accept, :accept]) end .then("quit") do tailing.quit diff --git a/spec/filewatch/spec_helper.rb b/spec/filewatch/spec_helper.rb index 0490536..f2d32d7 100644 --- a/spec/filewatch/spec_helper.rb +++ b/spec/filewatch/spec_helper.rb @@ -1,6 +1,6 @@ # encoding: utf-8 require "rspec_sequencing" -# require 'rspec/wait' +require 'rspec/wait' require "logstash/devutils/rspec/spec_helper" require "concurrent" require "timecop" diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index 9b38dc6..298b5f2 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -70,7 +70,7 @@ end events = input(conf) do |pipeline, queue| - wait(0.5).for{IO.read(log_completed_path)}.to match(/A\.log/) + wait(0.75).for { IO.read(log_completed_path) }.to match(/A\.log/) 2.times.collect { queue.pop } end expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world") @@ -137,7 +137,7 @@ CONFIG events = input(conf) do |pipeline, queue| - wait(0.5).for{IO.read(log_completed_path)}.to match(/#{file_path.to_s}/) + wait(0.75).for { IO.read(log_completed_path) }.to match(/#{file_path.to_s}/) 2.times.collect { queue.pop } end @@ -171,7 +171,7 @@ CONFIG events = input(conf) do |pipeline, queue| - wait(0.5).for{IO.read(log_completed_path)}.to match(/uncompressed\.log/) + wait(0.75).for{ IO.read(log_completed_path) }.to match(/uncompressed\.log/) 2.times.collect { queue.pop } end @@ -205,7 +205,7 @@ CONFIG events = input(conf) do |pipeline, queue| - wait(0.5).for{IO.read(log_completed_path).scan(/compressed\.log\.gz(ip)?/).size}.to eq(2) + wait(0.75).for { IO.read(log_completed_path).scan(/compressed\.log\.gz(ip)?/).size }.to eq(2) 4.times.collect { queue.pop } end diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index 367ce07..7aefc69 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -67,7 +67,7 @@ path => "#{path_path}" start_position => "beginning" sincedb_path => "#{sincedb_path}" - "file_sort_by" => "path" + file_sort_by => "path" delimiter => "#{TEST_FILE_DELIMITER}" } } @@ -176,7 +176,7 @@ context "when sincedb_path is a directory" do let(:name) { "E" } subject { LogStash::Inputs::File.new("path" => path_path, "sincedb_path" => directory) } - + after :each do FileUtils.rm_rf(sincedb_path) end From 5cd1171a39e274f8958a15c1132457d998bdda7d Mon Sep 17 00:00:00 2001 From: Kristof Feys Date: Wed, 1 Apr 2020 15:46:10 +0200 Subject: [PATCH 65/91] Typo in documentation (#235) * Typo in documentation * Update index.asciidoc --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index f4be83c..9df687b 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -360,7 +360,7 @@ If "read" is specified then the following other settings are ignored: . `start_position` (files are always read from the beginning) . `close_older` (files are automatically 'closed' when EOF is reached) -If "read" is specified then the following settings are heeded: +If "read" is specified then the following settings can be used: . `ignore_older` (older files are not processed) . `file_completed_action` (what action should be taken when the file is processed) From e4b5cedb7ea1175ee933d85e7b9b537db7b9612b Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 25 Mar 2020 18:45:29 +0100 Subject: [PATCH 66/91] Added settings 'check_archive_validity' to optionally enable integrity check on archives, close #261 Fixes #265 --- CHANGELOG.md | 4 + docs/index.asciidoc | 15 ++++ .../read_mode/handlers/read_zip_file.rb | 83 ++++++++++++------- lib/filewatch/settings.rb | 2 + lib/logstash/inputs/file.rb | 6 ++ logstash-input-file.gemspec | 2 +- spec/helpers/spec_helper.rb | 7 ++ spec/inputs/file_read_spec.rb | 31 +++++++ 8 files changed, 121 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88c8078..d882b68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 4.1.17 + - Added configuration setting `check_archive_validity` settings to enable gzipped files verification, + issue [#261](https://github.com/logstash-plugins/logstash-input-file/issues/261) + ## 4.1.16 - Added configuration setting exit_after_read to read to EOF and terminate the input [#240](https://github.com/logstash-plugins/logstash-input-file/pull/240) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 9df687b..6cd2436 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -164,6 +164,7 @@ see <> for the details [cols="<,<,<",options="header",] |======================================================================= |Setting |Input type|Required +| <> |<>|No | <> |<> or <>|No | <> |<>|No | <> |<>|No @@ -191,6 +192,20 @@ input plugins.   +[id="plugins-{type}s-{plugin}-check_archive_validity"] +===== `check_archive_validity` + + * Value type is <> + * The default is `false`. + +When set to `true`, this setting verifies that a compressed file is valid before +processing it. There are two passes through the file--one pass to +verify that the file is valid, and another pass to process the file. + +Validating a compressed file requires more processing time, but can prevent a +corrupt archive from causing looping. + + [id="plugins-{type}s-{plugin}-close_older"] ===== `close_older` diff --git a/lib/filewatch/read_mode/handlers/read_zip_file.rb b/lib/filewatch/read_mode/handlers/read_zip_file.rb index 3a0f764..9a86436 100644 --- a/lib/filewatch/read_mode/handlers/read_zip_file.rb +++ b/lib/filewatch/read_mode/handlers/read_zip_file.rb @@ -18,34 +18,40 @@ def handle_specifically(watched_file) # fast forward through the lines until we reach unseen content? # meaning that we can quit in the middle of a zip file key = watched_file.sincedb_key - begin - file_stream = FileInputStream.new(watched_file.path) - gzip_stream = GZIPInputStream.new(file_stream) - decoder = InputStreamReader.new(gzip_stream, "UTF-8") - buffered = BufferedReader.new(decoder) - while (line = buffered.readLine(false)) - watched_file.listener.accept(line) - # can't quit, if we did then we would incorrectly write a 'completed' sincedb entry - # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading) - # should we track lines read in the sincedb and - # fast forward through the lines until we reach unseen content? - # meaning that we can quit in the middle of a zip file - end - watched_file.listener.eof - rescue ZipException => e - logger.error("Cannot decompress the gzip file at path: #{watched_file.path}") - watched_file.listener.error - else - sincedb_collection.store_last_read(key, watched_file.last_stat_size) - sincedb_collection.request_disk_flush - watched_file.listener.deleted + + if @settings.check_archive_validity && corrupted?(watched_file) watched_file.unwatch - ensure - # rescue each close individually so all close attempts are tried - close_and_ignore_ioexception(buffered) unless buffered.nil? - close_and_ignore_ioexception(decoder) unless decoder.nil? - close_and_ignore_ioexception(gzip_stream) unless gzip_stream.nil? - close_and_ignore_ioexception(file_stream) unless file_stream.nil? + else + begin + file_stream = FileInputStream.new(watched_file.path) + gzip_stream = GZIPInputStream.new(file_stream) + decoder = InputStreamReader.new(gzip_stream, "UTF-8") + buffered = BufferedReader.new(decoder) + while (line = buffered.readLine(false)) + watched_file.listener.accept(line) + # can't quit, if we did then we would incorrectly write a 'completed' sincedb entry + # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading) + # should we track lines read in the sincedb and + # fast forward through the lines until we reach unseen content? + # meaning that we can quit in the middle of a zip file + end + watched_file.listener.eof + rescue ZipException => e + logger.error("Cannot decompress the gzip file at path: #{watched_file.path}", :exception => e.class, + :message => e.message, :backtrace => e.backtrace) + watched_file.listener.error + else + sincedb_collection.store_last_read(key, watched_file.last_stat_size) + sincedb_collection.request_disk_flush + watched_file.listener.deleted + watched_file.unwatch + ensure + # rescue each close individually so all close attempts are tried + close_and_ignore_ioexception(buffered) unless buffered.nil? + close_and_ignore_ioexception(decoder) unless decoder.nil? + close_and_ignore_ioexception(gzip_stream) unless gzip_stream.nil? + close_and_ignore_ioexception(file_stream) unless file_stream.nil? + end end sincedb_collection.clear_watched_file(key) end @@ -56,7 +62,28 @@ def close_and_ignore_ioexception(closeable) begin closeable.close rescue Exception => e # IOException can be thrown by any of the Java classes that implement the Closable interface. - logger.warn("Ignoring an IOException when closing an instance of #{closeable.class.name}", "exception" => e) + logger.warn("Ignoring an IOException when closing an instance of #{closeable.class.name}", + :exception => e.class, :message => e.message, :backtrace => e.backtrace) + end + end + + def corrupted?(watched_file) + begin + file_stream = FileInputStream.new(watched_file.path) + gzip_stream = GZIPInputStream.new(file_stream) + buffer = Java::byte[8192].new + start = Time.new + until gzip_stream.read(buffer) == -1 + end + return false + rescue ZipException => e + duration = Time.now - start + logger.warn("Detected corrupted archive #{watched_file.path} file won't be processed", :message => e.message, + :duration => duration.round(3)) + return true + ensure + close_and_ignore_ioexception(gzip_stream) unless gzip_stream.nil? + close_and_ignore_ioexception(file_stream) unless file_stream.nil? end end end diff --git a/lib/filewatch/settings.rb b/lib/filewatch/settings.rb index 311c112..e62efce 100644 --- a/lib/filewatch/settings.rb +++ b/lib/filewatch/settings.rb @@ -9,6 +9,7 @@ class Settings attr_reader :sincedb_path, :sincedb_write_interval, :sincedb_expiry_duration attr_reader :file_sort_by, :file_sort_direction attr_reader :exit_after_read + attr_reader :check_archive_validity def self.from_options(opts) new.add_options(opts) @@ -52,6 +53,7 @@ def add_options(opts) @file_sort_by = @opts[:file_sort_by] @file_sort_direction = @opts[:file_sort_direction] @exit_after_read = @opts[:exit_after_read] + @check_archive_validity = @opts[:check_archive_validity] self end diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index abcd146..b83646e 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -227,6 +227,11 @@ class File < LogStash::Inputs::Base # Sincedb still works, if you run LS once again after doing some changes - only new values will be read config :exit_after_read, :validate => :boolean, :default => false + # Before start read a compressed file, checks for its validity. + # This request a full read of the archive, so potentially could cost time. + # If not specified to true, and the file is corrupted, could end in cyclic processing of the broken file. + config :check_archive_validity, :validate => :boolean, :default => false + public class << self @@ -266,6 +271,7 @@ def register :file_sort_by => @file_sort_by, :file_sort_direction => @file_sort_direction, :exit_after_read => @exit_after_read, + :check_archive_validity => @check_archive_validity, } @completed_file_handlers = [] diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 2b64bff..1859d57 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.16' + s.version = '4.1.17' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/helpers/spec_helper.rb b/spec/helpers/spec_helper.rb index 70bfd8e..6f54305 100644 --- a/spec/helpers/spec_helper.rb +++ b/spec/helpers/spec_helper.rb @@ -16,6 +16,13 @@ def self.make_fixture_current(path, time = Time.now) ::File.utime(time, time, path) end + def self.corrupt_gzip(file_path) + f = File.open(file_path, "w") + f.seek(12) + f.puts 'corrupting_string' + f.close() + end + class TracerBase def initialize @tracer = Concurrent::Array.new diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index 298b5f2..2af41d2 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -214,6 +214,37 @@ expect(events[2].get("message")).to start_with("2010-03-12 23:51") expect(events[3].get("message")).to start_with("2010-03-12 23:51") end + + it "the corrupted file is untouched" do + directory = Stud::Temporary.directory + file_path = fixture_dir.join('compressed.log.gz') + corrupted_file_path = ::File.join(directory, 'corrupted.gz') + FileUtils.cp(file_path, corrupted_file_path) + + FileInput.corrupt_gzip(corrupted_file_path) + + log_completed_path = ::File.join(directory, "C_completed.txt") + f = File.new(log_completed_path, "w") + f.close() + + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{corrupted_file_path}" + mode => "read" + file_completed_action => "log_and_delete" + file_completed_log_path => "#{log_completed_path}" + check_archive_validity => true + } + } + CONFIG + + events = input(conf) do |pipeline, queue| + wait(1) + expect(IO.read(log_completed_path)).to be_empty + end + end end end end From a0384c08f89d661c2420fb35a737b438030a1bcc Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Fri, 3 Apr 2020 10:52:23 -0400 Subject: [PATCH 67/91] [DOC] Doc improvements (#266) * Doc improvements to mode setting * Update changelog with PR number * Rebase against master --- CHANGELOG.md | 7 +++++-- docs/index.asciidoc | 15 ++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d882b68..44e9c2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ ## 4.1.17 - - Added configuration setting `check_archive_validity` settings to enable gzipped files verification, - issue [#261](https://github.com/logstash-plugins/logstash-input-file/issues/261) + - Added configuration setting `check_archive_validity` settings to enable + gzipped files verification, issue + [#261](https://github.com/logstash-plugins/logstash-input-file/issues/261) + - [DOC] Added clarification for settings available with `read` mode [#235](https://github.com/logstash-plugins/logstash-input-file/pull/235) + - [DOC] Rearranged text and fixed formatting for `mode` setting [266](https://github.com/logstash-plugins/logstash-input-file/pull/266) ## 4.1.16 - Added configuration setting exit_after_read to read to EOF and terminate diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 6cd2436..d0926ca 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -370,16 +370,17 @@ A default of 4095 is set in internally. What mode do you want the file input to operate in. Tail a few files or read many content-complete files. Read mode now supports gzip file processing. -If "read" is specified then the following other settings are ignored: -. `start_position` (files are always read from the beginning) -. `close_older` (files are automatically 'closed' when EOF is reached) +If `read` is specified, these settings can be used: -If "read" is specified then the following settings can be used: +* `ignore_older` (older files are not processed) +* `file_completed_action` (what action should be taken when the file is processed) +* `file_completed_log_path` (which file should the completed file path be logged to) -. `ignore_older` (older files are not processed) -. `file_completed_action` (what action should be taken when the file is processed) -. `file_completed_log_path` (which file should the completed file path be logged to) +If `read` is specified, these settings are ignored: + +* `start_position` (files are always read from the beginning) +* `close_older` (files are automatically 'closed' when EOF is reached) [id="plugins-{type}s-{plugin}-path"] ===== `path` From aec0be9b41ef5d0120d78dfcc749c97fb879c564 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 29 Apr 2020 20:48:51 +0200 Subject: [PATCH 68/91] Fix: release watched files on completion (in read-mode) (#271) not doing so leads to a steady increase in watched collection's size over time (esp. in use-cases where user is pulling in new files). the left-over file is never to be processed again - it's being deleted anyway using the completion handler. --- CHANGELOG.md | 3 + lib/filewatch/read_mode/handlers/read_file.rb | 3 + lib/filewatch/watch.rb | 2 +- lib/filewatch/watched_files_collection.rb | 14 ++-- .../inputs/delete_completed_file_handler.rb | 5 ++ lib/logstash/inputs/file.rb | 34 ++++++--- logstash-input-file.gemspec | 2 +- .../watched_files_collection_spec.rb | 4 +- spec/helpers/spec_helper.rb | 1 + spec/inputs/file_read_spec.rb | 75 +++++++++++++++++++ 10 files changed, 124 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44e9c2b..81f3843 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.1.18 + - Fix: release watched files on completion (in read-mode) [#271](https://github.com/logstash-plugins/logstash-input-file/pull/271) + ## 4.1.17 - Added configuration setting `check_archive_validity` settings to enable gzipped files verification, issue diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index c85f6a3..d14afcc 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -22,6 +22,9 @@ def handle_specifically(watched_file) sincedb_collection.reading_completed(key) sincedb_collection.clear_watched_file(key) watched_file.listener.deleted + # NOTE: on top of un-watching we should also remove from the watched files collection + # if the file is getting deleted (on completion), that part currently resides in + # DeleteCompletedFileHandler - triggered above using `watched_file.listener.deleted` watched_file.unwatch end end diff --git a/lib/filewatch/watch.rb b/lib/filewatch/watch.rb index 3c88dd1..6cd4f9d 100644 --- a/lib/filewatch/watch.rb +++ b/lib/filewatch/watch.rb @@ -67,7 +67,7 @@ def iterate_on_state watched_files = @watched_files_collection.values @processor.process_all_states(watched_files) ensure - @watched_files_collection.delete(@processor.deletable_filepaths) + @watched_files_collection.remove_paths(@processor.deletable_filepaths) @processor.deletable_filepaths.clear end end # def each diff --git a/lib/filewatch/watched_files_collection.rb b/lib/filewatch/watched_files_collection.rb index 475d836..153cd48 100644 --- a/lib/filewatch/watched_files_collection.rb +++ b/lib/filewatch/watched_files_collection.rb @@ -15,13 +15,17 @@ def add(watched_file) @sort_method.call end - def delete(paths) - Array(paths).each do |f| - index = @pointers.delete(f) - @files.delete_at(index) - refresh_pointers + def remove_paths(paths) + removed_files = Array(paths).map do |path| + index = @pointers.delete(path) + if index + watched_file = @files.delete_at(index) + refresh_pointers + watched_file + end end @sort_method.call + removed_files end def close_all diff --git a/lib/logstash/inputs/delete_completed_file_handler.rb b/lib/logstash/inputs/delete_completed_file_handler.rb index c6a3e91..6f8e8d0 100644 --- a/lib/logstash/inputs/delete_completed_file_handler.rb +++ b/lib/logstash/inputs/delete_completed_file_handler.rb @@ -2,8 +2,13 @@ module LogStash module Inputs class DeleteCompletedFileHandler + def initialize(watch) + @watch = watch + end + def handle(path) Pathname.new(path).unlink rescue nil + @watch.watched_files_collection.remove_paths([path]) end end end end diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index b83646e..0f11807 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -6,6 +6,7 @@ require "pathname" require "socket" # for Socket.gethostname require "fileutils" +require "concurrent/atomic/atomic_reference" require_relative "file/patch" require_relative "file_listener" @@ -247,6 +248,9 @@ def validate_value(value, validator) end end + # @private used in specs + attr_reader :watcher + def register require "addressable/uri" require "digest/md5" @@ -274,8 +278,6 @@ def register :check_archive_validity => @check_archive_validity, } - @completed_file_handlers = [] - @path.each do |path| if Pathname.new(path).relative? raise ArgumentError.new("File paths must be absolute, relative path specified: #{path}") @@ -319,15 +321,10 @@ def register @watcher_class = FileWatch::ObservingTail else @watcher_class = FileWatch::ObservingRead - if @file_completed_action.include?('log') - @completed_file_handlers << LogCompletedFileHandler.new(@file_completed_log_path) - end - if @file_completed_action.include?('delete') - @completed_file_handlers << DeleteCompletedFileHandler.new - end end @codec = LogStash::Codecs::IdentityMapCodec.new(@codec) @completely_stopped = Concurrent::AtomicBoolean.new + @queue = Concurrent::AtomicReference.new end # def register def completely_stopped? @@ -344,13 +341,25 @@ def start_processing # if the pipeline restarts this input, # make sure previous files are closed stop + @watcher = @watcher_class.new(@filewatch_config) + + @completed_file_handlers = [] + if read_mode? + if @file_completed_action.include?('log') + @completed_file_handlers << LogCompletedFileHandler.new(@file_completed_log_path) + end + if @file_completed_action.include?('delete') + @completed_file_handlers << DeleteCompletedFileHandler.new(@watcher.watch) + end + end + @path.each { |path| @watcher.watch_this(path) } end def run(queue) start_processing - @queue = queue + @queue.set queue @watcher.subscribe(self) # halts here until quit is called # last action of the subscribe call is to write the sincedb exit_flush @@ -361,7 +370,7 @@ def post_process_this(event) event.set("[@metadata][host]", @host) event.set("host", @host) unless event.include?("host") decorate(event) - @queue << event + @queue.get << event end def handle_deletable_path(path) @@ -382,6 +391,11 @@ def stop end end + # @private used in specs + def queue + @queue.get + end + private def build_sincedb_base_from_settings(settings) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 1859d57..d6186f2 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.17' + s.version = '4.1.18' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filewatch/watched_files_collection_spec.rb b/spec/filewatch/watched_files_collection_spec.rb index 484aa89..50c2b59 100644 --- a/spec/filewatch/watched_files_collection_spec.rb +++ b/spec/filewatch/watched_files_collection_spec.rb @@ -85,9 +85,9 @@ module FileWatch collection.add(wf3) expect(collection.keys).to eq([filepath1, filepath2, filepath3]) - collection.delete([filepath2,filepath3]) + collection.remove_paths([filepath2,filepath3]) expect(collection.keys).to eq([filepath1]) - + expect(collection.values.size).to eq 1 end end diff --git a/spec/helpers/spec_helper.rb b/spec/helpers/spec_helper.rb index 6f54305..b2f0ddd 100644 --- a/spec/helpers/spec_helper.rb +++ b/spec/helpers/spec_helper.rb @@ -1,6 +1,7 @@ # encoding: utf-8 require "logstash/devutils/rspec/spec_helper" +require "rspec/wait" require "rspec_sequencing" module FileInput diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index 2af41d2..14fc933 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -247,4 +247,79 @@ end end end + + let(:temp_directory) { Stud::Temporary.directory } + let(:interval) { 0.1 } + let(:options) do + { + 'mode' => "read", + 'path' => "#{temp_directory}/*", + 'stat_interval' => interval, + 'discover_interval' => interval, + 'sincedb_path' => "#{temp_directory}/.sincedb", + 'sincedb_write_interval' => interval + } + end + + let(:queue) { Queue.new } + let(:plugin) { LogStash::Inputs::File.new(options) } + + describe 'delete on complete' do + + let(:options) do + super.merge({ 'file_completed_action' => "delete", 'exit_after_read' => false }) + end + + let(:sample_file) { File.join(temp_directory, "sample.log") } + + before do + plugin.register + @run_thread = Thread.new(plugin) do |plugin| + Thread.current.abort_on_exception = true + plugin.run queue + end + + File.open(sample_file, 'w') { |fd| fd.write("sample-content\n") } + + wait_for_start_processing(@run_thread) + end + + after { plugin.stop } + + it 'processes a file' do + wait_for_file_removal(sample_file) # watched discovery + + expect( plugin.queue.size ).to eql 1 + event = plugin.queue.pop + expect( event.get('message') ).to eql 'sample-content' + end + + it 'removes watched file from collection' do + wait_for_file_removal(sample_file) # watched discovery + sleep(0.25) # give CI some space to execute the removal + # TODO shouldn't be necessary once WatchedFileCollection does proper locking + watched_files = plugin.watcher.watch.watched_files_collection + expect( watched_files ).to be_empty + end + + private + + def wait_for_start_processing(run_thread, timeout: 1.0) + begin + Timeout.timeout(timeout) do + sleep(0.01) while run_thread.status != 'sleep' + sleep(timeout) unless plugin.queue + end + rescue Timeout::Error + raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue + else + raise "plugin did not start processing" unless plugin.queue + end + end + + def wait_for_file_removal(path, timeout: 3 * interval) + wait(timeout).for { File.exist?(path) }.to be_falsey + end + + end end From 15d3eba2dbd723243134affea3eb09888a7b3240 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 6 May 2020 13:46:41 +0200 Subject: [PATCH 69/91] Fix: watched files performance with huge filesets (#268) Plugin tracks all discovered files in a map-like collection (`FileWatch::WatchedFilesCollection`). The collection design was to re-sort all elements every time the collection gets updated ... this obviously does not scale with 10000s of watched files. Starting up LS with 60k files in a watched directory, locally, takes **\~ an hour before these changes**. **After these changes** for the plugin to start processing files it takes **~2 seconds**. Since the collection is known to have a noticeable (memory) footprint, there's changes towards reducing intermediate garbage - which also contributed to the decision to move the collection to native. Notable changes to tracking watched files with `FileWatch::WatchedFilesCollection`: - predictable `log(size)` modification (and access) times - implementation has clear locking semantics, previously operations weren't 100% atomic - hopefully, cleaner API as the collection re-resembles a *WatchedFile -> String (path)* map - the collection now needs an explicit update whenever a file changes (on re-stat), previous implicitness of picking up changes might have been the reasoning behind re-sorts (could be decoupled further by making `WatchedFile` immutable but there's other state anyway) --- Apart from the necessary changes to resolve the performance bottleneck, there's also a quick review of (trace) logging - annoying to not only see partial traces even in debug/trace levels. Few unused methods and instance variables were removed for a better code reading experience. --- CHANGELOG.md | 4 + lib/filewatch/discoverer.rb | 17 +- lib/filewatch/observing_base.rb | 3 +- lib/filewatch/processor.rb | 55 ++++ lib/filewatch/read_mode/handlers/base.rb | 14 +- lib/filewatch/read_mode/handlers/read_file.rb | 25 +- .../read_mode/handlers/read_zip_file.rb | 14 +- lib/filewatch/read_mode/processor.rb | 58 ++--- lib/filewatch/sincedb_collection.rb | 20 +- lib/filewatch/stat/generic.rb | 21 +- lib/filewatch/stat/windows_path.rb | 16 +- lib/filewatch/tail_mode/handlers/delete.rb | 6 +- lib/filewatch/tail_mode/processor.rb | 101 ++++---- lib/filewatch/watch.rb | 23 +- lib/filewatch/watched_file.rb | 39 +-- lib/filewatch/watched_files_collection.rb | 89 +------ lib/logstash/inputs/file.rb | 3 +- logstash-input-file.gemspec | 3 +- spec/filewatch/reading_spec.rb | 69 ++++- spec/filewatch/settings_spec.rb | 3 + spec/filewatch/spec_helper.rb | 26 +- spec/filewatch/tailing_spec.rb | 26 +- spec/filewatch/watched_file_spec.rb | 30 +++ .../watched_files_collection_spec.rb | 70 +++++- .../filewatch/JrubyFileWatchLibrary.java | 1 + .../filewatch/WatchedFilesCollection.java | 235 ++++++++++++++++++ 26 files changed, 667 insertions(+), 304 deletions(-) create mode 100644 lib/filewatch/processor.rb create mode 100644 src/main/java/org/logstash/filewatch/WatchedFilesCollection.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 81f3843..8129948 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 4.2.0 + - Fix: watched files performance with huge filesets [#268](https://github.com/logstash-plugins/logstash-input-file/pull/268) + - Updated logging to include full traces in debug (and trace) levels + ## 4.1.18 - Fix: release watched files on completion (in read-mode) [#271](https://github.com/logstash-plugins/logstash-input-file/pull/271) diff --git a/lib/filewatch/discoverer.rb b/lib/filewatch/discoverer.rb index 0b9d889..1072323 100644 --- a/lib/filewatch/discoverer.rb +++ b/lib/filewatch/discoverer.rb @@ -9,6 +9,8 @@ class Discoverer # associated with a sincedb entry if one can be found include LogStash::Util::Loggable + attr_reader :watched_files_collection + def initialize(watched_files_collection, sincedb_collection, settings) @watching = Concurrent::Array.new @exclude = Concurrent::Array.new @@ -37,8 +39,7 @@ def can_exclude?(watched_file, new_discovery) @exclude.each do |pattern| if watched_file.pathname.basename.fnmatch?(pattern) if new_discovery - logger.trace("Discoverer can_exclude?: #{watched_file.path}: skipping " + - "because it matches exclude #{pattern}") + logger.trace("skipping file because it matches exclude", :path => watched_file.path, :pattern => pattern) end watched_file.unwatch return true @@ -56,13 +57,13 @@ def discover_files_ongoing(path) end def discover_any_files(path, ongoing) - fileset = Dir.glob(path).select{|f| File.file?(f)} - logger.trace("discover_files", "count" => fileset.size) + fileset = Dir.glob(path).select { |f| File.file?(f) } + logger.trace("discover_files", :count => fileset.size) fileset.each do |file| - pathname = Pathname.new(file) new_discovery = false - watched_file = @watched_files_collection.watched_file_by_path(file) + watched_file = @watched_files_collection.get(file) if watched_file.nil? + pathname = Pathname.new(file) begin path_stat = PathStatClass.new(pathname) rescue Errno::ENOENT @@ -74,7 +75,7 @@ def discover_any_files(path, ongoing) # if it already unwatched or its excluded then we can skip next if watched_file.unwatched? || can_exclude?(watched_file, new_discovery) - logger.trace("discover_files handling:", "new discovery"=> new_discovery, "watched_file details" => watched_file.details) + logger.trace? && logger.trace("handling:", :new_discovery => new_discovery, :watched_file => watched_file.details) if new_discovery watched_file.initial_completed if ongoing @@ -86,7 +87,7 @@ def discover_any_files(path, ongoing) # associated with a different watched_file if @sincedb_collection.associate(watched_file) if watched_file.file_ignorable? - logger.trace("Discoverer discover_files: #{file}: skipping because it was last modified more than #{@settings.ignore_older} seconds ago") + logger.trace("skipping file because it was last modified more than #{@settings.ignore_older} seconds ago", :path => file) # on discovery ignorable watched_files are put into the ignored state and that # updates the size from the internal stat # so the existing contents are not read. diff --git a/lib/filewatch/observing_base.rb b/lib/filewatch/observing_base.rb index 7f9be79..ecdf7f7 100644 --- a/lib/filewatch/observing_base.rb +++ b/lib/filewatch/observing_base.rb @@ -62,8 +62,7 @@ def build_watch_and_dependencies @sincedb_collection = SincedbCollection.new(@settings) @sincedb_collection.open discoverer = Discoverer.new(watched_files_collection, @sincedb_collection, @settings) - @watch = Watch.new(discoverer, watched_files_collection, @settings) - @watch.add_processor build_specific_processor(@settings) + @watch = Watch.new(discoverer, build_specific_processor(@settings), @settings) end def watch_this(path) diff --git a/lib/filewatch/processor.rb b/lib/filewatch/processor.rb new file mode 100644 index 0000000..350fa87 --- /dev/null +++ b/lib/filewatch/processor.rb @@ -0,0 +1,55 @@ +# encoding: utf-8 +require "logstash/util/loggable" +require 'concurrent/atomic/atomic_reference' + +module FileWatch + class Processor + include LogStash::Util::Loggable + + attr_reader :watch + + def initialize(settings) + @settings = settings + @deletable_paths = Concurrent::AtomicReference.new [] + end + + def add_watch(watch) + @watch = watch + self + end + + def clear_deletable_paths + @deletable_paths.get_and_set [] + end + + def add_deletable_path(path) + @deletable_paths.get << path + end + + def restat(watched_file) + changed = watched_file.restat! + if changed + # the collection (when sorted by modified_at) needs to re-sort every time watched-file is modified, + # we can perform these update operation while processing files (stat interval) instead of having to + # re-sort the whole collection every time an entry is accessed + @watch.watched_files_collection.update(watched_file) + end + end + + private + + def error_details(error, watched_file) + details = { :path => watched_file.path, + :exception => error.class, + :message => error.message, + :backtrace => error.backtrace } + if logger.debug? + details[:file] = watched_file + else + details[:backtrace] = details[:backtrace].take(8) if details[:backtrace] + end + details + end + + end +end diff --git a/lib/filewatch/read_mode/handlers/base.rb b/lib/filewatch/read_mode/handlers/base.rb index ceac0b7..2bce4d0 100644 --- a/lib/filewatch/read_mode/handlers/base.rb +++ b/lib/filewatch/read_mode/handlers/base.rb @@ -34,19 +34,21 @@ def handle_specifically(watched_file) def open_file(watched_file) return true if watched_file.file_open? - logger.trace("opening #{watched_file.path}") + logger.trace? && logger.trace("opening", :path => watched_file.path) begin watched_file.open - rescue + rescue => e # don't emit this message too often. if a file that we can't # read is changing a lot, we'll try to open it more often, and spam the logs. now = Time.now.to_i logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL - logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}") + backtrace = e.backtrace + backtrace = backtrace.take(3) if backtrace && !logger.debug? + logger.warn("failed to open", :path => watched_file.path, :exception => e.class, :message => e.message, :backtrace => backtrace) watched_file.last_open_warning_at = now else - logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}") + logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message) end watched_file.watch # set it back to watch so we can try it again end @@ -75,7 +77,7 @@ def add_or_update_sincedb_collection(watched_file) watched_file.update_bytes_read(sincedb_value.position) else sincedb_value.set_watched_file(watched_file) - logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details) + logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details) watched_file.rotate_from(existing_watched_file) end @@ -92,7 +94,7 @@ def update_existing_sincedb_collection_value(watched_file, sincedb_value) def add_new_value_sincedb_collection(watched_file) sincedb_value = SincedbValue.new(0) sincedb_value.set_watched_file(watched_file) - logger.trace("add_new_value_sincedb_collection: #{watched_file.path}", "position" => sincedb_value.position) + logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position) sincedb_collection.set(watched_file.sincedb_key, sincedb_value) end end diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index d14afcc..0717a78 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -31,7 +31,7 @@ def handle_specifically(watched_file) end def controlled_read(watched_file, loop_control) - logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename) + logger.trace? && logger.trace("reading...", :filename => watched_file.filename, :iterations => loop_control.count, :amount => loop_control.size) loop_control.count.times do break if quit? begin @@ -43,22 +43,35 @@ def controlled_read(watched_file, loop_control) delta = line.bytesize + @settings.delimiter_byte_size sincedb_collection.increment(watched_file.sincedb_key, delta) end - rescue EOFError - logger.error("controlled_read: eof error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) + rescue EOFError => e + log_error("controlled_read: eof error reading file", watched_file, e) loop_control.flag_read_error break - rescue Errno::EWOULDBLOCK, Errno::EINTR - logger.error("controlled_read: block or interrupt error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) + rescue Errno::EWOULDBLOCK, Errno::EINTR => e + log_error("controlled_read: block or interrupt error reading file", watched_file, e) watched_file.listener.error loop_control.flag_read_error break rescue => e - logger.error("controlled_read: general error reading file", "path" => watched_file.path, "error" => e.inspect, "backtrace" => e.backtrace.take(8)) + log_error("controlled_read: general error reading file", watched_file, e) watched_file.listener.error loop_control.flag_read_error break end end end + + def log_error(msg, watched_file, error) + details = { :path => watched_file.path, + :exception => error.class, + :message => error.message, + :backtrace => error.backtrace } + if logger.debug? + details[:file] = watched_file + else + details[:backtrace] = details[:backtrace].take(8) if details[:backtrace] + end + logger.error(msg, details) + end end end end end diff --git a/lib/filewatch/read_mode/handlers/read_zip_file.rb b/lib/filewatch/read_mode/handlers/read_zip_file.rb index 9a86436..b9d8d2d 100644 --- a/lib/filewatch/read_mode/handlers/read_zip_file.rb +++ b/lib/filewatch/read_mode/handlers/read_zip_file.rb @@ -1,13 +1,15 @@ # encoding: utf-8 require 'java' -java_import java.io.InputStream -java_import java.io.InputStreamReader -java_import java.io.FileInputStream -java_import java.io.BufferedReader -java_import java.util.zip.GZIPInputStream -java_import java.util.zip.ZipException module FileWatch module ReadMode module Handlers + + java_import java.io.InputStream + java_import java.io.InputStreamReader + java_import java.io.FileInputStream + java_import java.io.BufferedReader + java_import java.util.zip.GZIPInputStream + java_import java.util.zip.ZipException + class ReadZipFile < Base def handle_specifically(watched_file) add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) diff --git a/lib/filewatch/read_mode/processor.rb b/lib/filewatch/read_mode/processor.rb index cdda265..09ee702 100644 --- a/lib/filewatch/read_mode/processor.rb +++ b/lib/filewatch/read_mode/processor.rb @@ -1,6 +1,5 @@ # encoding: utf-8 -require "logstash/util/loggable" - +require 'filewatch/processor' require_relative "handlers/base" require_relative "handlers/read_file" require_relative "handlers/read_zip_file" @@ -9,20 +8,7 @@ module FileWatch module ReadMode # Must handle # :read_file # :read_zip_file - class Processor - include LogStash::Util::Loggable - - attr_reader :watch, :deletable_filepaths - - def initialize(settings) - @settings = settings - @deletable_filepaths = [] - end - - def add_watch(watch) - @watch = watch - self - end + class Processor < FileWatch::Processor def initialize_handlers(sincedb_collection, observer) # we deviate from the tail mode handler initialization here @@ -48,24 +34,23 @@ def process_all_states(watched_files) private def process_watched(watched_files) - logger.trace("Watched processing") + logger.trace(__method__.to_s) # Handles watched_files in the watched state. # for a slice of them: # move to the active state # should never have been active before # how much of the max active window is available - to_take = @settings.max_active - watched_files.count{|wf| wf.active?} + to_take = @settings.max_active - watched_files.count { |wf| wf.active? } if to_take > 0 - watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file| - path = watched_file.path + watched_files.select(&:watched?).take(to_take).each do |watched_file| begin - watched_file.restat + restat(watched_file) watched_file.activate rescue Errno::ENOENT - common_deleted_reaction(watched_file, "Watched") + common_deleted_reaction(watched_file, __method__) next rescue => e - common_error_reaction(path, e, "Watched") + common_error_reaction(watched_file, e, __method__) next end break if watch.quit? @@ -74,7 +59,7 @@ def process_watched(watched_files) now = Time.now.to_i if (now - watch.lastwarn_max_files) > MAX_FILES_WARN_INTERVAL waiting = watched_files.size - @settings.max_active - logger.warn(@settings.max_warn_msg + ", files yet to open: #{waiting}") + logger.warn("#{@settings.max_warn_msg}, files yet to open: #{waiting}") watch.lastwarn_max_files = now end end @@ -83,17 +68,18 @@ def process_watched(watched_files) ## TODO add process_rotation_in_progress def process_active(watched_files) - logger.trace("Active processing") + logger.trace(__method__.to_s) # Handles watched_files in the active state. - watched_files.select {|wf| wf.active? }.each do |watched_file| - path = watched_file.path + watched_files.each do |watched_file| + next unless watched_file.active? + begin - watched_file.restat + restat(watched_file) rescue Errno::ENOENT - common_deleted_reaction(watched_file, "Active") + common_deleted_reaction(watched_file, __method__) next rescue => e - common_error_reaction(path, e, "Active") + common_error_reaction(watched_file, e, __method__) next end break if watch.quit? @@ -114,19 +100,19 @@ def process_active(watched_files) def common_detach_when_allread(watched_file) watched_file.unwatch watched_file.listener.reading_completed - deletable_filepaths << watched_file.path - logger.trace("Whole file read: #{watched_file.path}, removing from collection") + add_deletable_path watched_file.path + logger.trace? && logger.trace("whole file read, removing from collection", :path => watched_file.path) end def common_deleted_reaction(watched_file, action) # file has gone away or we can't read it anymore. watched_file.unwatch - deletable_filepaths << watched_file.path - logger.trace("#{action} - stat failed: #{watched_file.path}, removing from collection") + add_deletable_path watched_file.path + logger.trace? && logger.trace("#{action} - stat failed, removing from collection", :path => watched_file.path) end - def common_error_reaction(path, error, action) - logger.error("#{action} - other error #{path}: (#{error.message}, #{error.backtrace.take(8).inspect})") + def common_error_reaction(watched_file, error, action) + logger.error("#{action} - other error", error_details(error, watched_file)) end end end end diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb index 78c4e12..bbaa356 100644 --- a/lib/filewatch/sincedb_collection.rb +++ b/lib/filewatch/sincedb_collection.rb @@ -56,12 +56,12 @@ def open logger.trace("open: count of keys read: #{@sincedb.keys.size}") rescue => e #No existing sincedb to load - logger.trace("open: error: #{path}: #{e.inspect}") + logger.trace("open: error:", :path => path, :exception => e.class, :message => e.message) end end def associate(watched_file) - logger.trace("associate: finding", "inode" => watched_file.sincedb_key.inode, "path" => watched_file.path) + logger.trace? && logger.trace("associate: finding", :path => watched_file.path, :inode => watched_file.sincedb_key.inode) sincedb_value = find(watched_file) if sincedb_value.nil? # sincedb has no record of this inode @@ -71,7 +71,8 @@ def associate(watched_file) logger.trace("associate: unmatched") return true end - logger.trace("associate: found sincedb record", "filename" => watched_file.filename, "sincedb key" => watched_file.sincedb_key,"sincedb_value" => sincedb_value) + logger.trace? && logger.trace("associate: found sincedb record", :filename => watched_file.filename, + :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value) if sincedb_value.watched_file.nil? # not associated if sincedb_value.path_in_sincedb.nil? @@ -106,7 +107,8 @@ def associate(watched_file) # after the original is deleted # are not yet in the delete phase, let this play out existing_watched_file = sincedb_value.watched_file - logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename", "this watched_file details" => watched_file.details, "other watched_file details" => existing_watched_file.details) + logger.trace? && logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename", + :this_watched_file => watched_file.details, :existing_watched_file => existing_watched_file.details) watched_file.rotation_in_progress true end @@ -149,8 +151,8 @@ def set_watched_file(key, watched_file) end def watched_file_deleted(watched_file) - return unless member?(watched_file.sincedb_key) - get(watched_file.sincedb_key).unset_watched_file + value = @sincedb[watched_file.sincedb_key] + value.unset_watched_file if value end def store_last_read(key, pos) @@ -195,7 +197,7 @@ def handle_association(sincedb_value, watched_file) watched_file.initial_completed if watched_file.all_read? watched_file.ignore - logger.trace("handle_association fully read, ignoring.....", "watched file" => watched_file.details, "sincedb value" => sincedb_value) + logger.trace? && logger.trace("handle_association fully read, ignoring.....", :watched_file => watched_file.details, :sincedb_value => sincedb_value) end end @@ -214,8 +216,8 @@ def sincedb_write(time = Time.now.to_i) @write_method.call @serializer.expired_keys.each do |key| @sincedb[key].unset_watched_file - delete(key) - logger.trace("sincedb_write: cleaned", "key" => "'#{key}'") + delete(key) # delete + logger.trace? && logger.trace("sincedb_write: cleaned", :key => key) end @sincedb_last_write = time @write_requested = false diff --git a/lib/filewatch/stat/generic.rb b/lib/filewatch/stat/generic.rb index 6d83a72..0ecaaaf 100644 --- a/lib/filewatch/stat/generic.rb +++ b/lib/filewatch/stat/generic.rb @@ -3,24 +3,19 @@ module FileWatch module Stat class Generic - attr_reader :identifier, :inode, :modified_at, :size, :inode_struct + attr_reader :inode, :modified_at, :size, :inode_struct def initialize(source) - @source = source - @identifier = nil + @source = source # Pathname restat end - def add_identifier(identifier) self; end - def restat - @inner_stat = @source.stat - @inode = @inner_stat.ino.to_s - @modified_at = @inner_stat.mtime.to_f - @size = @inner_stat.size - @dev_major = @inner_stat.dev_major - @dev_minor = @inner_stat.dev_minor - @inode_struct = InodeStruct.new(@inode, @dev_major, @dev_minor) + stat = @source.stat + @inode = stat.ino.to_s + @modified_at = stat.mtime.to_f + @size = stat.size + @inode_struct = InodeStruct.new(@inode, stat.dev_major, stat.dev_minor) end def windows? @@ -28,7 +23,7 @@ def windows? end def inspect - "" + "<#{self.class.name} size=#{@size}, modified_at=#{@modified_at}, inode='#{@inode}', inode_struct=#{@inode_struct}>" end end end end diff --git a/lib/filewatch/stat/windows_path.rb b/lib/filewatch/stat/windows_path.rb index de773a4..68b1cda 100644 --- a/lib/filewatch/stat/windows_path.rb +++ b/lib/filewatch/stat/windows_path.rb @@ -3,22 +3,20 @@ module FileWatch module Stat class WindowsPath - attr_reader :identifier, :inode, :modified_at, :size, :inode_struct + attr_reader :inode, :modified_at, :size, :inode_struct def initialize(source) - @source = source + @source = source # Pathname @inode = Winhelper.identifier_from_path(@source.to_path) - @dev_major = 0 - @dev_minor = 0 # in windows the dev hi and low are in the identifier - @inode_struct = InodeStruct.new(@inode, @dev_major, @dev_minor) + @inode_struct = InodeStruct.new(@inode, 0, 0) restat end def restat - @inner_stat = @source.stat - @modified_at = @inner_stat.mtime.to_f - @size = @inner_stat.size + stat = @source.stat + @modified_at = stat.mtime.to_f + @size = stat.size end def windows? @@ -26,7 +24,7 @@ def windows? end def inspect - "" + "<#{self.class.name} size=#{@size}, modified_at=#{@modified_at}, inode=#{@inode}, inode_struct=#{@inode_struct}>" end end end end diff --git a/lib/filewatch/tail_mode/handlers/delete.rb b/lib/filewatch/tail_mode/handlers/delete.rb index cc7a79b..89b0224 100644 --- a/lib/filewatch/tail_mode/handlers/delete.rb +++ b/lib/filewatch/tail_mode/handlers/delete.rb @@ -7,11 +7,9 @@ def handle_specifically(watched_file) # TODO consider trying to find the renamed file - it will have the same inode. # Needs a rotate scheme rename hint from user e.g. "-YYYY-MM-DD-N." or "..N" # send the found content to the same listener (stream identity) - logger.trace("info", - "watched_file details" => watched_file.details, - "path" => watched_file.path) + logger.trace("delete", :path => watched_file.path, :watched_file => watched_file.details) if watched_file.bytes_unread > 0 - logger.warn(DATA_LOSS_WARNING, "unread_bytes" => watched_file.bytes_unread, "path" => watched_file.path) + logger.warn(DATA_LOSS_WARNING, :path => watched_file.path, :unread_bytes => watched_file.bytes_unread) end watched_file.listener.deleted # no need to worry about data in the buffer diff --git a/lib/filewatch/tail_mode/processor.rb b/lib/filewatch/tail_mode/processor.rb index d363806..6634dc9 100644 --- a/lib/filewatch/tail_mode/processor.rb +++ b/lib/filewatch/tail_mode/processor.rb @@ -1,5 +1,5 @@ # encoding: utf-8 -require "logstash/util/loggable" +require 'filewatch/processor' require_relative "handlers/base" require_relative "handlers/create_initial" require_relative "handlers/create" @@ -18,20 +18,7 @@ module FileWatch module TailMode # :delete - file can't be read # :timeout - file is closable # :unignore - file was ignored, but have now received new content - class Processor - include LogStash::Util::Loggable - - attr_reader :watch, :deletable_filepaths - - def initialize(settings) - @settings = settings - @deletable_filepaths = [] - end - - def add_watch(watch) - @watch = watch - self - end + class Processor < FileWatch::Processor def initialize_handlers(sincedb_collection, observer) @sincedb_collection = sincedb_collection @@ -91,11 +78,12 @@ def process_all_states(watched_files) private def process_closed(watched_files) - # logger.trace("Closed processing") + logger.trace(__method__.to_s) # Handles watched_files in the closed state. # if its size changed it is put into the watched state - watched_files.select {|wf| wf.closed? }.each do |watched_file| - common_restat_with_delay(watched_file, "Closed") do + watched_files.each do |watched_file| + next unless watched_file.closed? + common_restat_with_delay(watched_file, __method__) do # it won't do this if rotation is detected if watched_file.size_changed? # if the closed file changed, move it to the watched state @@ -108,13 +96,14 @@ def process_closed(watched_files) end def process_ignored(watched_files) - # logger.trace("Ignored processing") + logger.trace(__method__.to_s) # Handles watched_files in the ignored state. # if its size changed: # put it in the watched state # invoke unignore - watched_files.select {|wf| wf.ignored? }.each do |watched_file| - common_restat_with_delay(watched_file, "Ignored") do + watched_files.each do |watched_file| + next unless watched_file.ignored? + common_restat_with_delay(watched_file, __method__) do # it won't do this if rotation is detected if watched_file.size_changed? watched_file.watch @@ -128,11 +117,12 @@ def process_ignored(watched_files) def process_delayed_delete(watched_files) # defer the delete to one loop later to ensure that the stat really really can't find a renamed file # because a `stat` can be called right in the middle of the rotation rename cascade - logger.trace("Delayed Delete processing") - watched_files.select {|wf| wf.delayed_delete?}.each do |watched_file| - logger.trace(">>> Delayed Delete", "path" => watched_file.filename) - common_restat_without_delay(watched_file, ">>> Delayed Delete") do - logger.trace(">>> Delayed Delete: file at path found again", "watched_file" => watched_file.details) + logger.trace(__method__.to_s) + watched_files.each do |watched_file| + next unless watched_file.delayed_delete? + logger.trace(">>> Delayed Delete", :path => watched_file.path) + common_restat_without_delay(watched_file, __method__) do + logger.trace(">>> Delayed Delete: file at path found again", :watched_file => watched_file.details) watched_file.file_at_path_found_again end end @@ -140,33 +130,35 @@ def process_delayed_delete(watched_files) def process_restat_for_watched_and_active(watched_files) # do restat on all watched and active states once now. closed and ignored have been handled already - logger.trace("Watched + Active restat processing") - watched_files.select {|wf| wf.watched? || wf.active?}.each do |watched_file| - common_restat_with_delay(watched_file, "Watched") + logger.trace(__method__.to_s) + watched_files.each do |watched_file| + next if !watched_file.watched? && !watched_file.active? + common_restat_with_delay(watched_file, __method__) end end def process_rotation_in_progress(watched_files) - logger.trace("Rotation In Progress processing") - watched_files.select {|wf| wf.rotation_in_progress?}.each do |watched_file| + logger.trace(__method__.to_s) + watched_files.each do |watched_file| + next unless watched_file.rotation_in_progress? if !watched_file.all_read? if watched_file.file_open? # rotated file but original opened file is not fully read # we need to keep reading the open file, if we close it we lose it because the path is now pointing at a different file. - logger.trace(">>> Rotation In Progress - inode change detected and original content is not fully read, reading all", "watched_file details" => watched_file.details) + logger.trace(">>> Rotation In Progress - inode change detected and original content is not fully read, reading all", :watched_file => watched_file.details) # need to fully read open file while we can watched_file.set_maximum_read_loop grow(watched_file) watched_file.set_standard_read_loop else - logger.warn(">>> Rotation In Progress - inode change detected and original content is not fully read, file is closed and path points to new content", "watched_file details" => watched_file.details) + logger.warn(">>> Rotation In Progress - inode change detected and original content is not fully read, file is closed and path points to new content", :watched_file => watched_file.details) end end current_key = watched_file.sincedb_key sdb_value = @sincedb_collection.get(current_key) potential_key = watched_file.stat_sincedb_key potential_sdb_value = @sincedb_collection.get(potential_key) - logger.trace(">>> Rotation In Progress", "watched_file" => watched_file.details, "found_sdb_value" => sdb_value, "potential_key" => potential_key, "potential_sdb_value" => potential_sdb_value) + logger.trace(">>> Rotation In Progress", :watched_file => watched_file.details, :found_sdb_value => sdb_value, :potential_key => potential_key, :potential_sdb_value => potential_sdb_value) if potential_sdb_value.nil? logger.trace("---------- >>>> Rotation In Progress: rotating as existing file") watched_file.rotate_as_file @@ -189,13 +181,13 @@ def process_rotation_in_progress(watched_files) sdb_value.clear_watched_file unless sdb_value.nil? potential_sdb_value.set_watched_file(watched_file) else - logger.trace("---------- >>>> Rotation In Progress: rotating from...", "this watched_file details" => watched_file.details, "other watched_file details" => other_watched_file.details) + logger.trace("---------- >>>> Rotation In Progress: rotating from...", :this_watched_file => watched_file.details, :other_watched_file => other_watched_file.details) watched_file.rotate_from(other_watched_file) sdb_value.clear_watched_file unless sdb_value.nil? potential_sdb_value.set_watched_file(watched_file) end end - logger.trace("---------- >>>> Rotation In Progress: after handling rotation", "this watched_file details" => watched_file.details, "sincedb_value" => (potential_sdb_value || sdb_value)) + logger.trace("---------- >>>> Rotation In Progress: after handling rotation", :this_watched_file => watched_file.details, :sincedb_value => (potential_sdb_value || sdb_value)) end end @@ -206,11 +198,11 @@ def process_watched(watched_files) # and we allow the block to open the file and create a sincedb collection record if needed # some have never been active and some have # those that were active before but are watched now were closed under constraint - logger.trace("Watched processing") + logger.trace(__method__.to_s) # how much of the max active window is available - to_take = @settings.max_active - watched_files.count{|wf| wf.active?} + to_take = @settings.max_active - watched_files.count(&:active?) if to_take > 0 - watched_files.select {|wf| wf.watched?}.take(to_take).each do |watched_file| + watched_files.select(&:watched?).take(to_take).each do |watched_file| watched_file.activate if watched_file.initial? create_initial(watched_file) @@ -223,36 +215,37 @@ def process_watched(watched_files) now = Time.now.to_i if (now - watch.lastwarn_max_files) > MAX_FILES_WARN_INTERVAL waiting = watched_files.size - @settings.max_active - logger.warn(@settings.max_warn_msg + ", files yet to open: #{waiting}") + logger.warn("#{@settings.max_warn_msg}, files yet to open: #{waiting}") watch.lastwarn_max_files = now end end end def process_active(watched_files) - # logger.trace("Active processing") + logger.trace(__method__.to_s) # Handles watched_files in the active state. # files have been opened at this point - watched_files.select {|wf| wf.active? }.each do |watched_file| + watched_files.each do |watched_file| + next unless watched_file.active? break if watch.quit? path = watched_file.filename if watched_file.grown? - logger.trace("Active - file grew: #{path}: new size is #{watched_file.last_stat_size}, bytes read #{watched_file.bytes_read}") + logger.trace("#{__method__} file grew: new size is #{watched_file.last_stat_size}, bytes read #{watched_file.bytes_read}", :path => path) grow(watched_file) elsif watched_file.shrunk? if watched_file.bytes_unread > 0 - logger.warn("Active - shrunk: DATA LOSS!! truncate detected with #{watched_file.bytes_unread} unread bytes: #{path}") + logger.warn("potential data loss, file truncate detected with #{watched_file.bytes_unread} unread bytes", :path => path) end # we don't update the size here, its updated when we actually read - logger.trace("Active - file shrunk #{path}: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}") + logger.trace("#{__method__} file shrunk: new size is #{watched_file.last_stat_size}, old size #{watched_file.bytes_read}", :path => path) shrink(watched_file) else # same size, do nothing - logger.trace("Active - no change", "watched_file" => watched_file.details) + logger.trace("#{__method__} no change", :path => path) end # can any active files be closed to make way for waiting files? if watched_file.file_closable? - logger.trace("Watch each: active: file expired: #{path}") + logger.trace("#{__method__} file expired", :path => path) timeout(watched_file) watched_file.close end @@ -270,28 +263,28 @@ def common_restat_without_delay(watched_file, action, &block) def common_restat(watched_file, action, delay, &block) all_ok = true begin - watched_file.restat + restat(watched_file) if watched_file.rotation_in_progress? - logger.trace("-------------------- >>>>> restat - rotation_detected", "watched_file details" => watched_file.details, "new sincedb key" => watched_file.stat_sincedb_key) + logger.trace("-------------------- >>>>> restat - rotation_detected", :watched_file => watched_file.details, :new_sincedb_key => watched_file.stat_sincedb_key) # don't yield to closed and ignore processing else yield if block_given? end rescue Errno::ENOENT if delay - logger.trace("#{action} - delaying the stat fail on: #{watched_file.filename}") + logger.trace("#{action} - delaying the stat fail on", :filename => watched_file.filename) watched_file.delay_delete else # file has gone away or we can't read it anymore. - logger.trace("#{action} - after a delay, really can't find this file: #{watched_file.filename}") + logger.trace("#{action} - after a delay, really can't find this file", :path => watched_file.path) watched_file.unwatch - logger.trace("#{action} - removing from collection: #{watched_file.filename}") + logger.trace("#{action} - removing from collection", :filename => watched_file.filename) delete(watched_file) - deletable_filepaths << watched_file.path + add_deletable_path watched_file.path all_ok = false end rescue => e - logger.error("#{action} - other error #{watched_file.path}: (#{e.message}, #{e.backtrace.take(8).inspect})") + logger.error("#{action} - other error", error_details(e, watched_file)) all_ok = false end all_ok diff --git a/lib/filewatch/watch.rb b/lib/filewatch/watch.rb index 6cd4f9d..48e0e51 100644 --- a/lib/filewatch/watch.rb +++ b/lib/filewatch/watch.rb @@ -1,26 +1,25 @@ # encoding: utf-8 require "logstash/util/loggable" +require "concurrent/atomic/atomic_boolean" module FileWatch class Watch include LogStash::Util::Loggable attr_accessor :lastwarn_max_files - attr_reader :discoverer, :watched_files_collection + attr_reader :discoverer, :processor, :watched_files_collection - def initialize(discoverer, watched_files_collection, settings) + def initialize(discoverer, processor, settings) + @discoverer = discoverer + @watched_files_collection = discoverer.watched_files_collection @settings = settings + # we need to be threadsafe about the quit mutation @quit = Concurrent::AtomicBoolean.new(false) @lastwarn_max_files = 0 - @discoverer = discoverer - @watched_files_collection = watched_files_collection - end - def add_processor(processor) @processor = processor @processor.add_watch(self) - self end def watch(path) @@ -67,20 +66,16 @@ def iterate_on_state watched_files = @watched_files_collection.values @processor.process_all_states(watched_files) ensure - @watched_files_collection.remove_paths(@processor.deletable_filepaths) - @processor.deletable_filepaths.clear + @watched_files_collection.remove_paths(@processor.clear_deletable_paths) end - end # def each + end def quit @quit.make_true end def quit? - if @settings.exit_after_read - @exit = @watched_files_collection.empty? - end - @quit.true? || @exit + @quit.true? || (@settings.exit_after_read && @watched_files_collection.empty?) end private diff --git a/lib/filewatch/watched_file.rb b/lib/filewatch/watched_file.rb index 13a6933..5d6876f 100644 --- a/lib/filewatch/watched_file.rb +++ b/lib/filewatch/watched_file.rb @@ -6,7 +6,7 @@ class WatchedFile IO_BASED_STAT = 1 attr_reader :bytes_read, :state, :file, :buffer, :recent_states, :bytes_unread - attr_reader :path, :accessed_at, :modified_at, :pathname, :filename + attr_reader :path, :accessed_at, :pathname, :filename attr_reader :listener, :read_loop_count, :read_chunk_size, :stat attr_reader :loop_count_type, :loop_count_mode attr_accessor :last_open_warning_at @@ -16,7 +16,7 @@ class WatchedFile def initialize(pathname, stat, settings) @settings = settings @pathname = Pathname.new(pathname) # given arg pathname might be a string or a Pathname object - @path = @pathname.to_path + @path = @pathname.to_path.freeze @filename = @pathname.basename.to_s full_state_reset(stat) watch @@ -24,10 +24,6 @@ def initialize(pathname, stat, settings) set_accessed_at end - def no_restat_reset - full_state_reset(@stat) - end - def full_state_reset(this_stat = nil) if this_stat.nil? begin @@ -75,6 +71,7 @@ def set_stat(stat) @size = @stat.size @sdb_key_v1 = @stat.inode_struct end + private :set_stat def rotate_as_file(bytes_read = 0) # rotation, when a sincedb record exists for new inode, but no watched file to rotate from @@ -100,19 +97,33 @@ def rotation_detected? stat_sincedb_key != sincedb_key end - def restat + # @return true if the file was modified since last stat + def restat! + modified_at # to always be able to detect changes @stat.restat if rotation_detected? # switch to new state now rotation_in_progress + return true else @size = @stat.size update_bytes_unread + modified_at_changed? + end + end + + def modified_at(update = false) + if update || @modified_at.nil? + @modified_at = @stat.modified_at + else + @modified_at end end - def modified_at - @stat.modified_at + # @return whether modified_at changed since it was last read + # @see #restat! + def modified_at_changed? + modified_at != @stat.modified_at end def position_for_new_sincedb_value @@ -405,14 +416,14 @@ def file_can_close? end def details - detail = "@filename='#{filename}', @state='#{state}', @recent_states='#{@recent_states.inspect}', " - detail.concat("@bytes_read='#{@bytes_read}', @bytes_unread='#{@bytes_unread}', current_size='#{current_size}', ") - detail.concat("last_stat_size='#{last_stat_size}', file_open?='#{file_open?}', @initial=#{@initial}") - "" + detail = "@filename='#{@filename}', @state=#{@state.inspect}, @recent_states=#{@recent_states.inspect}, " + detail.concat("@bytes_read=#{@bytes_read}, @bytes_unread=#{@bytes_unread}, current_size=#{current_size}, ") + detail.concat("last_stat_size=#{last_stat_size}, file_open?=#{file_open?}, @initial=#{@initial}") + "" end def inspect - "\"" end def to_s diff --git a/lib/filewatch/watched_files_collection.rb b/lib/filewatch/watched_files_collection.rb index 153cd48..cbdd46e 100644 --- a/lib/filewatch/watched_files_collection.rb +++ b/lib/filewatch/watched_files_collection.rb @@ -1,89 +1,22 @@ # encoding: utf-8 -module FileWatch - class WatchedFilesCollection - - def initialize(settings) - @sort_by = settings.file_sort_by # "last_modified" | "path" - @sort_direction = settings.file_sort_direction # "asc" | "desc" - @sort_method = method("#{@sort_by}_#{@sort_direction}".to_sym) - @files = Concurrent::Array.new - @pointers = Concurrent::Hash.new - end - def add(watched_file) - @files << watched_file - @sort_method.call - end +require 'java' - def remove_paths(paths) - removed_files = Array(paths).map do |path| - index = @pointers.delete(path) - if index - watched_file = @files.delete_at(index) - refresh_pointers - watched_file - end - end - @sort_method.call - removed_files - end +module FileWatch + # @see `org.logstash.filewatch.WatchedFilesCollection` + class WatchedFilesCollection + # Closes all managed watched files. + # @see FileWatch::WatchedFile#file_close def close_all - @files.each(&:file_close) - end - - def empty? - @files.empty? - end - - def keys - @pointers.keys + each_file(&:file_close) # synchronized end - def values - @files - end - - def watched_file_by_path(path) - index = @pointers[path] - return nil unless index - @files[index] - end - - private - - def last_modified_asc - @files.sort! do |left, right| - left.modified_at <=> right.modified_at - end - refresh_pointers - end + # @return [Enumerable] managed path keys (snapshot) + alias keys paths - def last_modified_desc - @files.sort! do |left, right| - right.modified_at <=> left.modified_at - end - refresh_pointers - end - - def path_asc - @files.sort! do |left, right| - left.path <=> right.path - end - refresh_pointers - end + # @return [Enumerable] managed files (snapshot) + alias values files - def path_desc - @files.sort! do |left, right| - right.path <=> left.path - end - refresh_pointers - end - - def refresh_pointers - @files.each_with_index do |watched_file, index| - @pointers[watched_file.path] = index - end - end end end diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 0f11807..725e074 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -332,8 +332,9 @@ def completely_stopped? @completely_stopped.true? end + # The WatchedFile calls back here as `observer.listener_for(@path)` + # @param [String] path the identity def listener_for(path) - # path is the identity FileListener.new(path, self) end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index d6186f2..fc289b2 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.1.18' + s.version = '4.2.0' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -31,6 +31,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'addressable' end + s.add_runtime_dependency 'concurrent-ruby', '~> 1.0' s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0'] s.add_development_dependency 'stud', ['~> 0.0.19'] diff --git a/spec/filewatch/reading_spec.rb b/spec/filewatch/reading_spec.rb index e59b480..614950e 100644 --- a/spec/filewatch/reading_spec.rb +++ b/spec/filewatch/reading_spec.rb @@ -23,9 +23,12 @@ module FileWatch let(:start_new_files_at) { :end } # should be irrelevant for read mode let(:opts) do { - :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, - :delimiter => "\n", :discover_interval => discover_interval, - :ignore_older => 3600, :sincedb_path => sincedb_path + :stat_interval => stat_interval, + :start_new_files_at => start_new_files_at, + :delimiter => "\n", + :discover_interval => discover_interval, + :ignore_older => 3600, + :sincedb_path => sincedb_path } end let(:observer) { TestObserver.new } @@ -147,6 +150,50 @@ module FileWatch end end + context "when watching directory with files and adding a new file" do + let(:file_path2) { ::File.join(directory, "2.log") } + let(:file_path3) { ::File.join(directory, "3.log") } + + let(:opts) { super.merge(:file_sort_by => "last_modified") } + let(:lines) { [] } + let(:observer) { TestObserver.new(lines) } + + + let(:listener2) { observer.listener_for(file_path2) } + let(:listener3) { observer.listener_for(file_path3) } + + let(:actions) do + RSpec::Sequencing.run("create12") do + File.open(file_path, "w") { |file| file.write("string11\nstring12") } + File.open(file_path2, "w") { |file| file.write("string21\nstring22") } + end + .then("watch") do + reading.watch_this(watch_dir) + end + .then("wait12") do + wait(2).for { listener1.calls.last == :delete && listener2.calls.last == :delete }.to eq(true) + end + .then_after(2, "create3") do + File.open(file_path3, "w") { |file| file.write("string31\nstring32") } + end + .then("wait3") do + wait(2).for { listener3.calls.last == :delete }.to eq(true) + end + .then("quit") do + reading.quit + end + end + + it "reads all (3) files" do + actions.activate_quietly + reading.subscribe(observer) + actions.assert_no_errors + expect(lines.last).to eq 'string32' + expect(lines.sort).to eq %w(string11 string12 string21 string22 string31 string32) + expect( reading.watch.watched_files_collection.paths ).to eq [ file_path, file_path2, file_path3 ] + end + end + context "when watching a directory with files using exit_after_read" do let(:opts) { super.merge(:exit_after_read => true, :max_open_files => 2) } let(:file_path3) { ::File.join(directory, "3.log") } @@ -159,40 +206,45 @@ module FileWatch let(:listener6) { observer.listener_for(file_path6) } it "the file is read" do - File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } reading.watch_this(watch_dir) reading.subscribe(observer) expect(listener3.lines).to eq(["line1", "line2"]) end + it "multiple files are read" do - File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } File.open(file_path4, "w") { |file| file.write("line3\nline4\n") } reading.watch_this(watch_dir) reading.subscribe(observer) expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4"]) end + it "multiple files are read even if max_open_files is smaller then number of files" do - File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } File.open(file_path4, "w") { |file| file.write("line3\nline4\n") } File.open(file_path5, "w") { |file| file.write("line5\nline6\n") } reading.watch_this(watch_dir) reading.subscribe(observer) expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4", "line5", "line6"]) end + it "file as marked as reading_completed" do - File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } reading.watch_this(watch_dir) reading.subscribe(observer) expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed]) end + it "sincedb works correctly" do - File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } + File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } reading.watch_this(watch_dir) reading.subscribe(observer) sincedb_record_fields = File.read(sincedb_path).split(" ") position_field_index = 3 expect(sincedb_record_fields[position_field_index]).to eq("12") end + it "does not include new files added after start" do File.open(file_path3, "w") { |file| file.write("line1\nline2\n") } reading.watch_this(watch_dir) @@ -201,7 +253,6 @@ module FileWatch expect(listener3.lines).to eq(["line1", "line2"]) expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed]) expect(listener6.calls).to eq([]) - end end diff --git a/spec/filewatch/settings_spec.rb b/spec/filewatch/settings_spec.rb index 7b08002..fc1f325 100644 --- a/spec/filewatch/settings_spec.rb +++ b/spec/filewatch/settings_spec.rb @@ -1,3 +1,6 @@ +require 'logstash/devutils/rspec/spec_helper' +require 'logstash/inputs/friendly_durations' + describe FileWatch::Settings do context "when create from options" do diff --git a/spec/filewatch/spec_helper.rb b/spec/filewatch/spec_helper.rb index f2d32d7..4ebb1f7 100644 --- a/spec/filewatch/spec_helper.rb +++ b/spec/filewatch/spec_helper.rb @@ -117,17 +117,12 @@ class TestObserver class Listener attr_reader :path, :lines, :calls - def initialize(path) + def initialize(path, lines) @path = path - @lines = Concurrent::Array.new + @lines = lines || Concurrent::Array.new @calls = Concurrent::Array.new end - def add_lines(lines) - @lines = lines - self - end - def accept(line) @lines << line @calls << :accept @@ -161,12 +156,7 @@ def reading_completed attr_reader :listeners def initialize(combined_lines = nil) - listener_proc = if combined_lines.nil? - lambda{|k| Listener.new(k) } - else - lambda{|k| Listener.new(k).add_lines(combined_lines) } - end - @listeners = Concurrent::Hash.new {|hash, key| hash[key] = listener_proc.call(key) } + @listeners = Concurrent::Hash.new { |hash, key| hash[key] = new_listener(key, combined_lines) } end def listener_for(path) @@ -174,6 +164,14 @@ def listener_for(path) end def clear - @listeners.clear; end + @listeners.clear + end + + private + + def new_listener(path, lines = nil) + Listener.new(path, lines) + end + end end diff --git a/spec/filewatch/tailing_spec.rb b/spec/filewatch/tailing_spec.rb index bb0ff62..6f4cf84 100644 --- a/spec/filewatch/tailing_spec.rb +++ b/spec/filewatch/tailing_spec.rb @@ -10,15 +10,19 @@ module FileWatch let(:file_path) { ::File.join(directory, "1#{suffix}.log") } let(:file_path2) { ::File.join(directory, "2#{suffix}.log") } let(:file_path3) { ::File.join(directory, "3#{suffix}.log") } - let(:max) { 4095 } + let(:max) { 4095 } let(:stat_interval) { 0.1 } let(:discover_interval) { 4 } let(:start_new_files_at) { :end } let(:sincedb_path) { ::File.join(directory, "tailing.sdb") } let(:opts) do { - :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_open_files => max, - :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path, + :stat_interval => stat_interval, + :start_new_files_at => start_new_files_at, + :max_open_files => max, + :delimiter => "\n", + :discover_interval => discover_interval, + :sincedb_path => sincedb_path, :file_sort_by => "path" } end @@ -30,12 +34,11 @@ module FileWatch before do directory - wait(1.0).for{Dir.exist?(directory)}.to eq(true) + wait(1.0).for { Dir.exist?(directory) }.to eq(true) end after do FileUtils.rm_rf(directory) - wait(1.0).for{Dir.exist?(directory)}.to eq(false) end describe "max open files (set to 1)" do @@ -95,16 +98,16 @@ module FileWatch let(:actions) do RSpec::Sequencing .run("create file") do - File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") } + File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") } end .then_after(0.1, "begin watching") do tailing.watch_this(watch_dir) end - .then_after(2, "add content") do - File.open(file_path, "ab") { |file| file.write("line1\nline2\n") } + .then_after(1.0, "add content") do + File.open(file_path, "ab") { |file| file.write("line1\nline2\n") } end .then("wait") do - wait(0.75).for{listener1.lines}.to eq(["line1", "line2"]) + wait(0.75).for { listener1.lines }.to_not be_empty end .then("quit") do tailing.quit @@ -113,7 +116,6 @@ module FileWatch it "only the new content is read" do actions.activate_quietly - tailing.watch_this(watch_dir) tailing.subscribe(observer) actions.assert_no_errors expect(listener1.calls).to eq([:open, :accept, :accept]) @@ -132,7 +134,7 @@ module FileWatch File.open(file_path, "wb") { |file| file.write("line1\nline2\n") } end .then("wait") do - wait(0.75).for{listener1.lines.size}.to eq(2) + wait(0.75).for { listener1.lines }.to_not be_empty end .then("quit") do tailing.quit @@ -154,7 +156,7 @@ module FileWatch # so when a stat is taken on the file an error is raised let(:suffix) { "E" } let(:quit_after) { 0.2 } - let(:stat) { double("stat", :size => 100, :modified_at => Time.now.to_f, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) } + let(:stat) { double("stat", :size => 100, :modified_at => Time.now.to_f, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) } let(:watched_file) { WatchedFile.new(file_path, stat, tailing.settings) } before do allow(stat).to receive(:restat).and_raise(Errno::ENOENT) diff --git a/spec/filewatch/watched_file_spec.rb b/spec/filewatch/watched_file_spec.rb index a532ac1..a639491 100644 --- a/spec/filewatch/watched_file_spec.rb +++ b/spec/filewatch/watched_file_spec.rb @@ -35,5 +35,35 @@ module FileWatch expect(watched_file.recent_states).to eq([:watched, :active, :watched, :closed, :watched, :active, :unwatched, :active]) end end + + context 'restat' do + + let(:directory) { Stud::Temporary.directory } + let(:file_path) { ::File.join(directory, "restat.file.txt") } + let(:pathname) { Pathname.new(file_path) } + + before { FileUtils.touch file_path, :mtime => Time.now - 300 } + + it 'reports false value when no changes' do + file = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new) + mtime = file.modified_at + expect( file.modified_at_changed? ).to be false + expect( file.restat! ).to be_falsy + expect( file.modified_at_changed? ).to be false + expect( file.modified_at ).to eql mtime + expect( file.modified_at(true) ).to eql mtime + end + + it 'reports truthy when changes detected' do + file = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new) + mtime = file.modified_at + expect( file.modified_at_changed? ).to be false + FileUtils.touch file_path + expect( file.restat! ).to be_truthy + expect( file.modified_at_changed? ).to be true + expect( file.modified_at ).to eql mtime # until updated + expect( file.modified_at(true) ).to be > mtime + end + end end end diff --git a/spec/filewatch/watched_files_collection_spec.rb b/spec/filewatch/watched_files_collection_spec.rb index 50c2b59..d1b778e 100644 --- a/spec/filewatch/watched_files_collection_spec.rb +++ b/spec/filewatch/watched_files_collection_spec.rb @@ -4,15 +4,18 @@ module FileWatch describe WatchedFilesCollection do let(:time) { Time.now } - let(:filepath1){"/var/log/z.log"} - let(:filepath2){"/var/log/m.log"} - let(:filepath3){"/var/log/a.log"} - let(:stat1) { double("stat1", :size => 98, :modified_at => time - 30, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) } - let(:stat2) { double("stat2", :size => 99, :modified_at => time - 20, :identifier => nil, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) } - let(:stat3) { double("stat3", :size => 100, :modified_at => time, :identifier => nil, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) } + let(:filepath1) { "/var/log/z.log" } + let(:filepath2) { "/var/log/m.log" } + let(:filepath3) { "/var/log/a.log" } + let(:filepath4) { "/var/log/b.log" } + let(:stat1) { double("stat1", :size => 98, :modified_at => time - 30, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) } + let(:stat2) { double("stat2", :size => 99, :modified_at => time - 20, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) } + let(:stat3) { double("stat3", :size => 100, :modified_at => time, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) } + let(:stat4) { double("stat4", :size => 99, :modified_at => time, :inode => 234570, :inode_struct => InodeStruct.new("234570", 3, 2)) } let(:wf1) { WatchedFile.new(filepath1, stat1, Settings.new) } let(:wf2) { WatchedFile.new(filepath2, stat2, Settings.new) } let(:wf3) { WatchedFile.new(filepath3, stat3, Settings.new) } + let(:wf4) { WatchedFile.new(filepath4, stat4, Settings.new) } context "sort by last_modified in ascending order" do let(:sort_by) { "last_modified" } @@ -20,12 +23,29 @@ module FileWatch it "sorts earliest modified first" do collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction)) + expect(collection.empty?).to be true collection.add(wf2) + expect(collection.empty?).to be false expect(collection.values).to eq([wf2]) collection.add(wf3) expect(collection.values).to eq([wf2, wf3]) collection.add(wf1) expect(collection.values).to eq([wf1, wf2, wf3]) + expect(collection.keys.size).to eq 3 + end + + it "sorts by path when mtime is same" do + collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction)) + expect(collection.size).to eq 0 + collection.add(wf2) + collection.add(wf4) + collection.add(wf1) + expect(collection.size).to eq 3 + expect(collection.values).to eq([wf1, wf2, wf4]) + collection.add(wf3) + expect(collection.size).to eq 4 + expect(collection.values).to eq([wf1, wf2, wf3, wf4]) + expect(collection.keys.size).to eq 4 end end @@ -74,7 +94,7 @@ module FileWatch end end - context "when delete called" do + context "remove_paths" do let(:sort_by) { "path" } let(:sort_direction) { "desc" } @@ -85,9 +105,43 @@ module FileWatch collection.add(wf3) expect(collection.keys).to eq([filepath1, filepath2, filepath3]) - collection.remove_paths([filepath2,filepath3]) + ret = collection.remove_paths([filepath2, filepath3]) + expect(ret).to eq 2 expect(collection.keys).to eq([filepath1]) expect(collection.values.size).to eq 1 + + ret = collection.remove_paths([filepath2]) + expect(ret).to eq 0 + end + end + + context "update" do + let(:sort_by) { "last_modified" } + let(:sort_direction) { "asc" } + + let(:re_stat1) { double("restat1", :size => 99, :modified_at => time, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) } + let(:re_stat2) { double("restat2", :size => 99, :modified_at => time, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) } + + it "updates entry with changed mtime" do + collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction)) + collection.add(wf1) + collection.add(wf2) + collection.add(wf3) + expect(collection.files).to eq([wf1, wf2, wf3]) + + wf2.send(:set_stat, re_stat2) + expect( wf2.modified_at_changed? ).to be_truthy + + collection.update wf2 + expect(collection.files).to eq([wf1, wf3, wf2]) + + wf1.send(:set_stat, re_stat1) + expect( wf1.modified_at_changed? ).to be_truthy + collection.update wf1 + expect(collection.files).to eq([wf3, wf2, wf1]) + + collection.add(wf4) + expect(collection.files).to eq([wf3, wf4, wf2, wf1]) end end diff --git a/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java b/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java index e37d186..ab344cd 100644 --- a/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java +++ b/src/main/java/org/logstash/filewatch/JrubyFileWatchLibrary.java @@ -84,6 +84,7 @@ public final void load(final Ruby runtime, final boolean wrap) { clazz = runtime.defineClassUnder("Fnv", runtime.getObject(), JrubyFileWatchLibrary.Fnv::new, module); clazz.defineAnnotatedMethods(JrubyFileWatchLibrary.Fnv.class); + WatchedFilesCollection.load(runtime); } @JRubyClass(name = "FileExt") diff --git a/src/main/java/org/logstash/filewatch/WatchedFilesCollection.java b/src/main/java/org/logstash/filewatch/WatchedFilesCollection.java new file mode 100644 index 0000000..51fa8a9 --- /dev/null +++ b/src/main/java/org/logstash/filewatch/WatchedFilesCollection.java @@ -0,0 +1,235 @@ +package org.logstash.filewatch; + +import org.jruby.Ruby; +import org.jruby.RubyArray; +import org.jruby.RubyBoolean; +import org.jruby.RubyClass; +import org.jruby.RubyFloat; +import org.jruby.RubyHash; +import org.jruby.RubyObject; +import org.jruby.RubyString; +import org.jruby.anno.JRubyMethod; +import org.jruby.runtime.Block; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.Visibility; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.runtime.callsite.CachingCallSite; +import org.jruby.runtime.callsite.FunctionalCachingCallSite; + +import java.util.Comparator; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * FileWatch::WatchedFilesCollection for managing paths mapped to (watched) files. + * + * Implemented in native to avoid Ruby->Java type casting (which JRuby provides no control of as of 9.2). + * The collection already has a noticeable footprint when 10_000s of files are being watched at once, having + * the implementation in Java reduces 1000s of String conversions on every watch re-stat tick. + */ +public class WatchedFilesCollection extends RubyObject { + + // we could have used Ruby's SortedSet but it does not provide support for custom comparators + private SortedMap files; // FileWatch::WatchedFile -> String + private RubyHash filesInverse; // String -> FileWatch::WatchedFile + private String sortBy; + + public WatchedFilesCollection(Ruby runtime, RubyClass metaClass) { + super(runtime, metaClass); + } + + static void load(Ruby runtime) { + runtime.getOrCreateModule("FileWatch") + .defineClassUnder("WatchedFilesCollection", runtime.getObject(), WatchedFilesCollection::new) + .defineAnnotatedMethods(WatchedFilesCollection.class); + } + + @JRubyMethod + public IRubyObject initialize(final ThreadContext context, IRubyObject settings) { + final String sort_by = settings.callMethod(context, "file_sort_by").asJavaString(); + final String sort_direction = settings.callMethod(context, "file_sort_direction").asJavaString(); + + Comparator comparator; + switch (sort_by) { + case "last_modified" : + sortBy = "modified_at"; + comparator = (file1, file2) -> { + if (file1 == file2) return 0; // fast shortcut + RubyFloat mtime1 = modified_at(context, file1); + RubyFloat mtime2 = modified_at(context, file2); + int cmp = Double.compare(mtime1.getDoubleValue(), mtime2.getDoubleValue()); + // if mtime same (rare unless file1 == file2) - order consistently + if (cmp == 0) return path(context, file1).op_cmp(path(context, file2)); + return cmp; + }; + break; + case "path" : + sortBy = "path"; + comparator = (file1, file2) -> path(context, file1).op_cmp(path(context, file2)); + break; + default : + throw context.runtime.newArgumentError("sort_by: '" + sort_by + "' not supported"); + } + switch (sort_direction) { + case "asc" : + // all good - comparator uses ascending order + break; + case "desc" : + comparator = comparator.reversed(); + break; + default : + throw context.runtime.newArgumentError("sort_direction: '" + sort_direction + "' not supported"); + } + + this.files = new TreeMap<>(comparator); + this.filesInverse = RubyHash.newHash(context.runtime); + + // variableTableStore("@files", JavaUtil.convertJavaToRuby(context.runtime, this.files)); + // variableTableStore("@files_inverse", this.filesInverse); + + return this; + } + + @JRubyMethod + public IRubyObject add(ThreadContext context, IRubyObject file) { + RubyString path = getFilePath(context, file); + synchronized (this) { + RubyString prev_path = this.files.put(file, path); + assert prev_path == null || path.equals(prev_path); // file's path should not change! + this.filesInverse.op_aset(context, path, file); + } + return path; + } + + private static RubyString getFilePath(ThreadContext context, IRubyObject file) { + IRubyObject path = file.callMethod(context, "path"); + if (!(path instanceof RubyString)) { + throw context.runtime.newTypeError("expected file.path to return String but did not file: " + file.inspect()); + } + if (!path.isFrozen()) path = ((RubyString) path).dupFrozen(); // path = path.dup.freeze + return (RubyString) path; + } + + @JRubyMethod + public IRubyObject remove_paths(ThreadContext context, IRubyObject arg) { + IRubyObject[] paths; + if (arg instanceof RubyArray) { + paths = ((RubyArray) arg).toJavaArray(); + } else { + paths = new IRubyObject[] { arg }; + } + + int removedCount = 0; + synchronized (this) { + for (final IRubyObject path : paths) { + if (removePath(context, path.convertToString())) removedCount++; + } + } + return context.runtime.newFixnum(removedCount); + } + + private boolean removePath(ThreadContext context, RubyString path) { + IRubyObject file = this.filesInverse.delete(context, path, Block.NULL_BLOCK); + if (file.isNil()) return false; + return this.files.remove(file) != null; + } + + @JRubyMethod // synchronize { @files_inverse[path] } + public synchronized IRubyObject get(ThreadContext context, IRubyObject path) { + return this.filesInverse.op_aref(context, path); + } + + @JRubyMethod // synchronize { @files.size } + public synchronized IRubyObject size(ThreadContext context) { + return context.runtime.newFixnum(this.files.size()); + } + + @JRubyMethod(name = "empty?") // synchronize { @files.empty? } + public synchronized IRubyObject empty_p(ThreadContext context) { + return context.runtime.newBoolean(this.files.isEmpty()); + } + + @JRubyMethod + public synchronized IRubyObject each_file(ThreadContext context, Block block) { + for (IRubyObject watched_file : this.files.keySet()) { + block.yield(context, watched_file); + } + return context.nil; + } + + @JRubyMethod // synchronize { @files.values.to_a } + public IRubyObject paths(ThreadContext context) { + IRubyObject[] values; + synchronized (this) { + values = this.files.values().stream().toArray(IRubyObject[]::new); + } + return context.runtime.newArrayNoCopy(values); + } + + // NOTE: needs to return properly ordered files (can not use @files_inverse) + @JRubyMethod // synchronize { @files.key_set.to_a } + public IRubyObject files(ThreadContext context) { + IRubyObject[] keys; + synchronized (this) { + keys = this.files.keySet().stream().toArray(IRubyObject[]::new); + } + return context.runtime.newArrayNoCopy(keys); + } + + + @JRubyMethod + public IRubyObject update(ThreadContext context, IRubyObject file) { + // NOTE: modified_at might change on restat - to cope with that we need to potentially + // update the sorted collection, on such changes (when file_sort_by: last_modified) : + if (!"modified_at".equals(sortBy)) return context.nil; + + RubyString path = getFilePath(context, file); + synchronized (this) { + this.files.remove(file); // we need to "re-sort" changed file -> remove and add it back + modified_at(context, file, context.tru); // file.modified_at(update: true) + RubyString prev_path = this.files.put(file, path); + assert prev_path == null; + } + return context.tru; + } + + @JRubyMethod(required = 1, visibility = Visibility.PRIVATE) + @Override + public IRubyObject initialize_copy(IRubyObject original) { + final Ruby runtime = getRuntime(); + if (!(original instanceof WatchedFilesCollection)) { + throw runtime.newTypeError("Expecting an instance of class WatchedFilesCollection"); + } + + WatchedFilesCollection proto = (WatchedFilesCollection) original; + + this.files = new TreeMap<>(proto.files.comparator()); + synchronized (proto) { + this.files.putAll(proto.files); + this.filesInverse = (RubyHash) proto.filesInverse.dup(runtime.getCurrentContext()); + } + + return this; + } + + @Override + public IRubyObject inspect() { + return getRuntime().newString("#<" + metaClass.getRealClass().getName() + ": size=" + this.files.size() + ">"); + } + + private static final CachingCallSite modified_at_site = new FunctionalCachingCallSite("modified_at"); + private static final CachingCallSite path_site = new FunctionalCachingCallSite("path"); + + private static RubyString path(ThreadContext context, IRubyObject watched_file) { + return path_site.call(context, watched_file, watched_file).convertToString(); + } + + private static RubyFloat modified_at(ThreadContext context, IRubyObject watched_file) { + return modified_at_site.call(context, watched_file, watched_file).convertToFloat(); + } + + private static RubyFloat modified_at(ThreadContext context, IRubyObject watched_file, RubyBoolean update) { + return modified_at_site.call(context, watched_file, watched_file, update).convertToFloat(); + } + +} From be18de77292f0aafb667debbdef8fe7297065fe3 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 13 May 2020 17:56:43 +0200 Subject: [PATCH 70/91] Fix: skip sincedb eviction if read mode completion deletes file during flush If the value of the setting `sincedb_clean_after` is too short (1 ms) it happens that during the content reading loop (in read mode) the loops itself trigger the cleanup of the same value from the SincedbCollection and then it also try to access it again creating an error. This commit resolves the problem guarding against the presence in the sincebd_collection before accessing it. Fixes: #272 Closes: #273 --- CHANGELOG.md | 3 +++ lib/filewatch/read_mode/handlers/read_file.rb | 6 ++++-- lib/filewatch/settings.rb | 3 +-- lib/filewatch/sincedb_collection.rb | 2 +- logstash-input-file.gemspec | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8129948..7eeb387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.2.1 + - Fix: skip sincedb eviction if read mode completion deletes file during flush [#273](https://github.com/logstash-plugins/logstash-input-file/pull/273) + ## 4.2.0 - Fix: watched files performance with huge filesets [#268](https://github.com/logstash-plugins/logstash-input-file/pull/268) - Updated logging to include full traces in debug (and trace) levels diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index 0717a78..77cfe9f 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -19,8 +19,10 @@ def handle_specifically(watched_file) watched_file.listener.eof watched_file.file_close key = watched_file.sincedb_key - sincedb_collection.reading_completed(key) - sincedb_collection.clear_watched_file(key) + if sincedb_collection.get(key) + sincedb_collection.reading_completed(key) + sincedb_collection.clear_watched_file(key) + end watched_file.listener.deleted # NOTE: on top of un-watching we should also remove from the watched files collection # if the file is getting deleted (on completion), that part currently resides in diff --git a/lib/filewatch/settings.rb b/lib/filewatch/settings.rb index e62efce..0bc7af5 100644 --- a/lib/filewatch/settings.rb +++ b/lib/filewatch/settings.rb @@ -6,7 +6,7 @@ class Settings attr_reader :max_active, :max_warn_msg, :lastwarn_max_files attr_reader :sincedb_write_interval, :stat_interval, :discover_interval attr_reader :exclude, :start_new_files_at, :file_chunk_count, :file_chunk_size - attr_reader :sincedb_path, :sincedb_write_interval, :sincedb_expiry_duration + attr_reader :sincedb_path, :sincedb_expiry_duration attr_reader :file_sort_by, :file_sort_direction attr_reader :exit_after_read attr_reader :check_archive_validity @@ -41,7 +41,6 @@ def add_options(opts) @file_chunk_size = @opts[:file_chunk_size] @close_older = @opts[:close_older] @ignore_older = @opts[:ignore_older] - @sincedb_write_interval = @opts[:sincedb_write_interval] @stat_interval = @opts[:stat_interval] @discover_interval = @opts[:discover_interval] @exclude = Array(@opts[:exclude]) diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb index bbaa356..a19ed3c 100644 --- a/lib/filewatch/sincedb_collection.rb +++ b/lib/filewatch/sincedb_collection.rb @@ -216,7 +216,7 @@ def sincedb_write(time = Time.now.to_i) @write_method.call @serializer.expired_keys.each do |key| @sincedb[key].unset_watched_file - delete(key) # delete + delete(key) logger.trace? && logger.trace("sincedb_write: cleaned", :key => key) end @sincedb_last_write = time diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index fc289b2..2a1d85b 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.2.0' + s.version = '4.2.1' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 20db56d21eb90c3e58e016ce2df677acf4054d2c Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Fri, 25 Sep 2020 17:50:43 +0200 Subject: [PATCH 71/91] Fix: sincedb_clean_after not being respected (#276) In certain cases, in read mode sincedb is not updated. There was no periodic check for sincedb updates that would cause the sincedb_clean_after entries to cleanup. The cleanup relied on new files being discovered or new content being added to existing files -> causing sincedb updates. The fix here is to periodically flush sincedb (from the watch loop). Besides, to make the process more deterministic, there's a minor change to make sure the same "updated" timestamp is used to mark the last changed time. resolves #250 expected to also resolve #260 --- CHANGELOG.md | 3 ++ lib/filewatch/observing_base.rb | 10 ----- lib/filewatch/sincedb_collection.rb | 26 +++++------ lib/filewatch/watch.rb | 3 ++ logstash-input-file.gemspec | 2 +- spec/inputs/file_read_spec.rb | 70 +++++++++++++++++++++++------ 6 files changed, 77 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7eeb387..51f7abc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.2.2 + - Fix: sincedb_clean_after not being respected [#276](https://github.com/logstash-plugins/logstash-input-file/pull/276) + ## 4.2.1 - Fix: skip sincedb eviction if read mode completion deletes file during flush [#273](https://github.com/logstash-plugins/logstash-input-file/pull/273) diff --git a/lib/filewatch/observing_base.rb b/lib/filewatch/observing_base.rb index ecdf7f7..06cd317 100644 --- a/lib/filewatch/observing_base.rb +++ b/lib/filewatch/observing_base.rb @@ -83,15 +83,5 @@ def quit # sincedb_write("shutting down") end - # close_file(path) is to be used by external code - # when it knows that it is completely done with a file. - # Other files or folders may still be being watched. - # Caution, once unwatched, a file can't be watched again - # unless a new instance of this class begins watching again. - # The sysadmin should rename, move or delete the file. - def close_file(path) - @watch.unwatch(path) - sincedb_write - end end end diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb index a19ed3c..2d3458a 100644 --- a/lib/filewatch/sincedb_collection.rb +++ b/lib/filewatch/sincedb_collection.rb @@ -180,17 +180,17 @@ def watched_file_unset?(key) get(key).watched_file.nil? end - private - def flush_at_interval - now = Time.now.to_i - delta = now - @sincedb_last_write + now = Time.now + delta = now.to_i - @sincedb_last_write if delta >= @settings.sincedb_write_interval logger.debug("writing sincedb (delta since last write = #{delta})") sincedb_write(now) end end + private + def handle_association(sincedb_value, watched_file) watched_file.update_bytes_read(sincedb_value.position) sincedb_value.set_watched_file(watched_file) @@ -210,33 +210,33 @@ def set_key_value(key, value) end end - def sincedb_write(time = Time.now.to_i) - logger.trace("sincedb_write: to: #{path}") + def sincedb_write(time = Time.now) + logger.trace("sincedb_write: #{path} (time = #{time})") begin - @write_method.call + @write_method.call(time) @serializer.expired_keys.each do |key| @sincedb[key].unset_watched_file delete(key) logger.trace? && logger.trace("sincedb_write: cleaned", :key => key) end - @sincedb_last_write = time + @sincedb_last_write = time.to_i @write_requested = false rescue Errno::EACCES # no file handles free perhaps # maybe it will work next time - logger.trace("sincedb_write: error: #{path}: #{$!}") + logger.trace("sincedb_write: #{path} error: #{$!}") end end - def atomic_write + def atomic_write(time) FileHelper.write_atomically(@full_path) do |io| - @serializer.serialize(@sincedb, io) + @serializer.serialize(@sincedb, io, time.to_f) end end - def non_atomic_write + def non_atomic_write(time) IO.open(IO.sysopen(@full_path, "w+")) do |io| - @serializer.serialize(@sincedb, io) + @serializer.serialize(@sincedb, io, time.to_f) end end end diff --git a/lib/filewatch/watch.rb b/lib/filewatch/watch.rb index 48e0e51..8bee120 100644 --- a/lib/filewatch/watch.rb +++ b/lib/filewatch/watch.rb @@ -51,7 +51,10 @@ def subscribe(observer, sincedb_collection) glob = 0 end break if quit? + # NOTE: maybe the plugin should validate stat_interval <= sincedb_write_interval <= sincedb_clean_after sleep(@settings.stat_interval) + # we need to check potential expired keys (sincedb_clean_after) periodically + sincedb_collection.flush_at_interval end sincedb_collection.write_if_requested # does nothing if no requests to write were lodged. @watched_files_collection.close_all diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 2a1d85b..bc7afa6 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.2.1' + s.version = '4.2.2' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index 14fc933..d1533bd 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -301,25 +301,69 @@ watched_files = plugin.watcher.watch.watched_files_collection expect( watched_files ).to be_empty end + end - private + describe 'sincedb cleanup' do - def wait_for_start_processing(run_thread, timeout: 1.0) - begin - Timeout.timeout(timeout) do - sleep(0.01) while run_thread.status != 'sleep' - sleep(timeout) unless plugin.queue - end - rescue Timeout::Error - raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue - else - raise "plugin did not start processing" unless plugin.queue + let(:options) do + super.merge( + 'sincedb_path' => sincedb_path, + 'sincedb_clean_after' => '1.0 seconds', + 'sincedb_write_interval' => 0.25, + 'stat_interval' => 0.1, + ) + end + + let(:sincedb_path) { "#{temp_directory}/.sincedb" } + + let(:sample_file) { File.join(temp_directory, "sample.txt") } + + before do + plugin.register + @run_thread = Thread.new(plugin) do |plugin| + Thread.current.abort_on_exception = true + plugin.run queue end + + File.open(sample_file, 'w') { |fd| fd.write("line1\nline2\n") } + + wait_for_start_processing(@run_thread) end - def wait_for_file_removal(path, timeout: 3 * interval) - wait(timeout).for { File.exist?(path) }.to be_falsey + after { plugin.stop } + + it 'cleans up sincedb entry' do + wait_for_file_removal(sample_file) # watched discovery + + sincedb_content = File.read(sincedb_path).strip + expect( sincedb_content ).to_not be_empty + + Stud.try(3.times) do + sleep(1.5) # > sincedb_clean_after + + sincedb_content = File.read(sincedb_path).strip + expect( sincedb_content ).to be_empty + end end end + + private + + def wait_for_start_processing(run_thread, timeout: 1.0) + begin + Timeout.timeout(timeout) do + sleep(0.01) while run_thread.status != 'sleep' + sleep(timeout) unless plugin.queue + end + rescue Timeout::Error + raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue + else + raise "plugin did not start processing" unless plugin.queue + end + end + + def wait_for_file_removal(path, timeout: 3 * interval) + wait(timeout).for { File.exist?(path) }.to be_falsey + end end From 871e40e18348d5ec1ceb8f92ec442bb3f36c13f6 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Tue, 1 Dec 2020 18:09:22 +0100 Subject: [PATCH 72/91] Refactor: improve debug logging (log catched exceptions) (#280) tried to normalize traces involving `:filename` or `:path` to be more consistent - easier to grep. otherwise the intent is mostly (debug) logging exceptions the plugin's expected to rescue and continue running. particularly, I wasn't sure why a watched file keeps going through the read loop (having new content) but comes out as never reading anything on Windows (e.g. from here), knowing whether there's an EOF or file-access gone wrong would confirm whether there's a deeper issue in the plugin or not. have also raised a few traces to the debug level - ones that seem to provide more value than just tracing what the plugin is processing atm. --- CHANGELOG.md | 3 + lib/filewatch/read_mode/handlers/base.rb | 15 +++-- lib/filewatch/sincedb_collection.rb | 47 ++++++++-------- lib/filewatch/sincedb_record_serializer.rb | 16 ++---- lib/filewatch/tail_mode/handlers/base.rb | 55 +++++++++++-------- lib/filewatch/tail_mode/handlers/delete.rb | 2 +- lib/filewatch/tail_mode/handlers/shrink.rb | 5 +- lib/filewatch/tail_mode/handlers/unignore.rb | 8 +-- lib/logstash/inputs/file.rb | 4 +- lib/logstash/inputs/file_listener.rb | 17 +----- logstash-input-file.gemspec | 2 +- .../sincedb_record_serializer_spec.rb | 8 ++- 12 files changed, 88 insertions(+), 94 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51f7abc..a2d1b0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.2.3 + - Refactor: improve debug logging (log catched exceptions) [#280](https://github.com/logstash-plugins/logstash-input-file/pull/280) + ## 4.2.2 - Fix: sincedb_clean_after not being respected [#276](https://github.com/logstash-plugins/logstash-input-file/pull/276) diff --git a/lib/filewatch/read_mode/handlers/base.rb b/lib/filewatch/read_mode/handlers/base.rb index 2bce4d0..ce02f8e 100644 --- a/lib/filewatch/read_mode/handlers/base.rb +++ b/lib/filewatch/read_mode/handlers/base.rb @@ -19,7 +19,7 @@ def quit? end def handle(watched_file) - logger.trace("handling: #{watched_file.path}") + logger.trace? && logger.trace("handling:", :path => watched_file.path) unless watched_file.has_listener? watched_file.set_listener(@observer) end @@ -41,14 +41,14 @@ def open_file(watched_file) # don't emit this message too often. if a file that we can't # read is changing a lot, we'll try to open it more often, and spam the logs. now = Time.now.to_i - logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") + logger.trace? && logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL backtrace = e.backtrace backtrace = backtrace.take(3) if backtrace && !logger.debug? logger.warn("failed to open", :path => watched_file.path, :exception => e.class, :message => e.message, :backtrace => backtrace) watched_file.last_open_warning_at = now else - logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message) + logger.trace? && logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message) end watched_file.watch # set it back to watch so we can try it again end @@ -67,8 +67,7 @@ def add_or_update_sincedb_collection(watched_file) elsif sincedb_value.watched_file == watched_file update_existing_sincedb_collection_value(watched_file, sincedb_value) else - msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file" - logger.trace(msg) + logger.trace? && logger.trace("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file") existing_watched_file = sincedb_value.watched_file if existing_watched_file.nil? sincedb_value.set_watched_file(watched_file) @@ -77,7 +76,7 @@ def add_or_update_sincedb_collection(watched_file) watched_file.update_bytes_read(sincedb_value.position) else sincedb_value.set_watched_file(watched_file) - logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details) + logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details) watched_file.rotate_from(existing_watched_file) end @@ -86,7 +85,7 @@ def add_or_update_sincedb_collection(watched_file) end def update_existing_sincedb_collection_value(watched_file, sincedb_value) - logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}") + logger.trace? && logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}") # sincedb_value is the source of truth watched_file.update_bytes_read(sincedb_value.position) end @@ -94,7 +93,7 @@ def update_existing_sincedb_collection_value(watched_file, sincedb_value) def add_new_value_sincedb_collection(watched_file) sincedb_value = SincedbValue.new(0) sincedb_value.set_watched_file(watched_file) - logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position) + logger.trace? && logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position) sincedb_collection.set(watched_file.sincedb_key, sincedb_value) end end diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb index 2d3458a..b7dc19b 100644 --- a/lib/filewatch/sincedb_collection.rb +++ b/lib/filewatch/sincedb_collection.rb @@ -47,16 +47,16 @@ def open @time_sdb_opened = Time.now.to_f begin path.open do |file| - logger.trace("open: reading from #{path}") + logger.debug("open: reading from #{path}") @serializer.deserialize(file) do |key, value| - logger.trace("open: importing ... '#{key}' => '#{value}'") + logger.trace? && logger.trace("open: importing #{key.inspect} => #{value.inspect}") set_key_value(key, value) end end logger.trace("open: count of keys read: #{@sincedb.keys.size}") rescue => e #No existing sincedb to load - logger.trace("open: error:", :path => path, :exception => e.class, :message => e.message) + logger.debug("open: error opening #{path}", :exception => e.class, :message => e.message) end end @@ -68,35 +68,32 @@ def associate(watched_file) # and due to the window handling of many files # this file may not be opened in this session. # a new value will be added when the file is opened - logger.trace("associate: unmatched") + logger.trace("associate: unmatched", :filename => watched_file.filename) return true end logger.trace? && logger.trace("associate: found sincedb record", :filename => watched_file.filename, :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value) - if sincedb_value.watched_file.nil? - # not associated + if sincedb_value.watched_file.nil? # not associated if sincedb_value.path_in_sincedb.nil? handle_association(sincedb_value, watched_file) - logger.trace("associate: inode matched but no path in sincedb") + logger.trace? && logger.trace("associate: inode matched but no path in sincedb", :filename => watched_file.filename) return true end if sincedb_value.path_in_sincedb == watched_file.path - # the path on disk is the same as discovered path - # and the inode is the same. + # the path on disk is the same as discovered path and the inode is the same. handle_association(sincedb_value, watched_file) - logger.trace("associate: inode and path matched") + logger.trace? && logger.trace("associate: inode and path matched", :filename => watched_file.filename) return true end - # the path on disk is different from discovered unassociated path - # but they have the same key (inode) + # the path on disk is different from discovered unassociated path but they have the same key (inode) # treat as a new file, a new value will be added when the file is opened sincedb_value.clear_watched_file delete(watched_file.sincedb_key) - logger.trace("associate: matched but allocated to another") + logger.trace? && logger.trace("associate: matched but allocated to another", :filename => watched_file.filename) return true end if sincedb_value.watched_file.equal?(watched_file) # pointer equals - logger.trace("associate: already associated") + logger.trace? && logger.trace("associate: already associated", :filename => watched_file.filename) return true end # sincedb_value.watched_file is not this discovered watched_file but they have the same key (inode) @@ -107,7 +104,7 @@ def associate(watched_file) # after the original is deleted # are not yet in the delete phase, let this play out existing_watched_file = sincedb_value.watched_file - logger.trace? && logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename", + logger.trace? && logger.trace("associate: found sincedb_value has a watched_file - this is a rename", :this_watched_file => watched_file.details, :existing_watched_file => existing_watched_file.details) watched_file.rotation_in_progress true @@ -197,43 +194,43 @@ def handle_association(sincedb_value, watched_file) watched_file.initial_completed if watched_file.all_read? watched_file.ignore - logger.trace? && logger.trace("handle_association fully read, ignoring.....", :watched_file => watched_file.details, :sincedb_value => sincedb_value) + logger.trace? && logger.trace("handle_association fully read, ignoring", :watched_file => watched_file.details, :sincedb_value => sincedb_value) end end def set_key_value(key, value) if @time_sdb_opened < value.last_changed_at_expires(@settings.sincedb_expiry_duration) - logger.trace("open: setting #{key.inspect} to #{value.inspect}") set(key, value) else - logger.trace("open: record has expired, skipping: #{key.inspect} #{value.inspect}") + logger.debug("set_key_value: record has expired, skipping: #{key.inspect} => #{value.inspect}") end end def sincedb_write(time = Time.now) - logger.trace("sincedb_write: #{path} (time = #{time})") + logger.trace? && logger.trace("sincedb_write: #{path} (time = #{time})") begin - @write_method.call(time) - @serializer.expired_keys.each do |key| + expired_keys = @write_method.call(time) + expired_keys.each do |key| @sincedb[key].unset_watched_file delete(key) logger.trace? && logger.trace("sincedb_write: cleaned", :key => key) end @sincedb_last_write = time.to_i @write_requested = false - rescue Errno::EACCES - # no file handles free perhaps - # maybe it will work next time - logger.trace("sincedb_write: #{path} error: #{$!}") + rescue Errno::EACCES => e + # no file handles free perhaps - maybe it will work next time + logger.debug("sincedb_write: #{path} error:", :exception => e.class, :message => e.message) end end + # @return expired keys def atomic_write(time) FileHelper.write_atomically(@full_path) do |io| @serializer.serialize(@sincedb, io, time.to_f) end end + # @return expired keys def non_atomic_write(time) IO.open(IO.sysopen(@full_path, "w+")) do |io| @serializer.serialize(@sincedb, io, time.to_f) diff --git a/lib/filewatch/sincedb_record_serializer.rb b/lib/filewatch/sincedb_record_serializer.rb index 5d377b6..6dcc168 100644 --- a/lib/filewatch/sincedb_record_serializer.rb +++ b/lib/filewatch/sincedb_record_serializer.rb @@ -3,30 +3,25 @@ module FileWatch class SincedbRecordSerializer - attr_reader :expired_keys - def self.days_to_seconds(days) (24 * 3600) * days.to_f end def initialize(sincedb_value_expiry) @sincedb_value_expiry = sincedb_value_expiry - @expired_keys = [] - end - - def update_sincedb_value_expiry_from_days(days) - @sincedb_value_expiry = SincedbRecordSerializer.days_to_seconds(days) end + # @return Array expired keys (ones that were not written to the file) def serialize(db, io, as_of = Time.now.to_f) - @expired_keys.clear + expired_keys = [] db.each do |key, value| if as_of > value.last_changed_at_expires(@sincedb_value_expiry) - @expired_keys << key + expired_keys << key next end io.write(serialize_record(key, value)) end + expired_keys end def deserialize(io) @@ -36,8 +31,7 @@ def deserialize(io) end def serialize_record(k, v) - # effectively InodeStruct#to_s SincedbValue#to_s - "#{k} #{v}\n" + "#{k} #{v}\n" # effectively InodeStruct#to_s SincedbValue#to_s end def deserialize_record(record) diff --git a/lib/filewatch/tail_mode/handlers/base.rb b/lib/filewatch/tail_mode/handlers/base.rb index 5763fb9..4a6e7c2 100644 --- a/lib/filewatch/tail_mode/handlers/base.rb +++ b/lib/filewatch/tail_mode/handlers/base.rb @@ -18,7 +18,7 @@ def quit? end def handle(watched_file) - logger.trace("handling: #{watched_file.filename}") + logger.trace? && logger.trace("handling:", :path => watched_file.path) unless watched_file.has_listener? watched_file.set_listener(@observer) end @@ -37,7 +37,7 @@ def update_existing_specifically(watched_file, sincedb_value) def controlled_read(watched_file, loop_control) changed = false - logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename) + logger.trace? && logger.trace(__method__.to_s, :iterations => loop_control.count, :amount => loop_control.size, :filename => watched_file.filename) # from a real config (has 102 file inputs) # -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously # -- If we put all log files in one file input glob we will have indexing delay, because Logstash waits until the first file becomes EOF @@ -48,7 +48,7 @@ def controlled_read(watched_file, loop_control) loop_control.count.times do break if quit? begin - logger.debug("read_to_eof: get chunk") + logger.debug? && logger.debug("#{__method__} get chunk") result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult logger.trace(result.warning, result.additional) unless result.warning.empty? changed = true @@ -57,40 +57,42 @@ def controlled_read(watched_file, loop_control) # sincedb position is now independent from the watched_file bytes_read sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size) end - rescue EOFError + rescue EOFError => e # it only makes sense to signal EOF in "read" mode not "tail" + logger.debug(__method__.to_s, exception_details(watched_file.path, e, false)) loop_control.flag_read_error break - rescue Errno::EWOULDBLOCK, Errno::EINTR + rescue Errno::EWOULDBLOCK, Errno::EINTR => e + logger.debug(__method__.to_s, exception_details(watched_file.path, e, false)) watched_file.listener.error loop_control.flag_read_error break rescue => e - logger.error("read_to_eof: general error reading #{watched_file.path}", "error" => e.inspect, "backtrace" => e.backtrace.take(4)) + logger.error("#{__method__} general error reading", exception_details(watched_file.path, e)) watched_file.listener.error loop_control.flag_read_error break end end - logger.debug("read_to_eof: exit due to quit") if quit? + logger.debug("#{__method__} stopped loop due quit") if quit? sincedb_collection.request_disk_flush if changed end def open_file(watched_file) return true if watched_file.file_open? - logger.trace("opening #{watched_file.filename}") + logger.trace? && logger.trace("open_file", :filename => watched_file.filename) begin watched_file.open - rescue + rescue => e # don't emit this message too often. if a file that we can't # read is changing a lot, we'll try to open it more often, and spam the logs. now = Time.now.to_i - logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") + logger.trace? && logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'") if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL - logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}") + logger.warn("failed to open file", exception_details(watched_file.path, e)) watched_file.last_open_warning_at = now else - logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}") + logger.debug("open_file suppressed warning `failed to open file`", exception_details(watched_file.path, e, false)) end watched_file.watch # set it back to watch so we can try it again else @@ -108,26 +110,22 @@ def add_or_update_sincedb_collection(watched_file) update_existing_sincedb_collection_value(watched_file, sincedb_value) watched_file.initial_completed else - msg = "add_or_update_sincedb_collection: found sincedb record" - logger.trace(msg, - "sincedb key" => watched_file.sincedb_key, - "sincedb value" => sincedb_value - ) + logger.trace? && logger.trace("add_or_update_sincedb_collection: found sincedb record", + :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value) # detected a rotation, Discoverer can't handle this because this watched file is not a new discovery. # we must handle it here, by transferring state and have the sincedb value track this watched file # rotate_as_file and rotate_from will switch the sincedb key to the inode that the path is now pointing to # and pickup the sincedb_value from before. - msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file" - logger.trace(msg) + logger.debug("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file") existing_watched_file = sincedb_value.watched_file if existing_watched_file.nil? sincedb_value.set_watched_file(watched_file) - logger.trace("add_or_update_sincedb_collection: switching as new file") + logger.trace? && logger.trace("add_or_update_sincedb_collection: switching as new file") watched_file.rotate_as_file watched_file.update_bytes_read(sincedb_value.position) else sincedb_value.set_watched_file(watched_file) - logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details) + logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from:", :watched_file => watched_file.details) watched_file.rotate_from(existing_watched_file) end end @@ -135,13 +133,15 @@ def add_or_update_sincedb_collection(watched_file) end def update_existing_sincedb_collection_value(watched_file, sincedb_value) - logger.trace("update_existing_sincedb_collection_value: #{watched_file.filename}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}") + logger.trace? && logger.trace("update_existing_sincedb_collection_value", :position => sincedb_value.position, + :filename => watched_file.filename, :last_stat_size => watched_file.last_stat_size) update_existing_specifically(watched_file, sincedb_value) end def add_new_value_sincedb_collection(watched_file) sincedb_value = get_new_value_specifically(watched_file) - logger.trace("add_new_value_sincedb_collection", "position" => sincedb_value.position, "watched_file details" => watched_file.details) + logger.trace? && logger.trace("add_new_value_sincedb_collection", :position => sincedb_value.position, + :watched_file => watched_file.details) sincedb_collection.set(watched_file.sincedb_key, sincedb_value) sincedb_value end @@ -153,5 +153,14 @@ def get_new_value_specifically(watched_file) watched_file.update_bytes_read(position) value end + + private + + def exception_details(path, e, trace = true) + details = { :path => path, :exception => e.class, :message => e.message } + details[:backtrace] = e.backtrace if trace && logger.debug? + details + end + end end end end diff --git a/lib/filewatch/tail_mode/handlers/delete.rb b/lib/filewatch/tail_mode/handlers/delete.rb index 89b0224..5d203db 100644 --- a/lib/filewatch/tail_mode/handlers/delete.rb +++ b/lib/filewatch/tail_mode/handlers/delete.rb @@ -7,7 +7,7 @@ def handle_specifically(watched_file) # TODO consider trying to find the renamed file - it will have the same inode. # Needs a rotate scheme rename hint from user e.g. "-YYYY-MM-DD-N." or "..N" # send the found content to the same listener (stream identity) - logger.trace("delete", :path => watched_file.path, :watched_file => watched_file.details) + logger.trace? && logger.trace(__method__.to_s, :path => watched_file.path, :watched_file => watched_file.details) if watched_file.bytes_unread > 0 logger.warn(DATA_LOSS_WARNING, :path => watched_file.path, :unread_bytes => watched_file.bytes_unread) end diff --git a/lib/filewatch/tail_mode/handlers/shrink.rb b/lib/filewatch/tail_mode/handlers/shrink.rb index 1cd5a56..be9da9e 100644 --- a/lib/filewatch/tail_mode/handlers/shrink.rb +++ b/lib/filewatch/tail_mode/handlers/shrink.rb @@ -14,11 +14,10 @@ def handle_specifically(watched_file) end def update_existing_specifically(watched_file, sincedb_value) - # we have a match but size is smaller - # set all to zero + # we have a match but size is smaller - set all to zero watched_file.reset_bytes_unread sincedb_value.update_position(0) - logger.trace("update_existing_specifically: was truncated seeking to beginning", "watched file" => watched_file.details, "sincedb value" => sincedb_value) + logger.trace? && logger.trace("update_existing_specifically: was truncated seeking to beginning", :watched_file => watched_file.details, :sincedb_value => sincedb_value) end end end end end diff --git a/lib/filewatch/tail_mode/handlers/unignore.rb b/lib/filewatch/tail_mode/handlers/unignore.rb index 7b510fe..b59118b 100644 --- a/lib/filewatch/tail_mode/handlers/unignore.rb +++ b/lib/filewatch/tail_mode/handlers/unignore.rb @@ -13,9 +13,9 @@ def get_new_value_specifically(watched_file) # for file initially ignored their bytes_read was set to stat.size # use this value not the `start_new_files_at` for the position # logger.trace("get_new_value_specifically", "watched_file" => watched_file.inspect) - SincedbValue.new(watched_file.bytes_read).tap do |val| - val.set_watched_file(watched_file) - logger.trace("-------------------- >>>>> get_new_value_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => val) + SincedbValue.new(watched_file.bytes_read).tap do |sincedb_value| + sincedb_value.set_watched_file(watched_file) + logger.trace? && logger.trace("get_new_value_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value) end end @@ -26,7 +26,7 @@ def update_existing_specifically(watched_file, sincedb_value) # we will handle grow or shrink # for now we seek to where we were before the file got ignored (grow) # or to the start (shrink) - logger.trace("-------------------- >>>>> update_existing_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => sincedb_value) + logger.trace? && logger.trace("update_existing_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value) position = 0 if watched_file.shrunk? watched_file.update_bytes_read(0) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 725e074..074ce37 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -377,12 +377,12 @@ def post_process_this(event) def handle_deletable_path(path) return if tail_mode? return if @completed_file_handlers.empty? + @logger.debug? && @logger.debug(__method__.to_s, :path => path) @completed_file_handlers.each { |handler| handler.handle(path) } end def log_line_received(path, line) - return unless @logger.debug? - @logger.debug("Received line", :path => path, :text => line) + @logger.debug? && @logger.debug("Received line", :path => path, :text => line) end def stop diff --git a/lib/logstash/inputs/file_listener.rb b/lib/logstash/inputs/file_listener.rb index 3b6d15a..90aad4f 100644 --- a/lib/logstash/inputs/file_listener.rb +++ b/lib/logstash/inputs/file_listener.rb @@ -7,9 +7,9 @@ module LogStash module Inputs class FileListener attr_reader :input, :path, :data # construct with link back to the input plugin instance. - def initialize(path, input) + def initialize(path, input, data = nil) @path, @input = path, input - @data = nil + @data = data end def opened @@ -36,7 +36,7 @@ def deleted def accept(data) # and push transient data filled dup listener downstream input.log_line_received(path, data) - input.codec.accept(dup_adding_state(data)) + input.codec.accept(self.class.new(path, input, data)) end def process_event(event) @@ -45,17 +45,6 @@ def process_event(event) input.post_process_this(event) end - def add_state(data) - @data = data - self - end - - private - - # duplicate and add state for downstream - def dup_adding_state(line) - self.class.new(path, input).add_state(line) - end end class FlushableListener < FileListener diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index bc7afa6..97021de 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.2.2' + s.version = '4.2.3' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filewatch/sincedb_record_serializer_spec.rb b/spec/filewatch/sincedb_record_serializer_spec.rb index a751165..007dd1e 100644 --- a/spec/filewatch/sincedb_record_serializer_spec.rb +++ b/spec/filewatch/sincedb_record_serializer_spec.rb @@ -9,7 +9,9 @@ module FileWatch let(:io) { StringIO.new } let(:db) { Hash.new } - subject { SincedbRecordSerializer.new(SincedbRecordSerializer.days_to_seconds(14)) } + let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(14) } + + subject { SincedbRecordSerializer.new(sincedb_value_expiry) } context "deserialize from IO" do it 'reads V1 records' do @@ -82,8 +84,10 @@ module FileWatch end context "given a non default `sincedb_clean_after`" do + + let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(2) } + it "does not write expired db entries to an IO object" do - subject.update_sincedb_value_expiry_from_days(2) one_day_ago = Time.now.to_f - (1.0*24*3600) three_days_ago = one_day_ago - (2.0*24*3600) db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago) From be04655f4bcb535605cf48c13ffb331242670838 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 4 Jan 2021 12:49:20 +0000 Subject: [PATCH 73/91] [skip ci] update travis ci badge from .org to .com --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f4e337..7c81a58 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Logstash Plugin Travis Build -[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-file.svg)](https://travis-ci.org/logstash-plugins/logstash-input-file) +[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-file.svg)](https://travis-ci.com/logstash-plugins/logstash-input-file) This is a plugin for [Logstash](https://github.com/elastic/logstash). From bf4b8e573729000e97e4ccffbf8d792eef011d73 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 3 Mar 2021 10:53:01 +0000 Subject: [PATCH 74/91] [tests] change super to super() - jruby/jruby#6571 --- spec/filewatch/reading_spec.rb | 8 ++++---- spec/filewatch/rotate_spec.rb | 8 ++++---- spec/filewatch/tailing_spec.rb | 20 ++++++++++---------- spec/inputs/file_read_spec.rb | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/spec/filewatch/reading_spec.rb b/spec/filewatch/reading_spec.rb index 614950e..91f7b64 100644 --- a/spec/filewatch/reading_spec.rb +++ b/spec/filewatch/reading_spec.rb @@ -91,7 +91,7 @@ module FileWatch context "when watching a directory with files using striped reading" do let(:file_path2) { ::File.join(directory, "2.log") } # use a chunk size that does not align with the line boundaries - let(:opts) { super.merge(:file_chunk_size => 10, :file_chunk_count => 1, :file_sort_by => "path")} + let(:opts) { super().merge(:file_chunk_size => 10, :file_chunk_count => 1, :file_sort_by => "path")} let(:lines) { [] } let(:observer) { TestObserver.new(lines) } let(:listener2) { observer.listener_for(file_path2) } @@ -121,7 +121,7 @@ module FileWatch end context "when a non default delimiter is specified and it is not in the content" do - let(:opts) { super.merge(:delimiter => "\nø") } + let(:opts) { super().merge(:delimiter => "\nø") } let(:actions) do RSpec::Sequencing.run("create file") do File.open(file_path, "wb") { |file| file.write("line1\nline2") } @@ -154,7 +154,7 @@ module FileWatch let(:file_path2) { ::File.join(directory, "2.log") } let(:file_path3) { ::File.join(directory, "3.log") } - let(:opts) { super.merge(:file_sort_by => "last_modified") } + let(:opts) { super().merge(:file_sort_by => "last_modified") } let(:lines) { [] } let(:observer) { TestObserver.new(lines) } @@ -195,7 +195,7 @@ module FileWatch end context "when watching a directory with files using exit_after_read" do - let(:opts) { super.merge(:exit_after_read => true, :max_open_files => 2) } + let(:opts) { super().merge(:exit_after_read => true, :max_open_files => 2) } let(:file_path3) { ::File.join(directory, "3.log") } let(:file_path4) { ::File.join(directory, "4.log") } let(:file_path5) { ::File.join(directory, "5.log") } diff --git a/spec/filewatch/rotate_spec.rb b/spec/filewatch/rotate_spec.rb index f4e5979..cdef967 100644 --- a/spec/filewatch/rotate_spec.rb +++ b/spec/filewatch/rotate_spec.rb @@ -219,7 +219,7 @@ module FileWatch end context "create + rename rotation: when a new logfile is renamed to a path we have seen before but not all content from the previous the file is read" do - let(:opts) { super.merge( + let(:opts) { super().merge( :file_chunk_size => line1.bytesize.succ, :file_chunk_count => 1 ) } @@ -296,7 +296,7 @@ module FileWatch end context "copy + truncate rotation: when a logfile is copied to a new path and truncated before the open file is fully read" do - let(:opts) { super.merge( + let(:opts) { super().merge( :file_chunk_size => line1.bytesize.succ, :file_chunk_count => 1 ) } @@ -370,7 +370,7 @@ module FileWatch end context "? rotation: when an active file is renamed inside the glob and the reading lags behind" do - let(:opts) { super.merge( + let(:opts) { super().merge( :file_chunk_size => line1.bytesize.succ, :file_chunk_count => 2 ) } @@ -409,7 +409,7 @@ module FileWatch end context "? rotation: when a not active file is rotated outside the glob before the file is read" do - let(:opts) { super.merge( + let(:opts) { super().merge( :close_older => 3600, :max_open_files => 1, :file_sort_by => "path" diff --git a/spec/filewatch/tailing_spec.rb b/spec/filewatch/tailing_spec.rb index 6f4cf84..fbc4f9e 100644 --- a/spec/filewatch/tailing_spec.rb +++ b/spec/filewatch/tailing_spec.rb @@ -77,7 +77,7 @@ module FileWatch context "when close_older is set" do let(:wait_before_quit) { 0.8 } - let(:opts) { super.merge(:close_older => 0.1, :max_open_files => 1, :stat_interval => 0.1) } + let(:opts) { super().merge(:close_older => 0.1, :max_open_files => 1, :stat_interval => 0.1) } let(:suffix) { "B" } it "opens both files" do actions.activate_quietly @@ -278,7 +278,7 @@ module FileWatch context "when watching a directory with files and a file is renamed to match glob", :unix => true do let(:suffix) { "H" } - let(:opts) { super.merge(:close_older => 0) } + let(:opts) { super().merge(:close_older => 0) } let(:listener2) { observer.listener_for(file_path2) } let(:actions) do RSpec::Sequencing @@ -346,7 +346,7 @@ module FileWatch end context "when close older expiry is enabled" do - let(:opts) { super.merge(:close_older => 1) } + let(:opts) { super().merge(:close_older => 1) } let(:suffix) { "J" } let(:actions) do RSpec::Sequencing.run("create file") do @@ -370,7 +370,7 @@ module FileWatch end context "when close older expiry is enabled and after timeout the file is appended-to" do - let(:opts) { super.merge(:close_older => 0.5) } + let(:opts) { super().merge(:close_older => 0.5) } let(:suffix) { "K" } let(:actions) do RSpec::Sequencing @@ -406,7 +406,7 @@ module FileWatch end context "when ignore older expiry is enabled and all files are already expired" do - let(:opts) { super.merge(:ignore_older => 1) } + let(:opts) { super().merge(:ignore_older => 1) } let(:suffix) { "L" } let(:actions) do RSpec::Sequencing @@ -430,7 +430,7 @@ module FileWatch context "when a file is renamed before it gets activated", :unix => true do let(:max) { 1 } - let(:opts) { super.merge(:file_chunk_count => 8, :file_chunk_size => 6, :close_older => 0.1, :discover_interval => 6) } + let(:opts) { super().merge(:file_chunk_count => 8, :file_chunk_size => 6, :close_older => 0.1, :discover_interval => 6) } let(:suffix) { "M" } let(:start_new_files_at) { :beginning } # we are creating files and sincedb record before hand let(:actions) do @@ -469,7 +469,7 @@ module FileWatch end context "when ignore_older is less than close_older and all files are not expired" do - let(:opts) { super.merge(:ignore_older => 1, :close_older => 1.1) } + let(:opts) { super().merge(:ignore_older => 1, :close_older => 1.1) } let(:suffix) { "N" } let(:start_new_files_at) { :beginning } let(:actions) do @@ -497,7 +497,7 @@ module FileWatch end context "when ignore_older is less than close_older and all files are expired" do - let(:opts) { super.merge(:ignore_older => 10, :close_older => 1) } + let(:opts) { super().merge(:ignore_older => 10, :close_older => 1) } let(:suffix) { "P" } let(:actions) do RSpec::Sequencing @@ -522,7 +522,7 @@ module FileWatch end context "when ignore older and close older expiry is enabled and after timeout the file is appended-to" do - let(:opts) { super.merge(:ignore_older => 20, :close_older => 0.5) } + let(:opts) { super().merge(:ignore_older => 20, :close_older => 0.5) } let(:suffix) { "Q" } let(:actions) do RSpec::Sequencing @@ -551,7 +551,7 @@ module FileWatch end context "when a non default delimiter is specified and it is not in the content" do - let(:opts) { super.merge(:ignore_older => 20, :close_older => 1, :delimiter => "\nø") } + let(:opts) { super().merge(:ignore_older => 20, :close_older => 1, :delimiter => "\nø") } let(:suffix) { "R" } let(:actions) do RSpec::Sequencing diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index d1533bd..54c3685 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -267,7 +267,7 @@ describe 'delete on complete' do let(:options) do - super.merge({ 'file_completed_action' => "delete", 'exit_after_read' => false }) + super().merge({ 'file_completed_action' => "delete", 'exit_after_read' => false }) end let(:sample_file) { File.join(temp_directory, "sample.log") } @@ -306,7 +306,7 @@ describe 'sincedb cleanup' do let(:options) do - super.merge( + super().merge( 'sincedb_path' => sincedb_path, 'sincedb_clean_after' => '1.0 seconds', 'sincedb_write_interval' => 0.25, From 51a88e1b1ebbcea74042b1d1c1b4526fb3a22dbe Mon Sep 17 00:00:00 2001 From: Rishie Sharma Date: Tue, 23 Mar 2021 06:36:05 +0100 Subject: [PATCH 75/91] Fix: occasional sincedb write issue on Windows machines (#283) On Windows servers we occasionally get exceptions `unknown IOException: java.io.IOException: The handle is invalid` when writing to sincedb file which crashes the plugin and somehow messes with the sincedb file which results in old log files being reprocessed causing duplicates. This change eliminates those exceptions. --- CHANGELOG.md | 3 +++ lib/filewatch/sincedb_collection.rb | 2 +- logstash-input-file.gemspec | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2d1b0d..e850de7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.2.4 + - Fix: sincedb_write issue on Windows machines [#283](https://github.com/logstash-plugins/logstash-input-file/pull/283) + ## 4.2.3 - Refactor: improve debug logging (log catched exceptions) [#280](https://github.com/logstash-plugins/logstash-input-file/pull/280) diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb index b7dc19b..d053b27 100644 --- a/lib/filewatch/sincedb_collection.rb +++ b/lib/filewatch/sincedb_collection.rb @@ -232,7 +232,7 @@ def atomic_write(time) # @return expired keys def non_atomic_write(time) - IO.open(IO.sysopen(@full_path, "w+")) do |io| + File.open(@full_path, "w+") do |io| @serializer.serialize(@sincedb, io, time.to_f) end end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 97021de..7c0044b 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.2.3' + s.version = '4.2.4' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 9be8dc26356c40050a06b074acf33d009fecf90e Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Fri, 30 Apr 2021 08:38:24 -0700 Subject: [PATCH 76/91] Add ECS Compatibility mode (#291) * ecs: add ECS Compatibility mode * Apply suggestions from code review Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- CHANGELOG.md | 3 ++ docs/index.asciidoc | 30 +++++++++++ lib/logstash/inputs/file.rb | 23 ++++++++- lib/logstash/inputs/file_listener.rb | 1 - logstash-input-file.gemspec | 3 +- spec/inputs/file_tail_spec.rb | 76 ++++++++++++++++++---------- 6 files changed, 105 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e850de7..3ee2d78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.3.0 + - Add ECS Compatibility Mode [#291](https://github.com/logstash-plugins/logstash-input-file/pull/291) + ## 4.2.4 - Fix: sincedb_write issue on Windows machines [#283](https://github.com/logstash-plugins/logstash-input-file/pull/283) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index d0926ca..4f3add2 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -78,6 +78,21 @@ Read mode also allows for an action to take place after processing the file comp In the past attempts to simulate a Read mode while still assuming infinite streams was not ideal and a dedicated Read mode is an improvement. +[id="plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin adds metadata about event's source, and can be configured to do so +in an {ecs-ref}[ECS-compatible] way with <>. +This metadata is added after the event has been decoded by the appropriate codec, +and will never overwrite existing values. + +|======== +| ECS Disabled | ECS v1 | Description + +| `host` | `[host][name]` | The name of the {ls} host that processed the event +| `path` | `[log][file][path]` | The full path to the log file from which the event originates +|======== + ==== Tracking of current position in watched files The plugin keeps track of the current position in each file by @@ -168,6 +183,7 @@ see <> for the details | <> |<> or <>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -242,6 +258,20 @@ This value is a multiple to `stat_interval`, e.g. if `stat_interval` is "500 ms" files could be discovered every 15 X 500 milliseconds - 7.5 seconds. In practice, this will be the best case because the time taken to read new content needs to be factored in. +[id="plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is <> +* Supported values are: +** `disabled`: sets non-ECS metadata on event (such as top-level `host`, `path`) +** `v1`: sets ECS-compatible metadata on event (such as `[host][name]`, `[log][file][path]`) +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the +{ecs-ref}[Elastic Common Schema (ECS)]. + [id="plugins-{type}s-{plugin}-exclude"] ===== `exclude` diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 074ce37..d305df6 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -2,6 +2,7 @@ require "logstash/namespace" require "logstash/inputs/base" require "logstash/codecs/identity_map_codec" +require 'logstash/plugin_mixins/ecs_compatibility_support' require "pathname" require "socket" # for Socket.gethostname @@ -88,6 +89,8 @@ module LogStash module Inputs class File < LogStash::Inputs::Base config_name "file" + include PluginMixins::ECSCompatibilitySupport(:disabled, :v1) + # The path(s) to the file(s) to use as an input. # You can use filename patterns here, such as `/var/log/*.log`. # If you use a pattern like `/var/log/**/*.log`, a recursive search @@ -325,6 +328,9 @@ def register @codec = LogStash::Codecs::IdentityMapCodec.new(@codec) @completely_stopped = Concurrent::AtomicBoolean.new @queue = Concurrent::AtomicReference.new + + @source_host_field = ecs_select[disabled: 'host', v1:'[host][name]'] + @source_path_field = ecs_select[disabled: 'path', v1:'[log][file][path]'] end # def register def completely_stopped? @@ -369,7 +375,11 @@ def run(queue) def post_process_this(event) event.set("[@metadata][host]", @host) - event.set("host", @host) unless event.include?("host") + attempt_set(event, @source_host_field, @host) + + source_path = event.get('[@metadata][path]') and + attempt_set(event, @source_path_field, source_path) + decorate(event) @queue.get << event end @@ -407,6 +417,17 @@ def build_sincedb_base_from_settings(settings) end end + # Attempt to set an event's field to the provided value + # without overwriting an existing value or producing an error + def attempt_set(event, field_reference, value) + return false if event.include?(field_reference) + + event.set(field_reference, value) + rescue => e + logger.trace("failed to set #{field_reference} to `#{value}`", :exception => e.message) + false + end + def build_sincedb_base_from_env # This section is going to be deprecated eventually, as path.data will be # the default, not an environment variable (SINCEDB_DIR or LOGSTASH_HOME) diff --git a/lib/logstash/inputs/file_listener.rb b/lib/logstash/inputs/file_listener.rb index 90aad4f..5cf689d 100644 --- a/lib/logstash/inputs/file_listener.rb +++ b/lib/logstash/inputs/file_listener.rb @@ -41,7 +41,6 @@ def accept(data) def process_event(event) event.set("[@metadata][path]", path) - event.set("path", path) unless event.include?("path") input.post_process_this(event) end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 7c0044b..94d6e7b 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.2.4' + s.version = '4.3.0' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -33,6 +33,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'concurrent-ruby', '~> 1.0' s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0'] + s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1' s.add_development_dependency 'stud', ['~> 0.0.19'] s.add_development_dependency 'logstash-devutils' diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index 7aefc69..fecb904 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -3,7 +3,9 @@ require "helpers/spec_helper" require "logstash/devutils/rspec/shared_examples" require "logstash/inputs/file" +require "logstash/plugin_mixins/ecs_compatibility_support/spec_helper" +require "json" require "tempfile" require "stud/temporary" require "logstash/codecs/multiline" @@ -99,41 +101,59 @@ end end - context "when path and host fields exist" do - let(:name) { "C" } - it "should not overwrite them" do - conf = <<-CONFIG - input { - file { - type => "blah" - path => "#{path_path}" - start_position => "beginning" - sincedb_path => "#{sincedb_path}" - delimiter => "#{TEST_FILE_DELIMITER}" - codec => "json" - } - } - CONFIG - File.open(tmpfile_path, "w") do |fd| - fd.puts('{"path": "my_path", "host": "my_host"}') - fd.puts('{"my_field": "my_val"}') - fd.fsync + context "when path and host fields exist", :ecs_compatibility_support do + ecs_compatibility_matrix(:disabled, :v1) do |ecs_select| + + before(:each) do + allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) end - events = input(conf) do |pipeline, queue| - 2.times.collect { queue.pop } + let(:file_path_target_field ) { ecs_select[disabled: "path", v1: '[log][file][path]'] } + let(:source_host_target_field) { ecs_select[disabled: "host", v1: '[host][name]'] } + + let(:event_with_existing) do + LogStash::Event.new.tap do |e| + e.set(file_path_target_field, 'my_path') + e.set(source_host_target_field, 'my_host') + end.to_hash end - existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1] + let(:name) { "C" } + it "should not overwrite them" do + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{path_path}" + start_position => "beginning" + sincedb_path => "#{sincedb_path}" + delimiter => "#{TEST_FILE_DELIMITER}" + codec => "json" + } + } + CONFIG - expect(events[existing_path_index].get("path")).to eq "my_path" - expect(events[existing_path_index].get("host")).to eq "my_host" - expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + File.open(tmpfile_path, "w") do |fd| + fd.puts(event_with_existing.to_json) + fd.puts('{"my_field": "my_val"}') + fd.fsync + end - expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}" - expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" - expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + events = input(conf) do |pipeline, queue| + 2.times.collect { queue.pop } + end + + existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1] + + expect(events[existing_path_index].get(file_path_target_field)).to eq "my_path" + expect(events[existing_path_index].get(source_host_target_field)).to eq "my_host" + expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + + expect(events[added_path_index].get(file_path_target_field)).to eq "#{tmpfile_path}" + expect(events[added_path_index].get(source_host_target_field)).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + end end end From 36245480f3bf2ff6fac4a3d39df7e6e31e01bdb0 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 9 Jun 2021 16:57:01 +0200 Subject: [PATCH 77/91] Refactor: unify event updates to happen in one place (#297) + Test: a try that actually re-tries on `RSpec::Expectations::ExpectationNotMetError` + Test: re-try instead of relying on timeout --- lib/logstash/inputs/file.rb | 7 +++---- lib/logstash/inputs/file_listener.rb | 3 +-- spec/inputs/file_read_spec.rb | 9 ++++++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index d305df6..f1e7405 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -373,12 +373,11 @@ def run(queue) @completely_stopped.make_true end # def run - def post_process_this(event) + def post_process_this(event, path) + event.set("[@metadata][path]", path) event.set("[@metadata][host]", @host) attempt_set(event, @source_host_field, @host) - - source_path = event.get('[@metadata][path]') and - attempt_set(event, @source_path_field, source_path) + attempt_set(event, @source_path_field, path) if path decorate(event) @queue.get << event diff --git a/lib/logstash/inputs/file_listener.rb b/lib/logstash/inputs/file_listener.rb index 5cf689d..5a264a6 100644 --- a/lib/logstash/inputs/file_listener.rb +++ b/lib/logstash/inputs/file_listener.rb @@ -40,8 +40,7 @@ def accept(data) end def process_event(event) - event.set("[@metadata][path]", path) - input.post_process_this(event) + input.post_process_this(event, path) end end diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index 54c3685..02bf83d 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -338,7 +338,7 @@ sincedb_content = File.read(sincedb_path).strip expect( sincedb_content ).to_not be_empty - Stud.try(3.times) do + try(3) do sleep(1.5) # > sincedb_clean_after sincedb_content = File.read(sincedb_path).strip @@ -363,7 +363,10 @@ def wait_for_start_processing(run_thread, timeout: 1.0) end end - def wait_for_file_removal(path, timeout: 3 * interval) - wait(timeout).for { File.exist?(path) }.to be_falsey + def wait_for_file_removal(path) + timeout = interval + try(5) do + wait(timeout).for { File.exist?(path) }.to be_falsey + end end end From 55aac0a4d5827e047e19580397ad4c09fc907b72 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 9 Jun 2021 11:45:22 -0400 Subject: [PATCH 78/91] Only try and set permissions if permissions have changed. (#295) * Only try and set permissions if permissions have changed. This is a workaround for https://github.com/jruby/jruby/issues/6693 - stat is currently reporting incorrect values of `uid` and `gid` on aarch64 linux nodes, and appears to be setting `uid` and `gid` to 0, ie `root` It is highly unlikely that `chown` needs to be called on the file - and even more unlikely that `chown` would succeed anyway - `chmod` typically requires superuser privileges, unless the change of ownerhip is a change of group to another group the user is already a member of. This workaround updates the code to only attempt to call `chown` in the unlikely event that file ownership of the temp file is different from the original file, which should avoid the scenario that is currently occurring due to the above jruby bug. This commit also falls back to a non-atomic write in the event of a failure to write the file atomically. --- CHANGELOG.md | 7 +++++++ lib/filewatch/helper.rb | 7 +++++-- lib/filewatch/sincedb_collection.rb | 15 +++++++++++++-- logstash-input-file.gemspec | 2 +- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ee2d78..302f30b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 4.3.1 + - Add extra safety to `chown` call in `atomic_write`, avoiding plugin crashes and falling back to a + `non_atomic_write` in the event of failure [#295](https://github.com/logstash-plugins/logstash-input-file/pull/295) + - Refactor: unify event updates to happen in one place [#297](https://github.com/logstash-plugins/logstash-input-file/pull/297) + - Test: Actually retry tests on `RSpec::Expectations::ExpectationNotMetError` and retry instead of relying on timeout + [#297](https://github.com/logstash-plugins/logstash-input-file/pull/297) + ## 4.3.0 - Add ECS Compatibility Mode [#291](https://github.com/logstash-plugins/logstash-input-file/pull/291) diff --git a/lib/filewatch/helper.rb b/lib/filewatch/helper.rb index 5da8c0f..2fbe1a2 100644 --- a/lib/filewatch/helper.rb +++ b/lib/filewatch/helper.rb @@ -30,6 +30,7 @@ def write_atomically(file_name) temp_file.binmode return_val = yield temp_file temp_file.close + new_stat = File.stat(temp_file) # Overwrite original file with temp file File.rename(temp_file.path, file_name) @@ -37,8 +38,10 @@ def write_atomically(file_name) # Unable to get permissions of the original file => return return return_val if old_stat.nil? - # Set correct uid/gid on new file - File.chown(old_stat.uid, old_stat.gid, file_name) if old_stat + # Set correct uid/gid on new file if ownership is different. + if old_stat && (old_stat.gid != new_stat.gid || old_stat.uid != new_stat.uid) + File.chown(old_stat.uid, old_stat.gid, file_name) if old_stat + end return_val end diff --git a/lib/filewatch/sincedb_collection.rb b/lib/filewatch/sincedb_collection.rb index d053b27..3694cef 100644 --- a/lib/filewatch/sincedb_collection.rb +++ b/lib/filewatch/sincedb_collection.rb @@ -225,13 +225,24 @@ def sincedb_write(time = Time.now) # @return expired keys def atomic_write(time) - FileHelper.write_atomically(@full_path) do |io| - @serializer.serialize(@sincedb, io, time.to_f) + logger.trace? && logger.trace("non_atomic_write: ", :time => time) + begin + FileHelper.write_atomically(@full_path) do |io| + @serializer.serialize(@sincedb, io, time.to_f) + end + rescue Errno::EPERM, Errno::EACCES => e + logger.warn("sincedb_write: unable to write atomically due to permissions error, falling back to non-atomic write: #{path} error:", :exception => e.class, :message => e.message) + @write_method = method(:non_atomic_write) + non_atomic_write(time) + rescue => e + logger.warn("sincedb_write: unable to write atomically, attempting non-atomic write: #{path} error:", :exception => e.class, :message => e.message) + non_atomic_write(time) end end # @return expired keys def non_atomic_write(time) + logger.trace? && logger.trace("non_atomic_write: ", :time => time) File.open(@full_path, "w+") do |io| @serializer.serialize(@sincedb, io, time.to_f) end diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 94d6e7b..b5cf6c3 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.3.0' + s.version = '4.3.1' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 7200a7aefe125d050e144ed862fc792ca93142bc Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 4 Aug 2021 09:41:01 -0700 Subject: [PATCH 79/91] ecs: add v8 alias to v1 implementation (#301) * ecs: add v8 alias to v1 implementation * version bump and changelog --- CHANGELOG.md | 3 +++ docs/index.asciidoc | 4 ++-- lib/logstash/inputs/file.rb | 2 +- logstash-input-file.gemspec | 4 ++-- spec/inputs/file_tail_spec.rb | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 302f30b..0d0112e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.4.0 + - Add support for ECS v8 [#301](https://github.com/logstash-plugins/logstash-input-file/pull/301) + ## 4.3.1 - Add extra safety to `chown` call in `atomic_write`, avoiding plugin crashes and falling back to a `non_atomic_write` in the event of failure [#295](https://github.com/logstash-plugins/logstash-input-file/pull/295) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 4f3add2..de84ea2 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -87,7 +87,7 @@ This metadata is added after the event has been decoded by the appropriate codec and will never overwrite existing values. |======== -| ECS Disabled | ECS v1 | Description +| ECS Disabled | ECS `v1`, `v8` | Description | `host` | `[host][name]` | The name of the {ls} host that processed the event | `path` | `[log][file][path]` | The full path to the log file from which the event originates @@ -264,7 +264,7 @@ In practice, this will be the best case because the time taken to read new conte * Value type is <> * Supported values are: ** `disabled`: sets non-ECS metadata on event (such as top-level `host`, `path`) -** `v1`: sets ECS-compatible metadata on event (such as `[host][name]`, `[log][file][path]`) +** `v1`,`v8`: sets ECS-compatible metadata on event (such as `[host][name]`, `[log][file][path]`) * Default value depends on which version of Logstash is running: ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default ** Otherwise, the default value is `disabled`. diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index f1e7405..fe489f1 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -89,7 +89,7 @@ module LogStash module Inputs class File < LogStash::Inputs::Base config_name "file" - include PluginMixins::ECSCompatibilitySupport(:disabled, :v1) + include PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1) # The path(s) to the file(s) to use as an input. # You can use filename patterns here, such as `/var/log/*.log`. diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index b5cf6c3..09b7fe6 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.3.1' + s.version = '4.4.0' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -33,7 +33,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'concurrent-ruby', '~> 1.0' s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0'] - s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1' + s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3' s.add_development_dependency 'stud', ['~> 0.0.19'] s.add_development_dependency 'logstash-devutils' diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index fecb904..f4ceb39 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -103,7 +103,7 @@ context "when path and host fields exist", :ecs_compatibility_support do - ecs_compatibility_matrix(:disabled, :v1) do |ecs_select| + ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select| before(:each) do allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) From 17997011acab6134269f4af9c1a951d175d8f396 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Thu, 28 Oct 2021 11:51:17 +0200 Subject: [PATCH 80/91] Update to Gradle 7.2 (#305) Update the Gradle wrapper used to version 7.2 and fixed build.gradle script. --- CHANGELOG.md | 3 +++ build.gradle | 4 ++-- gradle/wrapper/gradle-wrapper.properties | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d0112e..13c50d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## Unreleased + - Fix: update to Gradle 7 [#305](https://github.com/logstash-plugins/logstash-input-file/pull/305) + ## 4.4.0 - Add support for ECS v8 [#301](https://github.com/logstash-plugins/logstash-input-file/pull/301) diff --git a/build.gradle b/build.gradle index 47ff4f5..2cfbc47 100644 --- a/build.gradle +++ b/build.gradle @@ -37,13 +37,13 @@ dependencies { task sourcesJar(type: Jar, dependsOn: classes) { from sourceSets.main.allSource classifier 'sources' - extension 'jar' + archiveExtension = 'jar' } task javadocJar(type: Jar, dependsOn: javadoc) { from javadoc.destinationDir classifier 'javadoc' - extension 'jar' + archiveExtension = 'jar' } task copyGemjar(type: Copy, dependsOn: sourcesJar) { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 5028f28..ffed3a2 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 1891b34c9ca9a2c0cdba8537a5bb03a69dd6782d Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Mon, 2 May 2022 09:18:11 +0200 Subject: [PATCH 81/91] Fix failing tests (#309) There are two failures that this PR fixes: - tests that referred host or path attribute running against Logstash 8, which by default has ECS enabled. The fix consisted in surrounding the test with the ECS harness and use the ecs_select to refer attributes with proper name. - another fix regards the test having timed_out, the codec is auto flushed. It presented 2 problems: - the first is that to request the subject.stop to trigger the auto_flush call (called by exit_flush method). - the second regard a regression introduced with eliminate PR https://github.com/logstash-plugins/logstash-codec-multiline/pull/70. The test worked because the the IdentityMapCodec used a shared codec. When this behavior was removed the test, that engage multiple threads, exposed a problem of overwriting of codec reference. This overwriting drove to the lost of data, used in test logic. The fix consists in clone redefinition to make it almost a singleton. --- spec/helpers/spec_helper.rb | 2 +- spec/inputs/file_tail_spec.rb | 310 +++++++++++++++++++--------------- 2 files changed, 174 insertions(+), 138 deletions(-) diff --git a/spec/helpers/spec_helper.rb b/spec/helpers/spec_helper.rb index b2f0ddd..ddef385 100644 --- a/spec/helpers/spec_helper.rb +++ b/spec/helpers/spec_helper.rb @@ -57,7 +57,7 @@ def close @tracer.push [:close, true] end def clone - self.class.new + self end end end diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index f4ceb39..73354df 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -157,39 +157,49 @@ end end - context "running the input twice" do - let(:name) { "D" } - it "should read old files" do - conf = <<-CONFIG - input { - file { - type => "blah" - path => "#{path_path}" - start_position => "beginning" - codec => "json" - } - } - CONFIG + context "running the input twice", :ecs_compatibility_support do + ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select| - File.open(tmpfile_path, "w") do |fd| - fd.puts('{"path": "my_path", "host": "my_host"}') - fd.puts('{"my_field": "my_val"}') - fd.fsync + before(:each) do + allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) end - # arbitrary old file (2 days) - FileInput.make_file_older(tmpfile_path, 48 * 60 * 60) + + let(:file_path_target_field ) { ecs_select[disabled: "path", v1: '[log][file][path]'] } + let(:source_host_target_field) { ecs_select[disabled: "host", v1: '[host][name]'] } + + let(:name) { "D" } + it "should read old files" do + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{path_path}" + start_position => "beginning" + codec => "json" + } + } + CONFIG - events = input(conf) do |pipeline, queue| - 2.times.collect { queue.pop } + File.open(tmpfile_path, "w") do |fd| + fd.puts('{"path": "my_path", "host": "my_host"}') + fd.puts('{"my_field": "my_val"}') + fd.fsync + end + # arbitrary old file (2 days) + FileInput.make_file_older(tmpfile_path, 48 * 60 * 60) + + events = input(conf) do |pipeline, queue| + 2.times.collect { queue.pop } + end + existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1] + expect(events[existing_path_index].get("path")).to eq "my_path" + expect(events[existing_path_index].get("host")).to eq "my_host" + expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + + expect(events[added_path_index].get(file_path_target_field)).to eq "#{tmpfile_path}" + expect(events[added_path_index].get(source_host_target_field)).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" + expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" end - existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1] - expect(events[existing_path_index].get("path")).to eq "my_path" - expect(events[existing_path_index].get("host")).to eq "my_host" - expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" - - expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}" - expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" - expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}" end end @@ -233,54 +243,62 @@ FileUtils.rm_rf(sincedb_path) end - context "when data exists and then more data is appended" do - subject { described_class.new(conf) } + context "when data exists and then more data is appended", :ecs_compatibility_support do + ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select| - before do - File.open(tmpfile_path, "w") do |fd| - fd.puts("ignore me 1") - fd.puts("ignore me 2") - fd.fsync + before(:each) do + allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) end - mlconf.update("pattern" => "^\s", "what" => "previous") - conf.update("type" => "blah", - "path" => path_path, - "sincedb_path" => sincedb_path, - "stat_interval" => 0.1, - "codec" => mlcodec, - "delimiter" => TEST_FILE_DELIMITER) - end - it "reads the appended data only" do - subject.register - actions = RSpec::Sequencing - .run_after(1, "append two lines after delay") do - File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") } - end - .then("wait for one event") do - wait(0.75).for{events.size}.to eq(1) - end - .then("quit") do - subject.stop - end - .then("wait for flushed event") do - wait(0.75).for{events.size}.to eq(2) + let(:file_path_target_field ) { ecs_select[disabled: "path", v1: '[log][file][path]'] } + subject { described_class.new(conf) } + + before do + File.open(tmpfile_path, "w") do |fd| + fd.puts("ignore me 1") + fd.puts("ignore me 2") + fd.fsync end + mlconf.update("pattern" => "^\s", "what" => "previous") + conf.update("type" => "blah", + "path" => path_path, + "sincedb_path" => sincedb_path, + "stat_interval" => 0.1, + "codec" => mlcodec, + "delimiter" => TEST_FILE_DELIMITER) + end - subject.run(events) - actions.assert_no_errors + it "reads the appended data only" do + subject.register + actions = RSpec::Sequencing + .run_after(1, "append two lines after delay") do + File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") } + end + .then("wait for one event") do + wait(0.75).for{events.size}.to eq(1) + end + .then("quit") do + subject.stop + end + .then("wait for flushed event") do + wait(0.75).for{events.size}.to eq(2) + end - event1 = events[0] - expect(event1).not_to be_nil - expect(event1.get("path")).to eq tmpfile_path - expect(event1.get("[@metadata][path]")).to eq tmpfile_path - expect(event1.get("message")).to eq "hello" - - event2 = events[1] - expect(event2).not_to be_nil - expect(event2.get("path")).to eq tmpfile_path - expect(event2.get("[@metadata][path]")).to eq tmpfile_path - expect(event2.get("message")).to eq "world" + subject.run(events) + actions.assert_no_errors + + event1 = events[0] + expect(event1).not_to be_nil + expect(event1.get(file_path_target_field)).to eq tmpfile_path + expect(event1.get("[@metadata][path]")).to eq tmpfile_path + expect(event1.get("message")).to eq "hello" + + event2 = events[1] + expect(event2).not_to be_nil + expect(event2.get(file_path_target_field)).to eq tmpfile_path + expect(event2.get("[@metadata][path]")).to eq tmpfile_path + expect(event2.get("message")).to eq "world" + end end end @@ -311,12 +329,21 @@ .then_after(0.1, "identity is mapped") do wait(0.75).for{subject.codec.identity_map[tmpfile_path]}.not_to be_nil, "identity is not mapped" end - .then("wait for auto_flush") do - wait(0.75).for{subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)}.to eq([true]), "autoflush didn't" + .then("wait accept") do + wait(0.75).for { + subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept) + }.to eq([true]), "accept didn't" end - .then("quit") do + .then("request a stop") do + # without this the subject.run doesn't invokes the #exit_flush which is the only @codec.flush_mapped invocation subject.stop end + .then("wait for auto_flush") do + wait(2).for { + subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush) + .reduce {|b1, b2| b1 and b2} # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]] + }.to eq(true), "autoflush didn't" + end subject.run(events) actions.assert_no_errors expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true]) @@ -356,74 +383,50 @@ end end - context "when wildcard path and a multiline codec is specified" do - subject { described_class.new(conf) } - let(:suffix) { "J" } - let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") } - before do - mlconf.update("pattern" => "^\s", "what" => "previous") - conf.update( - "type" => "blah", - "path" => path_path, - "start_position" => "beginning", - "sincedb_path" => sincedb_path, - "stat_interval" => 0.05, - "codec" => mlcodec, - "file_sort_by" => "path", - "delimiter" => TEST_FILE_DELIMITER) + context "when wildcard path and a multiline codec is specified", :ecs_compatibility_support do + ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select| - subject.register - end + before(:each) do + allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) + end - it "collects separate multiple line events from each file" do - subject - actions = RSpec::Sequencing - .run_after(0.1, "create files") do - File.open(tmpfile_path, "wb") do |fd| - fd.puts("line1.1-of-J") - fd.puts(" line1.2-of-J") - fd.puts(" line1.3-of-J") - end - File.open(tmpfile_path2, "wb") do |fd| - fd.puts("line1.1-of-K") - fd.puts(" line1.2-of-K") - fd.puts(" line1.3-of-K") - end - end - .then("assert both files are mapped as identities and stop") do - wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities" - end - .then("stop") do - subject.stop - end - subject.run(events) - # wait for actions to complete - actions.assert_no_errors - expect(events.size).to eq(2) - e1, e2 = events - e1_message = e1.get("message") - e2_message = e2.get("message") - - expect(e1.get("path")).to match(/J.txt/) - expect(e2.get("path")).to match(/K.txt/) - expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER} line1.2-of-J#{TEST_FILE_DELIMITER} line1.3-of-J") - expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER} line1.2-of-K#{TEST_FILE_DELIMITER} line1.3-of-K") - end + let(:file_path_target_field ) { ecs_select[disabled: "path", v1: '[log][file][path]'] } - context "if auto_flush is enabled on the multiline codec" do - let(:mlconf) { { "auto_flush_interval" => 0.5 } } - let(:suffix) { "M" } - it "an event is generated via auto_flush" do + subject { described_class.new(conf) } + let(:suffix) { "J" } + let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") } + before do + mlconf.update("pattern" => "^\s", "what" => "previous") + conf.update( + "type" => "blah", + "path" => path_path, + "start_position" => "beginning", + "sincedb_path" => sincedb_path, + "stat_interval" => 0.05, + "codec" => mlcodec, + "file_sort_by" => "path", + "delimiter" => TEST_FILE_DELIMITER) + + subject.register + end + + it "collects separate multiple line events from each file" do + subject actions = RSpec::Sequencing .run_after(0.1, "create files") do File.open(tmpfile_path, "wb") do |fd| - fd.puts("line1.1-of-a") - fd.puts(" line1.2-of-a") - fd.puts(" line1.3-of-a") + fd.puts("line1.1-of-J") + fd.puts(" line1.2-of-J") + fd.puts(" line1.3-of-J") + end + File.open(tmpfile_path2, "wb") do |fd| + fd.puts("line1.1-of-K") + fd.puts(" line1.2-of-K") + fd.puts(" line1.3-of-K") end end - .then("wait for auto_flush") do - wait(2).for{events.size}.to eq(1), "events size is not 1" + .then("assert both files are mapped as identities and stop") do + wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities" end .then("stop") do subject.stop @@ -431,10 +434,43 @@ subject.run(events) # wait for actions to complete actions.assert_no_errors - e1 = events.first + expect(events.size).to eq(2) + e1, e2 = events e1_message = e1.get("message") - expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") - expect(e1.get("path")).to match(/M.txt$/) + e2_message = e2.get("message") + + expect(e1.get(file_path_target_field)).to match(/J.txt/) + expect(e2.get(file_path_target_field)).to match(/K.txt/) + expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER} line1.2-of-J#{TEST_FILE_DELIMITER} line1.3-of-J") + expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER} line1.2-of-K#{TEST_FILE_DELIMITER} line1.3-of-K") + end + + context "if auto_flush is enabled on the multiline codec" do + let(:mlconf) { { "auto_flush_interval" => 0.5 } } + let(:suffix) { "M" } + it "an event is generated via auto_flush" do + actions = RSpec::Sequencing + .run_after(0.1, "create files") do + File.open(tmpfile_path, "wb") do |fd| + fd.puts("line1.1-of-a") + fd.puts(" line1.2-of-a") + fd.puts(" line1.3-of-a") + end + end + .then("wait for auto_flush") do + wait(2).for{events.size}.to eq(1), "events size is not 1" + end + .then("stop") do + subject.stop + end + subject.run(events) + # wait for actions to complete + actions.assert_no_errors + e1 = events.first + e1_message = e1.get("message") + expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a") + expect(e1.get(file_path_target_field)).to match(/M.txt$/) + end end end end From 66b8851d9ba36d005c115408d2f8d54f48f9945f Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Wed, 11 May 2022 08:51:14 -0400 Subject: [PATCH 82/91] Doc: Set version attribute in plugin source file (#308) Update links to long format to pick up version Bump to v4.4.1 Note that we're resetting the version attribute to `current` at the end of the file to avoid carry over to other files. After we resolve formatting and clean up older files, we can remove the reset in the source file. --- CHANGELOG.md | 3 ++- docs/index.asciidoc | 18 ++++++++++++++---- logstash-input-file.gemspec | 2 +- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13c50d6..c2e51d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ -## Unreleased +## 4.4.1 - Fix: update to Gradle 7 [#305](https://github.com/logstash-plugins/logstash-input-file/pull/305) + - [DOC] Add version attributes to doc source file [#308](https://github.com/logstash-plugins/logstash-input-file/pull/308) ## 4.4.0 - Add support for ECS v8 [#301](https://github.com/logstash-plugins/logstash-input-file/pull/301) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index de84ea2..f883fbb 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -9,6 +9,11 @@ START - GENERATED VARIABLES, DO NOT EDIT! :release_date: %RELEASE_DATE% :changelog_url: %CHANGELOG_URL% :include_path: ../../../../logstash/docs/include + +ifeval::["{versioned_docs}"=="true"] +:branch: %BRANCH% +:ecs_version: %ECS_VERSION% +endif::[] /////////////////////////////////////////// END - GENERATED VARIABLES, DO NOT EDIT! /////////////////////////////////////////// @@ -82,7 +87,7 @@ was not ideal and a dedicated Read mode is an improvement. ==== Compatibility with the Elastic Common Schema (ECS) This plugin adds metadata about event's source, and can be configured to do so -in an {ecs-ref}[ECS-compatible] way with <>. +in an https://www.elastic.co/guide/en/ecs/{ecs_version}/index.html[ECS-compatible] way with <>. This metadata is added after the event has been decoded by the appropriate codec, and will never overwrite existing values. @@ -270,7 +275,7 @@ In practice, this will be the best case because the time taken to read new conte ** Otherwise, the default value is `disabled`. Controls this plugin's compatibility with the -{ecs-ref}[Elastic Common Schema (ECS)]. +https://www.elastic.co/guide/en/ecs/{ecs_version}/index.html[Elastic Common Schema (ECS)]. [id="plugins-{type}s-{plugin}-exclude"] ===== `exclude` @@ -426,7 +431,7 @@ of `/var/log` will be done for all `*.log` files. Paths must be absolute and cannot be relative. You may also configure multiple paths. See an example -on the {logstash-ref}/configuration-file-structure.html#array[Logstash configuration page]. +on the https://www.elastic.co/guide/en/logstash/{branch}/configuration-file-structure.html#array[Logstash configuration page]. [id="plugins-{type}s-{plugin}-sincedb_clean_after"] ===== `sincedb_clean_after` @@ -439,7 +444,7 @@ The sincedb record now has a last active timestamp associated with it. If no changes are detected in a tracked file in the last N days its sincedb tracking record expires and will not be persisted. This option helps protect against the inode recycling problem. -Filebeat has a {filebeat-ref}/inode-reuse-issue.html[FAQ about inode recycling]. +Filebeat has an https://www.elastic.co/guide/en/beats/filebeat/{branch}}/inode-reuse-issue.html[FAQ about inode recycling]. [id="plugins-{type}s-{plugin}-sincedb_path"] ===== `sincedb_path` @@ -533,4 +538,9 @@ Supported values: `us` `usec` `usecs`, e.g. "600 us", "800 usec", "900 usecs" [NOTE] `micro` `micros` and `microseconds` are not supported +ifeval::["{versioned_docs}"=="true"] +:branch: current +:ecs_version: current +endif::[] + :default_codec!: diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 09b7fe6..3ccb681 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.4.0' + s.version = '4.4.1' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From e3924d6217fce2862fe820668bfd137b327854fb Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Wed, 11 May 2022 18:17:00 -0400 Subject: [PATCH 83/91] Doc: Fix attribute by removing extra character (#310) Bump to 4.4.2 Fix changelog link to point to PR --- CHANGELOG.md | 3 +++ docs/index.asciidoc | 2 +- logstash-input-file.gemspec | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2e51d2..67e0ccc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.4.2 + - Doc: Fix attribute by removing extra character [#310](https://github.com/logstash-plugins/logstash-input-file/pull/310) + ## 4.4.1 - Fix: update to Gradle 7 [#305](https://github.com/logstash-plugins/logstash-input-file/pull/305) - [DOC] Add version attributes to doc source file [#308](https://github.com/logstash-plugins/logstash-input-file/pull/308) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index f883fbb..98a62ab 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -444,7 +444,7 @@ The sincedb record now has a last active timestamp associated with it. If no changes are detected in a tracked file in the last N days its sincedb tracking record expires and will not be persisted. This option helps protect against the inode recycling problem. -Filebeat has an https://www.elastic.co/guide/en/beats/filebeat/{branch}}/inode-reuse-issue.html[FAQ about inode recycling]. +Filebeat has an https://www.elastic.co/guide/en/beats/filebeat/{branch}/inode-reuse-issue.html[FAQ about inode recycling]. [id="plugins-{type}s-{plugin}-sincedb_path"] ===== `sincedb_path` diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 3ccb681..8ad4318 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.4.1' + s.version = '4.4.2' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From ef9b8d5279a6c1610cf522b2accc2ef6b7d19d60 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Mon, 6 Jun 2022 11:19:50 +0200 Subject: [PATCH 84/91] Fix ReadFile handler to consider the value stored in sincedb on plugin restart (#307) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes read mode to restart the read from reference stored in sincedb in case the file wasn't completely consumed Update the file pointer of a read mode file to the max between the read bytes or the sincedb reference for the same file. This solves a problem, that when a pipeline is restarted, it's able to recover from the last known reference, without restarting from the beginning, and reprocessing already processed lines. Co-authored-by: João Duarte --- CHANGELOG.md | 3 ++ lib/filewatch/read_mode/handlers/read_file.rb | 10 +++++ logstash-input-file.gemspec | 2 +- .../read_mode_handlers_read_file_spec.rb | 40 +++++++++++++++++++ spec/filewatch/spec_helper.rb | 2 + spec/helpers/spec_helper.rb | 8 +++- spec/inputs/file_tail_spec.rb | 5 +-- 7 files changed, 65 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67e0ccc..688d2a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.4.3 + - Fixes read mode to restart the read from reference stored in sincedb in case the file wasn't completely consumed. [#307](https://github.com/logstash-plugins/logstash-input-file/pull/307) + ## 4.4.2 - Doc: Fix attribute by removing extra character [#310](https://github.com/logstash-plugins/logstash-input-file/pull/310) diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index 77cfe9f..2b6cd2b 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -2,9 +2,19 @@ module FileWatch module ReadMode module Handlers class ReadFile < Base + + # seek file to which ever is furthest: either current bytes read or sincedb position + private + def seek_to_furthest_position(watched_file) + previous_pos = sincedb_collection.find(watched_file).position + watched_file.file_seek([watched_file.bytes_read, previous_pos].max) + end + + public def handle_specifically(watched_file) if open_file(watched_file) add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key) + seek_to_furthest_position(watched_file) loop do break if quit? loop_control = watched_file.loop_control_adjusted_for_stat_size diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 8ad4318..25bde8e 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.4.2' + s.version = '4.4.3' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filewatch/read_mode_handlers_read_file_spec.rb b/spec/filewatch/read_mode_handlers_read_file_spec.rb index f1a6ff5..5b0a16e 100644 --- a/spec/filewatch/read_mode_handlers_read_file_spec.rb +++ b/spec/filewatch/read_mode_handlers_read_file_spec.rb @@ -36,5 +36,45 @@ module FileWatch processor.read_file(watched_file) end end + + context "when restart from existing sincedb" do + let(:settings) do + Settings.from_options( + :sincedb_write_interval => 0, + :sincedb_path => File::NULL, + :file_chunk_size => 10 + ) + end + + let(:processor) { double("fake processor") } + let(:observer) { TestObserver.new } + let(:watch) { double("watch") } + + before(:each) { + allow(watch).to receive(:quit?).and_return(false)#.and_return(false).and_return(true) + allow(processor).to receive(:watch).and_return(watch) + } + + it "read from where it left" do + listener = observer.listener_for(Pathname.new(pathname).to_path) + sut = ReadMode::Handlers::ReadFile.new(processor, sdb_collection, observer, settings) + + # simulate a previous partial read of the file + sincedb_value = SincedbValue.new(0) + sincedb_value.set_watched_file(watched_file) + sdb_collection.set(watched_file.sincedb_key, sincedb_value) + + + # simulate a consumption of first line, (size + newline) bytes + sdb_collection.increment(watched_file.sincedb_key, File.readlines(pathname)[0].size + 2) + + # exercise + sut.handle(watched_file) + + # verify + expect(listener.lines.size).to eq(1) + expect(listener.lines[0]).to start_with("2010-03-12 23:51:21 SEA4 192.0.2.222 play 3914 OK") + end + end end end diff --git a/spec/filewatch/spec_helper.rb b/spec/filewatch/spec_helper.rb index 4ebb1f7..f6c6c98 100644 --- a/spec/filewatch/spec_helper.rb +++ b/spec/filewatch/spec_helper.rb @@ -80,6 +80,8 @@ def sysread(amount) multiplier = amount / string.length string * multiplier end + def sysseek(offset, whence) + end end FIXTURE_DIR = File.join('spec', 'fixtures') diff --git a/spec/helpers/spec_helper.rb b/spec/helpers/spec_helper.rb index ddef385..404c93f 100644 --- a/spec/helpers/spec_helper.rb +++ b/spec/helpers/spec_helper.rb @@ -31,7 +31,13 @@ def initialize def trace_for(symbol) params = @tracer.map {|k,v| k == symbol ? v : nil}.compact - params.empty? ? false : params + if params.empty? + false + else + # merge all params with same key + # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]] + params.reduce {|b1, b2| b1 and b2} + end end def clear diff --git a/spec/inputs/file_tail_spec.rb b/spec/inputs/file_tail_spec.rb index 73354df..8dae213 100644 --- a/spec/inputs/file_tail_spec.rb +++ b/spec/inputs/file_tail_spec.rb @@ -332,7 +332,7 @@ .then("wait accept") do wait(0.75).for { subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept) - }.to eq([true]), "accept didn't" + }.to eq(true), "accept didn't" end .then("request a stop") do # without this the subject.run doesn't invokes the #exit_flush which is the only @codec.flush_mapped invocation @@ -341,12 +341,11 @@ .then("wait for auto_flush") do wait(2).for { subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush) - .reduce {|b1, b2| b1 and b2} # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]] }.to eq(true), "autoflush didn't" end subject.run(events) actions.assert_no_errors - expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true]) + expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq(true) end end From dcecc68877d27be373083b4254949df89c6109cc Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Tue, 14 Jun 2022 09:17:00 +0200 Subject: [PATCH 85/91] Avoid to call a Bufferedreader 's package-private and switch to the public version.(#312) Package private methods mustn't be called also if JRuby permit us. In JDK 11 `readLine()` invokes `readLine(false)`m in JDK 17 the package private `readLine(boolean, boolean[])` takes one more parameter and this breaks the tests. This commit made use of the public API. --- lib/filewatch/read_mode/handlers/read_zip_file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/filewatch/read_mode/handlers/read_zip_file.rb b/lib/filewatch/read_mode/handlers/read_zip_file.rb index b9d8d2d..10b3486 100644 --- a/lib/filewatch/read_mode/handlers/read_zip_file.rb +++ b/lib/filewatch/read_mode/handlers/read_zip_file.rb @@ -29,7 +29,7 @@ def handle_specifically(watched_file) gzip_stream = GZIPInputStream.new(file_stream) decoder = InputStreamReader.new(gzip_stream, "UTF-8") buffered = BufferedReader.new(decoder) - while (line = buffered.readLine(false)) + while (line = buffered.readLine()) watched_file.listener.accept(line) # can't quit, if we did then we would incorrectly write a 'completed' sincedb entry # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading) From c543a2e9e12b6b07709a4e306c29dbecb660f991 Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Mon, 11 Jul 2022 17:34:06 -0400 Subject: [PATCH 86/91] Doc: Remove conditional test text (#313) --- docs/index.asciidoc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 98a62ab..159f7d0 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -9,11 +9,6 @@ START - GENERATED VARIABLES, DO NOT EDIT! :release_date: %RELEASE_DATE% :changelog_url: %CHANGELOG_URL% :include_path: ../../../../logstash/docs/include - -ifeval::["{versioned_docs}"=="true"] -:branch: %BRANCH% -:ecs_version: %ECS_VERSION% -endif::[] /////////////////////////////////////////// END - GENERATED VARIABLES, DO NOT EDIT! /////////////////////////////////////////// @@ -538,9 +533,4 @@ Supported values: `us` `usec` `usecs`, e.g. "600 us", "800 usec", "900 usecs" [NOTE] `micro` `micros` and `microseconds` are not supported -ifeval::["{versioned_docs}"=="true"] -:branch: current -:ecs_version: current -endif::[] - :default_codec!: From 9fe467cdee1c87220b13ffacc14766018a94fd66 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Tue, 20 Sep 2022 00:44:53 -0700 Subject: [PATCH 87/91] release previously-merged fix for JDK12+ support (#315) --- CHANGELOG.md | 3 +++ logstash-input-file.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 688d2a8..b08bb04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.4.4 + - Fixes gzip file handling in read mode when run on JDK12+, including JDK17 that is bundled with Logstash 8.4+ [#312](https://github.com/logstash-plugins/logstash-input-file/pull/312) + ## 4.4.3 - Fixes read mode to restart the read from reference stored in sincedb in case the file wasn't completely consumed. [#307](https://github.com/logstash-plugins/logstash-input-file/pull/307) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 25bde8e..f6b66ea 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.4.3' + s.version = '4.4.4' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 58b8fb7077686ea4af8209e10c6dc0b30a34beb3 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Mon, 17 Apr 2023 14:12:41 +0200 Subject: [PATCH 88/91] Added Gradle's vendoring task already present in other mixed Java/Ruby plugins (#320) This PR is to keep the plugin aligned to others that uses a mix of Ruby and Java, and to provide standard task sets. With this PR, after vendored the plugin with ./gradle vendor the plugin is usable directly in development mode, setting :path in Gemfile --- build.gradle | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2cfbc47..d6c229a 100644 --- a/build.gradle +++ b/build.gradle @@ -17,6 +17,9 @@ * under the License. */ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + apply plugin: "java" apply plugin: "distribution" apply plugin: "idea" @@ -28,7 +31,8 @@ repositories { mavenCentral() } -project.sourceCompatibility = 1.8 +project.sourceCompatibility = JavaVersion.VERSION_1_8 +project.targetCompatibility = JavaVersion.VERSION_1_8 dependencies { compileOnly group: 'org.jruby', name: 'jruby-complete', version: "9.1.13.0" @@ -59,3 +63,37 @@ task cleanGemjar { clean.dependsOn(cleanGemjar) jar.finalizedBy(copyGemjar) + + +task generateGemJarRequiresFile { + doLast { + File jars_file = file('lib/logstash-input-file_jars.rb') + jars_file.newWriter().withWriter { w -> + w << "# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.\n\n" + w << "require \'jar_dependencies\'\n" + configurations.runtimeClasspath.allDependencies.each { + w << "require_jar(\'${it.group}\', \'${it.name}\', \'${it.version}\')\n" + } + w << "\nrequire_jar(\'${project.group}\', \'${project.name}\', \'${project.version}\')\n" + } + } +} + +task vendor { + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + configurations.runtimeClasspath.allDependencies.each { dep -> + File f = configurations.runtimeClasspath.filter { it.absolutePath.contains("${dep.group}/${dep.name}/${dep.version}") }.singleFile + String groupPath = dep.group.replaceAll('\\.', '/') + File newJarFile = file("${vendorPathPrefix}/${groupPath}/${dep.name}/${dep.version}/${dep.name}-${dep.version}.jar") + newJarFile.mkdirs() + Files.copy(f.toPath(), newJarFile.toPath(), REPLACE_EXISTING) + } + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${project.name}/${project.version}/${project.name}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + } +} + +vendor.dependsOn(jar, generateGemJarRequiresFile) \ No newline at end of file From b2dd26249737c6dc426ef8efdda1c4227d9f6c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Wed, 18 Oct 2023 14:40:13 +0100 Subject: [PATCH 89/91] handle EOF when checking archive validity (#321) --- CHANGELOG.md | 3 ++ .../read_mode/handlers/read_zip_file.rb | 4 +-- logstash-input-file.gemspec | 2 +- spec/helpers/spec_helper.rb | 6 ++++ spec/inputs/file_read_spec.rb | 31 +++++++++++++++++++ 5 files changed, 43 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b08bb04..e33ec63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.4.5 + - Handle EOF when checking archive validity [#321](https://github.com/logstash-plugins/logstash-input-file/pull/321) + ## 4.4.4 - Fixes gzip file handling in read mode when run on JDK12+, including JDK17 that is bundled with Logstash 8.4+ [#312](https://github.com/logstash-plugins/logstash-input-file/pull/312) diff --git a/lib/filewatch/read_mode/handlers/read_zip_file.rb b/lib/filewatch/read_mode/handlers/read_zip_file.rb index 10b3486..e7eb21f 100644 --- a/lib/filewatch/read_mode/handlers/read_zip_file.rb +++ b/lib/filewatch/read_mode/handlers/read_zip_file.rb @@ -71,14 +71,14 @@ def close_and_ignore_ioexception(closeable) def corrupted?(watched_file) begin + start = Time.new file_stream = FileInputStream.new(watched_file.path) gzip_stream = GZIPInputStream.new(file_stream) buffer = Java::byte[8192].new - start = Time.new until gzip_stream.read(buffer) == -1 end return false - rescue ZipException => e + rescue ZipException, Java::JavaIo::EOFException => e duration = Time.now - start logger.warn("Detected corrupted archive #{watched_file.path} file won't be processed", :message => e.message, :duration => duration.round(3)) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index f6b66ea..4f10bcd 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.4.4' + s.version = '4.4.5' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/helpers/spec_helper.rb b/spec/helpers/spec_helper.rb index 404c93f..095b327 100644 --- a/spec/helpers/spec_helper.rb +++ b/spec/helpers/spec_helper.rb @@ -24,6 +24,12 @@ def self.corrupt_gzip(file_path) f.close() end + def self.truncate_gzip(file_path) + f = File.open(file_path, "ab") + f.truncate(100) + f.close() + end + class TracerBase def initialize @tracer = Concurrent::Array.new diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index 02bf83d..3cd54de 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -245,6 +245,37 @@ expect(IO.read(log_completed_path)).to be_empty end end + + it "the truncated file is untouched" do + directory = Stud::Temporary.directory + file_path = fixture_dir.join('compressed.log.gz') + truncated_file_path = ::File.join(directory, 'truncated.gz') + FileUtils.cp(file_path, truncated_file_path) + + FileInput.truncate_gzip(truncated_file_path) + + log_completed_path = ::File.join(directory, "C_completed.txt") + f = File.new(log_completed_path, "w") + f.close() + + conf = <<-CONFIG + input { + file { + type => "blah" + path => "#{truncated_file_path}" + mode => "read" + file_completed_action => "log_and_delete" + file_completed_log_path => "#{log_completed_path}" + check_archive_validity => true + } + } + CONFIG + + events = input(conf) do |pipeline, queue| + wait(1) + expect(IO.read(log_completed_path)).to be_empty + end + end end end From 70aadf2510490ffafb17a11ed931dde6476f2211 Mon Sep 17 00:00:00 2001 From: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:43:00 +0100 Subject: [PATCH 90/91] Change read mode to immediately stop consuming lines when shutting down (#322) Changed read mode to immediately stop consuming buffered lines when quit is requested --- CHANGELOG.md | 3 ++ lib/filewatch/read_mode/handlers/read_file.rb | 1 + logstash-input-file.gemspec | 2 +- spec/inputs/file_read_spec.rb | 46 ++++++++----------- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e33ec63..255c5fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.4.6 + - Change read mode to immediately stop consuming buffered lines when shutdown is requested [#322](https://github.com/logstash-plugins/logstash-input-file/pull/322) + ## 4.4.5 - Handle EOF when checking archive validity [#321](https://github.com/logstash-plugins/logstash-input-file/pull/321) diff --git a/lib/filewatch/read_mode/handlers/read_file.rb b/lib/filewatch/read_mode/handlers/read_file.rb index 2b6cd2b..824ac2c 100644 --- a/lib/filewatch/read_mode/handlers/read_file.rb +++ b/lib/filewatch/read_mode/handlers/read_file.rb @@ -54,6 +54,7 @@ def controlled_read(watched_file, loop_control) # sincedb position is independent from the watched_file bytes_read delta = line.bytesize + @settings.delimiter_byte_size sincedb_collection.increment(watched_file.sincedb_key, delta) + break if quit? end rescue EOFError => e log_error("controlled_read: eof error reading file", watched_file, e) diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 4f10bcd..032112f 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '4.4.5' + s.version = '4.4.6' s.licenses = ['Apache-2.0'] s.summary = "Streams events from files" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/inputs/file_read_spec.rb b/spec/inputs/file_read_spec.rb index 3cd54de..ef820b9 100644 --- a/spec/inputs/file_read_spec.rb +++ b/spec/inputs/file_read_spec.rb @@ -181,25 +181,27 @@ end context "for a compressed file" do + let(:tmp_directory) { Stud::Temporary.directory } + let(:all_files_path) { fixture_dir.join("compressed.*.*") } + let(:gz_file_path) { fixture_dir.join('compressed.log.gz') } + let(:gzip_file_path) { fixture_dir.join('compressed.log.gzip') } + let(:sincedb_path) { ::File.join(tmp_directory, "sincedb.db") } + let(:log_completed_path) { ::File.join(tmp_directory, "completed.log") } + it "the file is read" do - file_path = fixture_dir.join('compressed.log.gz') - file_path2 = fixture_dir.join('compressed.log.gzip') - FileInput.make_fixture_current(file_path.to_path) - FileInput.make_fixture_current(file_path2.to_path) - tmpfile_path = fixture_dir.join("compressed.*.*") - directory = Stud::Temporary.directory - sincedb_path = ::File.join(directory, "readmode_C_sincedb.txt") - log_completed_path = ::File.join(directory, "C_completed.txt") + FileInput.make_fixture_current(gz_file_path.to_path) + FileInput.make_fixture_current(gzip_file_path.to_path) conf = <<-CONFIG input { file { type => "blah" - path => "#{tmpfile_path}" + path => "#{all_files_path}" sincedb_path => "#{sincedb_path}" mode => "read" file_completed_action => "log" file_completed_log_path => "#{log_completed_path}" + exit_after_read => true } } CONFIG @@ -216,17 +218,11 @@ end it "the corrupted file is untouched" do - directory = Stud::Temporary.directory - file_path = fixture_dir.join('compressed.log.gz') - corrupted_file_path = ::File.join(directory, 'corrupted.gz') - FileUtils.cp(file_path, corrupted_file_path) + corrupted_file_path = ::File.join(tmp_directory, 'corrupted.gz') + FileUtils.cp(gz_file_path, corrupted_file_path) FileInput.corrupt_gzip(corrupted_file_path) - log_completed_path = ::File.join(directory, "C_completed.txt") - f = File.new(log_completed_path, "w") - f.close() - conf = <<-CONFIG input { file { @@ -236,28 +232,23 @@ file_completed_action => "log_and_delete" file_completed_log_path => "#{log_completed_path}" check_archive_validity => true + exit_after_read => true } } CONFIG - events = input(conf) do |pipeline, queue| + input(conf) do |pipeline, queue| wait(1) expect(IO.read(log_completed_path)).to be_empty end end it "the truncated file is untouched" do - directory = Stud::Temporary.directory - file_path = fixture_dir.join('compressed.log.gz') - truncated_file_path = ::File.join(directory, 'truncated.gz') - FileUtils.cp(file_path, truncated_file_path) + truncated_file_path = ::File.join(tmp_directory, 'truncated.gz') + FileUtils.cp(gz_file_path, truncated_file_path) FileInput.truncate_gzip(truncated_file_path) - log_completed_path = ::File.join(directory, "C_completed.txt") - f = File.new(log_completed_path, "w") - f.close() - conf = <<-CONFIG input { file { @@ -267,11 +258,12 @@ file_completed_action => "log_and_delete" file_completed_log_path => "#{log_completed_path}" check_archive_validity => true + exit_after_read => true } } CONFIG - events = input(conf) do |pipeline, queue| + input(conf) do |pipeline, queue| wait(1) expect(IO.read(log_completed_path)).to be_empty end From 55a4a7099f05f29351672417036c1342850c7adc Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Wed, 22 May 2024 15:24:31 +0200 Subject: [PATCH 91/91] Upgrade Gradle to 8.7 and fix deprecations (#326) Update Gradle to 8.7 and fixes deprecation warnings. --- build.gradle | 28 +-- gradle/wrapper/gradle-wrapper.jar | Bin 54413 -> 43453 bytes gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 301 ++++++++++++++--------- gradlew.bat | 76 +++--- 5 files changed, 245 insertions(+), 164 deletions(-) diff --git a/build.gradle b/build.gradle index d6c229a..a0d33cb 100644 --- a/build.gradle +++ b/build.gradle @@ -20,9 +20,11 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING -apply plugin: "java" -apply plugin: "distribution" -apply plugin: "idea" +plugins { + id 'java' + id 'distribution' + id 'idea' +} group = 'org.logstash.filewatch' version file("JAR_VERSION").text.replaceAll("\\s","") @@ -31,25 +33,17 @@ repositories { mavenCentral() } -project.sourceCompatibility = JavaVersion.VERSION_1_8 -project.targetCompatibility = JavaVersion.VERSION_1_8 +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + withSourcesJar() + withJavadocJar() +} dependencies { compileOnly group: 'org.jruby', name: 'jruby-complete', version: "9.1.13.0" } -task sourcesJar(type: Jar, dependsOn: classes) { - from sourceSets.main.allSource - classifier 'sources' - archiveExtension = 'jar' -} - -task javadocJar(type: Jar, dependsOn: javadoc) { - from javadoc.destinationDir - classifier 'javadoc' - archiveExtension = 'jar' -} - task copyGemjar(type: Copy, dependsOn: sourcesJar) { from project.jar into project.file('lib/jars/') diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 0d4a9516871afd710a9d84d89e31ba77745607bd..e6441136f3d4ba8a0da8d277868979cfbc8ad796 100644 GIT binary patch literal 43453 zcma&N1CXTcmMvW9vTb(Rwr$&4wr$(C?dmSu>@vG-+vuvg^_??!{yS%8zW-#zn-LkA z5&1^$^{lnmUON?}LBF8_K|(?T0Ra(xUH{($5eN!MR#ZihR#HxkUPe+_R8Cn`RRs(P z_^*#_XlXmGv7!4;*Y%p4nw?{bNp@UZHv1?Um8r6)Fei3p@ClJn0ECfg1hkeuUU@Or zDaPa;U3fE=3L}DooL;8f;P0ipPt0Z~9P0)lbStMS)ag54=uL9ia-Lm3nh|@(Y?B`; zx_#arJIpXH!U{fbCbI^17}6Ri*H<>OLR%c|^mh8+)*h~K8Z!9)DPf zR2h?lbDZQ`p9P;&DQ4F0sur@TMa!Y}S8irn(%d-gi0*WxxCSk*A?3lGh=gcYN?FGl z7D=Js!i~0=u3rox^eO3i@$0=n{K1lPNU zwmfjRVmLOCRfe=seV&P*1Iq=^i`502keY8Uy-WNPwVNNtJFx?IwAyRPZo2Wo1+S(xF37LJZ~%i)kpFQ3Fw=mXfd@>%+)RpYQLnr}B~~zoof(JVm^^&f zxKV^+3D3$A1G;qh4gPVjhrC8e(VYUHv#dy^)(RoUFM?o%W-EHxufuWf(l*@-l+7vt z=l`qmR56K~F|v<^Pd*p~1_y^P0P^aPC##d8+HqX4IR1gu+7w#~TBFphJxF)T$2WEa zxa?H&6=Qe7d(#tha?_1uQys2KtHQ{)Qco)qwGjrdNL7thd^G5i8Os)CHqc>iOidS} z%nFEDdm=GXBw=yXe1W-ShHHFb?Cc70+$W~z_+}nAoHFYI1MV1wZegw*0y^tC*s%3h zhD3tN8b=Gv&rj}!SUM6|ajSPp*58KR7MPpI{oAJCtY~JECm)*m_x>AZEu>DFgUcby z1Qaw8lU4jZpQ_$;*7RME+gq1KySGG#Wql>aL~k9tLrSO()LWn*q&YxHEuzmwd1?aAtI zBJ>P=&$=l1efe1CDU;`Fd+_;&wI07?V0aAIgc(!{a z0Jg6Y=inXc3^n!U0Atk`iCFIQooHqcWhO(qrieUOW8X(x?(RD}iYDLMjSwffH2~tB z)oDgNBLB^AJBM1M^c5HdRx6fBfka`(LD-qrlh5jqH~);#nw|iyp)()xVYak3;Ybik z0j`(+69aK*B>)e_p%=wu8XC&9e{AO4c~O1U`5X9}?0mrd*m$_EUek{R?DNSh(=br# z#Q61gBzEpmy`$pA*6!87 zSDD+=@fTY7<4A?GLqpA?Pb2z$pbCc4B4zL{BeZ?F-8`s$?>*lXXtn*NC61>|*w7J* z$?!iB{6R-0=KFmyp1nnEmLsA-H0a6l+1uaH^g%c(p{iT&YFrbQ$&PRb8Up#X3@Zsk zD^^&LK~111%cqlP%!_gFNa^dTYT?rhkGl}5=fL{a`UViaXWI$k-UcHJwmaH1s=S$4 z%4)PdWJX;hh5UoK?6aWoyLxX&NhNRqKam7tcOkLh{%j3K^4Mgx1@i|Pi&}<^5>hs5 zm8?uOS>%)NzT(%PjVPGa?X%`N2TQCKbeH2l;cTnHiHppPSJ<7y-yEIiC!P*ikl&!B z%+?>VttCOQM@ShFguHVjxX^?mHX^hSaO_;pnyh^v9EumqSZTi+#f&_Vaija0Q-e*| z7ulQj6Fs*bbmsWp{`auM04gGwsYYdNNZcg|ph0OgD>7O}Asn7^Z=eI>`$2*v78;sj-}oMoEj&@)9+ycEOo92xSyY344^ z11Hb8^kdOvbf^GNAK++bYioknrpdN>+u8R?JxG=!2Kd9r=YWCOJYXYuM0cOq^FhEd zBg2puKy__7VT3-r*dG4c62Wgxi52EMCQ`bKgf*#*ou(D4-ZN$+mg&7$u!! z-^+Z%;-3IDwqZ|K=ah85OLwkO zKxNBh+4QHh)u9D?MFtpbl)us}9+V!D%w9jfAMYEb>%$A;u)rrI zuBudh;5PN}_6J_}l55P3l_)&RMlH{m!)ai-i$g)&*M`eN$XQMw{v^r@-125^RRCF0 z^2>|DxhQw(mtNEI2Kj(;KblC7x=JlK$@78`O~>V!`|1Lm-^JR$-5pUANAnb(5}B}JGjBsliK4& zk6y(;$e&h)lh2)L=bvZKbvh@>vLlreBdH8No2>$#%_Wp1U0N7Ank!6$dFSi#xzh|( zRi{Uw%-4W!{IXZ)fWx@XX6;&(m_F%c6~X8hx=BN1&q}*( zoaNjWabE{oUPb!Bt$eyd#$5j9rItB-h*5JiNi(v^e|XKAj*8(k<5-2$&ZBR5fF|JA z9&m4fbzNQnAU}r8ab>fFV%J0z5awe#UZ|bz?Ur)U9bCIKWEzi2%A+5CLqh?}K4JHi z4vtM;+uPsVz{Lfr;78W78gC;z*yTch~4YkLr&m-7%-xc ztw6Mh2d>_iO*$Rd8(-Cr1_V8EO1f*^@wRoSozS) zy1UoC@pruAaC8Z_7~_w4Q6n*&B0AjOmMWa;sIav&gu z|J5&|{=a@vR!~k-OjKEgPFCzcJ>#A1uL&7xTDn;{XBdeM}V=l3B8fE1--DHjSaxoSjNKEM9|U9#m2<3>n{Iuo`r3UZp;>GkT2YBNAh|b z^jTq-hJp(ebZh#Lk8hVBP%qXwv-@vbvoREX$TqRGTgEi$%_F9tZES@z8Bx}$#5eeG zk^UsLBH{bc2VBW)*EdS({yw=?qmevwi?BL6*=12k9zM5gJv1>y#ML4!)iiPzVaH9% zgSImetD@dam~e>{LvVh!phhzpW+iFvWpGT#CVE5TQ40n%F|p(sP5mXxna+Ev7PDwA zamaV4m*^~*xV+&p;W749xhb_X=$|LD;FHuB&JL5?*Y2-oIT(wYY2;73<^#46S~Gx| z^cez%V7x$81}UWqS13Gz80379Rj;6~WdiXWOSsdmzY39L;Hg3MH43o*y8ibNBBH`(av4|u;YPq%{R;IuYow<+GEsf@R?=@tT@!}?#>zIIn0CoyV!hq3mw zHj>OOjfJM3F{RG#6ujzo?y32m^tgSXf@v=J$ELdJ+=5j|=F-~hP$G&}tDZsZE?5rX ztGj`!S>)CFmdkccxM9eGIcGnS2AfK#gXwj%esuIBNJQP1WV~b~+D7PJTmWGTSDrR` zEAu4B8l>NPuhsk5a`rReSya2nfV1EK01+G!x8aBdTs3Io$u5!6n6KX%uv@DxAp3F@{4UYg4SWJtQ-W~0MDb|j-$lwVn znAm*Pl!?Ps&3wO=R115RWKb*JKoexo*)uhhHBncEDMSVa_PyA>k{Zm2(wMQ(5NM3# z)jkza|GoWEQo4^s*wE(gHz?Xsg4`}HUAcs42cM1-qq_=+=!Gk^y710j=66(cSWqUe zklbm8+zB_syQv5A2rj!Vbw8;|$@C!vfNmNV!yJIWDQ>{+2x zKjuFX`~~HKG~^6h5FntRpnnHt=D&rq0>IJ9#F0eM)Y-)GpRjiN7gkA8wvnG#K=q{q z9dBn8_~wm4J<3J_vl|9H{7q6u2A!cW{bp#r*-f{gOV^e=8S{nc1DxMHFwuM$;aVI^ zz6A*}m8N-&x8;aunp1w7_vtB*pa+OYBw=TMc6QK=mbA-|Cf* zvyh8D4LRJImooUaSb7t*fVfih<97Gf@VE0|z>NcBwBQze);Rh!k3K_sfunToZY;f2 z^HmC4KjHRVg+eKYj;PRN^|E0>Gj_zagfRbrki68I^#~6-HaHg3BUW%+clM1xQEdPYt_g<2K+z!$>*$9nQ>; zf9Bei{?zY^-e{q_*|W#2rJG`2fy@{%6u0i_VEWTq$*(ZN37|8lFFFt)nCG({r!q#9 z5VK_kkSJ3?zOH)OezMT{!YkCuSSn!K#-Rhl$uUM(bq*jY? zi1xbMVthJ`E>d>(f3)~fozjg^@eheMF6<)I`oeJYx4*+M&%c9VArn(OM-wp%M<-`x z7sLP1&3^%Nld9Dhm@$3f2}87!quhI@nwd@3~fZl_3LYW-B?Ia>ui`ELg z&Qfe!7m6ze=mZ`Ia9$z|ARSw|IdMpooY4YiPN8K z4B(ts3p%2i(Td=tgEHX z0UQ_>URBtG+-?0E;E7Ld^dyZ;jjw0}XZ(}-QzC6+NN=40oDb2^v!L1g9xRvE#@IBR zO!b-2N7wVfLV;mhEaXQ9XAU+>=XVA6f&T4Z-@AX!leJ8obP^P^wP0aICND?~w&NykJ#54x3_@r7IDMdRNy4Hh;h*!u(Ol(#0bJdwEo$5437-UBjQ+j=Ic>Q2z` zJNDf0yO6@mr6y1#n3)s(W|$iE_i8r@Gd@!DWDqZ7J&~gAm1#~maIGJ1sls^gxL9LLG_NhU!pTGty!TbhzQnu)I*S^54U6Yu%ZeCg`R>Q zhBv$n5j0v%O_j{QYWG!R9W?5_b&67KB$t}&e2LdMvd(PxN6Ir!H4>PNlerpBL>Zvyy!yw z-SOo8caEpDt(}|gKPBd$qND5#a5nju^O>V&;f890?yEOfkSG^HQVmEbM3Ugzu+UtH zC(INPDdraBN?P%kE;*Ae%Wto&sgw(crfZ#Qy(<4nk;S|hD3j{IQRI6Yq|f^basLY; z-HB&Je%Gg}Jt@={_C{L$!RM;$$|iD6vu#3w?v?*;&()uB|I-XqEKqZPS!reW9JkLewLb!70T7n`i!gNtb1%vN- zySZj{8-1>6E%H&=V}LM#xmt`J3XQoaD|@XygXjdZ1+P77-=;=eYpoEQ01B@L*a(uW zrZeZz?HJsw_4g0vhUgkg@VF8<-X$B8pOqCuWAl28uB|@r`19DTUQQsb^pfqB6QtiT z*`_UZ`fT}vtUY#%sq2{rchyfu*pCg;uec2$-$N_xgjZcoumE5vSI{+s@iLWoz^Mf; zuI8kDP{!XY6OP~q5}%1&L}CtfH^N<3o4L@J@zg1-mt{9L`s^z$Vgb|mr{@WiwAqKg zp#t-lhrU>F8o0s1q_9y`gQNf~Vb!F%70f}$>i7o4ho$`uciNf=xgJ>&!gSt0g;M>*x4-`U)ysFW&Vs^Vk6m%?iuWU+o&m(2Jm26Y(3%TL; zA7T)BP{WS!&xmxNw%J=$MPfn(9*^*TV;$JwRy8Zl*yUZi8jWYF>==j~&S|Xinsb%c z2?B+kpet*muEW7@AzjBA^wAJBY8i|#C{WtO_or&Nj2{=6JTTX05}|H>N2B|Wf!*3_ z7hW*j6p3TvpghEc6-wufFiY!%-GvOx*bZrhZu+7?iSrZL5q9}igiF^*R3%DE4aCHZ zqu>xS8LkW+Auv%z-<1Xs92u23R$nk@Pk}MU5!gT|c7vGlEA%G^2th&Q*zfg%-D^=f z&J_}jskj|Q;73NP4<4k*Y%pXPU2Thoqr+5uH1yEYM|VtBPW6lXaetokD0u z9qVek6Q&wk)tFbQ8(^HGf3Wp16gKmr>G;#G(HRBx?F`9AIRboK+;OfHaLJ(P>IP0w zyTbTkx_THEOs%Q&aPrxbZrJlio+hCC_HK<4%f3ZoSAyG7Dn`=X=&h@m*|UYO-4Hq0 z-Bq&+Ie!S##4A6OGoC~>ZW`Y5J)*ouaFl_e9GA*VSL!O_@xGiBw!AF}1{tB)z(w%c zS1Hmrb9OC8>0a_$BzeiN?rkPLc9%&;1CZW*4}CDDNr2gcl_3z+WC15&H1Zc2{o~i) z)LLW=WQ{?ricmC`G1GfJ0Yp4Dy~Ba;j6ZV4r{8xRs`13{dD!xXmr^Aga|C=iSmor% z8hi|pTXH)5Yf&v~exp3o+sY4B^^b*eYkkCYl*T{*=-0HniSA_1F53eCb{x~1k3*`W zr~};p1A`k{1DV9=UPnLDgz{aJH=-LQo<5%+Em!DNN252xwIf*wF_zS^!(XSm(9eoj z=*dXG&n0>)_)N5oc6v!>-bd(2ragD8O=M|wGW z!xJQS<)u70m&6OmrF0WSsr@I%T*c#Qo#Ha4d3COcX+9}hM5!7JIGF>7<~C(Ear^Sn zm^ZFkV6~Ula6+8S?oOROOA6$C&q&dp`>oR-2Ym3(HT@O7Sd5c~+kjrmM)YmgPH*tL zX+znN>`tv;5eOfX?h{AuX^LK~V#gPCu=)Tigtq9&?7Xh$qN|%A$?V*v=&-2F$zTUv z`C#WyIrChS5|Kgm_GeudCFf;)!WH7FI60j^0o#65o6`w*S7R@)88n$1nrgU(oU0M9 zx+EuMkC>(4j1;m6NoGqEkpJYJ?vc|B zOlwT3t&UgL!pX_P*6g36`ZXQ; z9~Cv}ANFnJGp(;ZhS(@FT;3e)0)Kp;h^x;$*xZn*k0U6-&FwI=uOGaODdrsp-!K$Ac32^c{+FhI-HkYd5v=`PGsg%6I`4d9Jy)uW0y%) zm&j^9WBAp*P8#kGJUhB!L?a%h$hJgQrx!6KCB_TRo%9{t0J7KW8!o1B!NC)VGLM5! zpZy5Jc{`r{1e(jd%jsG7k%I+m#CGS*BPA65ZVW~fLYw0dA-H_}O zrkGFL&P1PG9p2(%QiEWm6x;U-U&I#;Em$nx-_I^wtgw3xUPVVu zqSuKnx&dIT-XT+T10p;yjo1Y)z(x1fb8Dzfn8e yu?e%!_ptzGB|8GrCfu%p?(_ zQccdaaVK$5bz;*rnyK{_SQYM>;aES6Qs^lj9lEs6_J+%nIiuQC*fN;z8md>r_~Mfl zU%p5Dt_YT>gQqfr@`cR!$NWr~+`CZb%dn;WtzrAOI>P_JtsB76PYe*<%H(y>qx-`Kq!X_; z<{RpAqYhE=L1r*M)gNF3B8r(<%8mo*SR2hu zccLRZwGARt)Hlo1euqTyM>^!HK*!Q2P;4UYrysje@;(<|$&%vQekbn|0Ruu_Io(w4#%p6ld2Yp7tlA`Y$cciThP zKzNGIMPXX%&Ud0uQh!uQZz|FB`4KGD?3!ND?wQt6!n*f4EmCoJUh&b?;B{|lxs#F- z31~HQ`SF4x$&v00@(P+j1pAaj5!s`)b2RDBp*PB=2IB>oBF!*6vwr7Dp%zpAx*dPr zb@Zjq^XjN?O4QcZ*O+8>)|HlrR>oD*?WQl5ri3R#2?*W6iJ>>kH%KnnME&TT@ZzrHS$Q%LC?n|e>V+D+8D zYc4)QddFz7I8#}y#Wj6>4P%34dZH~OUDb?uP%-E zwjXM(?Sg~1!|wI(RVuxbu)-rH+O=igSho_pDCw(c6b=P zKk4ATlB?bj9+HHlh<_!&z0rx13K3ZrAR8W)!@Y}o`?a*JJsD+twZIv`W)@Y?Amu_u zz``@-e2X}27$i(2=9rvIu5uTUOVhzwu%mNazS|lZb&PT;XE2|B&W1>=B58#*!~D&) zfVmJGg8UdP*fx(>Cj^?yS^zH#o-$Q-*$SnK(ZVFkw+er=>N^7!)FtP3y~Xxnu^nzY zikgB>Nj0%;WOltWIob|}%lo?_C7<``a5hEkx&1ku$|)i>Rh6@3h*`slY=9U}(Ql_< zaNG*J8vb&@zpdhAvv`?{=zDedJ23TD&Zg__snRAH4eh~^oawdYi6A3w8<Ozh@Kw)#bdktM^GVb zrG08?0bG?|NG+w^&JvD*7LAbjED{_Zkc`3H!My>0u5Q}m!+6VokMLXxl`Mkd=g&Xx z-a>m*#G3SLlhbKB!)tnzfWOBV;u;ftU}S!NdD5+YtOjLg?X}dl>7m^gOpihrf1;PY zvll&>dIuUGs{Qnd- zwIR3oIrct8Va^Tm0t#(bJD7c$Z7DO9*7NnRZorrSm`b`cxz>OIC;jSE3DO8`hX955ui`s%||YQtt2 z5DNA&pG-V+4oI2s*x^>-$6J?p=I>C|9wZF8z;VjR??Icg?1w2v5Me+FgAeGGa8(3S z4vg*$>zC-WIVZtJ7}o9{D-7d>zCe|z#<9>CFve-OPAYsneTb^JH!Enaza#j}^mXy1 z+ULn^10+rWLF6j2>Ya@@Kq?26>AqK{A_| zQKb*~F1>sE*=d?A?W7N2j?L09_7n+HGi{VY;MoTGr_)G9)ot$p!-UY5zZ2Xtbm=t z@dpPSGwgH=QtIcEulQNI>S-#ifbnO5EWkI;$A|pxJd885oM+ zGZ0_0gDvG8q2xebj+fbCHYfAXuZStH2j~|d^sBAzo46(K8n59+T6rzBwK)^rfPT+B zyIFw)9YC-V^rhtK`!3jrhmW-sTmM+tPH+;nwjL#-SjQPUZ53L@A>y*rt(#M(qsiB2 zx6B)dI}6Wlsw%bJ8h|(lhkJVogQZA&n{?Vgs6gNSXzuZpEyu*xySy8ro07QZ7Vk1!3tJphN_5V7qOiyK8p z#@jcDD8nmtYi1^l8ml;AF<#IPK?!pqf9D4moYk>d99Im}Jtwj6c#+A;f)CQ*f-hZ< z=p_T86jog%!p)D&5g9taSwYi&eP z#JuEK%+NULWus;0w32-SYFku#i}d~+{Pkho&^{;RxzP&0!RCm3-9K6`>KZpnzS6?L z^H^V*s!8<>x8bomvD%rh>Zp3>Db%kyin;qtl+jAv8Oo~1g~mqGAC&Qi_wy|xEt2iz zWAJEfTV%cl2Cs<1L&DLRVVH05EDq`pH7Oh7sR`NNkL%wi}8n>IXcO40hp+J+sC!W?!krJf!GJNE8uj zg-y~Ns-<~D?yqbzVRB}G>0A^f0!^N7l=$m0OdZuqAOQqLc zX?AEGr1Ht+inZ-Qiwnl@Z0qukd__a!C*CKuGdy5#nD7VUBM^6OCpxCa2A(X;e0&V4 zM&WR8+wErQ7UIc6LY~Q9x%Sn*Tn>>P`^t&idaOEnOd(Ufw#>NoR^1QdhJ8s`h^|R_ zXX`c5*O~Xdvh%q;7L!_!ohf$NfEBmCde|#uVZvEo>OfEq%+Ns7&_f$OR9xsihRpBb z+cjk8LyDm@U{YN>+r46?nn{7Gh(;WhFw6GAxtcKD+YWV?uge>;+q#Xx4!GpRkVZYu zzsF}1)7$?%s9g9CH=Zs+B%M_)+~*j3L0&Q9u7!|+T`^O{xE6qvAP?XWv9_MrZKdo& z%IyU)$Q95AB4!#hT!_dA>4e@zjOBD*Y=XjtMm)V|+IXzjuM;(l+8aA5#Kaz_$rR6! zj>#&^DidYD$nUY(D$mH`9eb|dtV0b{S>H6FBfq>t5`;OxA4Nn{J(+XihF(stSche7$es&~N$epi&PDM_N`As;*9D^L==2Q7Z2zD+CiU(|+-kL*VG+&9!Yb3LgPy?A zm7Z&^qRG_JIxK7-FBzZI3Q<;{`DIxtc48k> zc|0dmX;Z=W$+)qE)~`yn6MdoJ4co;%!`ddy+FV538Y)j(vg}5*k(WK)KWZ3WaOG!8 z!syGn=s{H$odtpqFrT#JGM*utN7B((abXnpDM6w56nhw}OY}0TiTG1#f*VFZr+^-g zbP10`$LPq_;PvrA1XXlyx2uM^mrjTzX}w{yuLo-cOClE8MMk47T25G8M!9Z5ypOSV zAJUBGEg5L2fY)ZGJb^E34R2zJ?}Vf>{~gB!8=5Z) z9y$>5c)=;o0HeHHSuE4U)#vG&KF|I%-cF6f$~pdYJWk_dD}iOA>iA$O$+4%@>JU08 zS`ep)$XLPJ+n0_i@PkF#ri6T8?ZeAot$6JIYHm&P6EB=BiaNY|aA$W0I+nz*zkz_z zkEru!tj!QUffq%)8y0y`T&`fuus-1p>=^hnBiBqD^hXrPs`PY9tU3m0np~rISY09> z`P3s=-kt_cYcxWd{de@}TwSqg*xVhp;E9zCsnXo6z z?f&Sv^U7n4`xr=mXle94HzOdN!2kB~4=%)u&N!+2;z6UYKUDqi-s6AZ!haB;@&B`? z_TRX0%@suz^TRdCb?!vNJYPY8L_}&07uySH9%W^Tc&1pia6y1q#?*Drf}GjGbPjBS zbOPcUY#*$3sL2x4v_i*Y=N7E$mR}J%|GUI(>WEr+28+V z%v5{#e!UF*6~G&%;l*q*$V?&r$Pp^sE^i-0$+RH3ERUUdQ0>rAq2(2QAbG}$y{de( z>{qD~GGuOk559Y@%$?N^1ApVL_a704>8OD%8Y%8B;FCt%AoPu8*D1 zLB5X>b}Syz81pn;xnB}%0FnwazlWfUV)Z-~rZg6~b z6!9J$EcE&sEbzcy?CI~=boWA&eeIa%z(7SE^qgVLz??1Vbc1*aRvc%Mri)AJaAG!p z$X!_9Ds;Zz)f+;%s&dRcJt2==P{^j3bf0M=nJd&xwUGlUFn?H=2W(*2I2Gdu zv!gYCwM10aeus)`RIZSrCK=&oKaO_Ry~D1B5!y0R=%!i2*KfXGYX&gNv_u+n9wiR5 z*e$Zjju&ODRW3phN925%S(jL+bCHv6rZtc?!*`1TyYXT6%Ju=|X;6D@lq$8T zW{Y|e39ioPez(pBH%k)HzFITXHvnD6hw^lIoUMA;qAJ^CU?top1fo@s7xT13Fvn1H z6JWa-6+FJF#x>~+A;D~;VDs26>^oH0EI`IYT2iagy23?nyJ==i{g4%HrAf1-*v zK1)~@&(KkwR7TL}L(A@C_S0G;-GMDy=MJn2$FP5s<%wC)4jC5PXoxrQBFZ_k0P{{s@sz+gX`-!=T8rcB(=7vW}^K6oLWMmp(rwDh}b zwaGGd>yEy6fHv%jM$yJXo5oMAQ>c9j`**}F?MCry;T@47@r?&sKHgVe$MCqk#Z_3S z1GZI~nOEN*P~+UaFGnj{{Jo@16`(qVNtbU>O0Hf57-P>x8Jikp=`s8xWs^dAJ9lCQ z)GFm+=OV%AMVqVATtN@|vp61VVAHRn87}%PC^RAzJ%JngmZTasWBAWsoAqBU+8L8u z4A&Pe?fmTm0?mK-BL9t+{y7o(7jm+RpOhL9KnY#E&qu^}B6=K_dB}*VlSEiC9fn)+V=J;OnN)Ta5v66ic1rG+dGAJ1 z1%Zb_+!$=tQ~lxQrzv3x#CPb?CekEkA}0MYSgx$Jdd}q8+R=ma$|&1a#)TQ=l$1tQ z=tL9&_^vJ)Pk}EDO-va`UCT1m#Uty1{v^A3P~83_#v^ozH}6*9mIjIr;t3Uv%@VeW zGL6(CwCUp)Jq%G0bIG%?{_*Y#5IHf*5M@wPo6A{$Um++Co$wLC=J1aoG93&T7Ho}P z=mGEPP7GbvoG!uD$k(H3A$Z))+i{Hy?QHdk>3xSBXR0j!11O^mEe9RHmw!pvzv?Ua~2_l2Yh~_!s1qS`|0~0)YsbHSz8!mG)WiJE| z2f($6TQtt6L_f~ApQYQKSb=`053LgrQq7G@98#igV>y#i==-nEjQ!XNu9 z~;mE+gtj4IDDNQJ~JVk5Ux6&LCSFL!y=>79kE9=V}J7tD==Ga+IW zX)r7>VZ9dY=V&}DR))xUoV!u(Z|%3ciQi_2jl}3=$Agc(`RPb z8kEBpvY>1FGQ9W$n>Cq=DIpski};nE)`p3IUw1Oz0|wxll^)4dq3;CCY@RyJgFgc# zKouFh!`?Xuo{IMz^xi-h=StCis_M7yq$u) z?XHvw*HP0VgR+KR6wI)jEMX|ssqYvSf*_3W8zVTQzD?3>H!#>InzpSO)@SC8q*ii- z%%h}_#0{4JG;Jm`4zg};BPTGkYamx$Xo#O~lBirRY)q=5M45n{GCfV7h9qwyu1NxOMoP4)jjZMxmT|IQQh0U7C$EbnMN<3)Kk?fFHYq$d|ICu>KbY_hO zTZM+uKHe(cIZfEqyzyYSUBZa8;Fcut-GN!HSA9ius`ltNebF46ZX_BbZNU}}ZOm{M2&nANL9@0qvih15(|`S~z}m&h!u4x~(%MAO$jHRWNfuxWF#B)E&g3ghSQ9|> z(MFaLQj)NE0lowyjvg8z0#m6FIuKE9lDO~Glg}nSb7`~^&#(Lw{}GVOS>U)m8bF}x zVjbXljBm34Cs-yM6TVusr+3kYFjr28STT3g056y3cH5Tmge~ASxBj z%|yb>$eF;WgrcOZf569sDZOVwoo%8>XO>XQOX1OyN9I-SQgrm;U;+#3OI(zrWyow3 zk==|{lt2xrQ%FIXOTejR>;wv(Pb8u8}BUpx?yd(Abh6? zsoO3VYWkeLnF43&@*#MQ9-i-d0t*xN-UEyNKeyNMHw|A(k(_6QKO=nKMCxD(W(Yop zsRQ)QeL4X3Lxp^L%wzi2-WVSsf61dqliPUM7srDB?Wm6Lzn0&{*}|IsKQW;02(Y&| zaTKv|`U(pSzuvR6Rduu$wzK_W-Y-7>7s?G$)U}&uK;<>vU}^^ns@Z!p+9?St1s)dG zK%y6xkPyyS1$~&6v{kl?Md6gwM|>mt6Upm>oa8RLD^8T{0?HC!Z>;(Bob7el(DV6x zi`I)$&E&ngwFS@bi4^xFLAn`=fzTC;aimE^!cMI2n@Vo%Ae-ne`RF((&5y6xsjjAZ zVguVoQ?Z9uk$2ON;ersE%PU*xGO@T*;j1BO5#TuZKEf(mB7|g7pcEA=nYJ{s3vlbg zd4-DUlD{*6o%Gc^N!Nptgay>j6E5;3psI+C3Q!1ZIbeCubW%w4pq9)MSDyB{HLm|k zxv-{$$A*pS@csolri$Ge<4VZ}e~78JOL-EVyrbxKra^d{?|NnPp86!q>t<&IP07?Z z^>~IK^k#OEKgRH+LjllZXk7iA>2cfH6+(e&9ku5poo~6y{GC5>(bRK7hwjiurqAiZ zg*DmtgY}v83IjE&AbiWgMyFbaRUPZ{lYiz$U^&Zt2YjG<%m((&_JUbZcfJ22(>bi5 z!J?<7AySj0JZ&<-qXX;mcV!f~>G=sB0KnjWca4}vrtunD^1TrpfeS^4dvFr!65knK zZh`d;*VOkPs4*-9kL>$GP0`(M!j~B;#x?Ba~&s6CopvO86oM?-? zOw#dIRc;6A6T?B`Qp%^<U5 z19x(ywSH$_N+Io!6;e?`tWaM$`=Db!gzx|lQ${DG!zb1Zl&|{kX0y6xvO1o z220r<-oaS^^R2pEyY;=Qllqpmue|5yI~D|iI!IGt@iod{Opz@*ml^w2bNs)p`M(Io z|E;;m*Xpjd9l)4G#KaWfV(t8YUn@A;nK^#xgv=LtnArX|vWQVuw3}B${h+frU2>9^ z!l6)!Uo4`5k`<<;E(ido7M6lKTgWezNLq>U*=uz&s=cc$1%>VrAeOoUtA|T6gO4>UNqsdK=NF*8|~*sl&wI=x9-EGiq*aqV!(VVXA57 zw9*o6Ir8Lj1npUXvlevtn(_+^X5rzdR>#(}4YcB9O50q97%rW2me5_L=%ffYPUSRc z!vv?Kv>dH994Qi>U(a<0KF6NH5b16enCp+mw^Hb3Xs1^tThFpz!3QuN#}KBbww`(h z7GO)1olDqy6?T$()R7y%NYx*B0k_2IBiZ14&8|JPFxeMF{vSTxF-Vi3+ZOI=Thq2} zyQgjYY1_7^ZQHh{?P))4+qUiQJLi1&{yE>h?~jU%tjdV0h|FENbM3X(KnJdPKc?~k zh=^Ixv*+smUll!DTWH!jrV*wSh*(mx0o6}1@JExzF(#9FXgmTXVoU+>kDe68N)dkQ zH#_98Zv$}lQwjKL@yBd;U(UD0UCl322=pav<=6g>03{O_3oKTq;9bLFX1ia*lw;#K zOiYDcBJf)82->83N_Y(J7Kr_3lE)hAu;)Q(nUVydv+l+nQ$?|%MWTy`t>{havFSQloHwiIkGK9YZ79^9?AZo0ZyQlVR#}lF%dn5n%xYksXf8gnBm=wO7g_^! zauQ-bH1Dc@3ItZ-9D_*pH}p!IG7j8A_o94#~>$LR|TFq zZ-b00*nuw|-5C2lJDCw&8p5N~Z1J&TrcyErds&!l3$eSz%`(*izc;-?HAFD9AHb-| z>)id`QCrzRws^9(#&=pIx9OEf2rmlob8sK&xPCWS+nD~qzU|qG6KwA{zbikcfQrdH z+ zQg>O<`K4L8rN7`GJB0*3<3`z({lWe#K!4AZLsI{%z#ja^OpfjU{!{)x0ZH~RB0W5X zTwN^w=|nA!4PEU2=LR05x~}|B&ZP?#pNgDMwD*ajI6oJqv!L81gu=KpqH22avXf0w zX3HjbCI!n9>l046)5rr5&v5ja!xkKK42zmqHzPx$9Nn_MZk`gLeSLgC=LFf;H1O#B zn=8|^1iRrujHfbgA+8i<9jaXc;CQBAmQvMGQPhFec2H1knCK2x!T`e6soyrqCamX% zTQ4dX_E*8so)E*TB$*io{$c6X)~{aWfaqdTh=xEeGvOAN9H&-t5tEE-qso<+C!2>+ zskX51H-H}#X{A75wqFe-J{?o8Bx|>fTBtl&tcbdR|132Ztqu5X0i-pisB-z8n71%q%>EF}yy5?z=Ve`}hVh{Drv1YWL zW=%ug_&chF11gDv3D6B)Tz5g54H0mDHNjuKZ+)CKFk4Z|$RD zfRuKLW`1B>B?*RUfVd0+u8h3r-{@fZ{k)c!93t1b0+Q9vOaRnEn1*IL>5Z4E4dZ!7 ztp4GP-^1d>8~LMeb}bW!(aAnB1tM_*la=Xx)q(I0Y@__Zd$!KYb8T2VBRw%e$iSdZ zkwdMwd}eV9q*;YvrBFTv1>1+}{H!JK2M*C|TNe$ZSA>UHKk);wz$(F$rXVc|sI^lD zV^?_J!3cLM;GJuBMbftbaRUs$;F}HDEDtIeHQ)^EJJ1F9FKJTGH<(Jj`phE6OuvE) zqK^K`;3S{Y#1M@8yRQwH`?kHMq4tHX#rJ>5lY3DM#o@or4&^_xtBC(|JpGTfrbGkA z2Tu+AyT^pHannww!4^!$5?@5v`LYy~T`qs7SYt$JgrY(w%C+IWA;ZkwEF)u5sDvOK zGk;G>Mh&elvXDcV69J_h02l&O;!{$({fng9Rlc3ID#tmB^FIG^w{HLUpF+iB`|

NnX)EH+Nua)3Y(c z&{(nX_ht=QbJ%DzAya}!&uNu!4V0xI)QE$SY__m)SAKcN0P(&JcoK*Lxr@P zY&P=}&B3*UWNlc|&$Oh{BEqwK2+N2U$4WB7Fd|aIal`FGANUa9E-O)!gV`((ZGCc$ zBJA|FFrlg~9OBp#f7aHodCe{6= zay$6vN~zj1ddMZ9gQ4p32(7wD?(dE>KA2;SOzXRmPBiBc6g`eOsy+pVcHu=;Yd8@{ zSGgXf@%sKKQz~;!J;|2fC@emm#^_rnO0esEn^QxXgJYd`#FPWOUU5b;9eMAF zZhfiZb|gk8aJIw*YLp4!*(=3l8Cp{(%p?ho22*vN9+5NLV0TTazNY$B5L6UKUrd$n zjbX%#m7&F#U?QNOBXkiiWB*_tk+H?N3`vg;1F-I+83{M2!8<^nydGr5XX}tC!10&e z7D36bLaB56WrjL&HiiMVtpff|K%|*{t*ltt^5ood{FOG0<>k&1h95qPio)2`eL${YAGIx(b4VN*~nKn6E~SIQUuRH zQ+5zP6jfnP$S0iJ@~t!Ai3o`X7biohli;E zT#yXyl{bojG@-TGZzpdVDXhbmF%F9+-^YSIv|MT1l3j zrxOFq>gd2%U}?6}8mIj?M zc077Zc9fq(-)4+gXv?Az26IO6eV`RAJz8e3)SC7~>%rlzDwySVx*q$ygTR5kW2ds- z!HBgcq0KON9*8Ff$X0wOq$`T7ml(@TF)VeoF}x1OttjuVHn3~sHrMB++}f7f9H%@f z=|kP_?#+fve@{0MlbkC9tyvQ_R?lRdRJ@$qcB(8*jyMyeME5ns6ypVI1Xm*Zr{DuS zZ!1)rQfa89c~;l~VkCiHI|PCBd`S*2RLNQM8!g9L6?n`^evQNEwfO@&JJRme+uopQX0%Jo zgd5G&#&{nX{o?TQwQvF1<^Cg3?2co;_06=~Hcb6~4XWpNFL!WU{+CK;>gH%|BLOh7@!hsa(>pNDAmpcuVO-?;Bic17R}^|6@8DahH)G z!EmhsfunLL|3b=M0MeK2vqZ|OqUqS8npxwge$w-4pFVXFq$_EKrZY?BuP@Az@(k`L z`ViQBSk`y+YwRT;&W| z2e3UfkCo^uTA4}Qmmtqs+nk#gNr2W4 zTH%hhErhB)pkXR{B!q5P3-OM+M;qu~f>}IjtF%>w{~K-0*jPVLl?Chz&zIdxp}bjx zStp&Iufr58FTQ36AHU)0+CmvaOpKF;W@sMTFpJ`j;3d)J_$tNQI^c<^1o<49Z(~K> z;EZTBaVT%14(bFw2ob@?JLQ2@(1pCdg3S%E4*dJ}dA*v}_a4_P(a`cHnBFJxNobAv zf&Zl-Yt*lhn-wjZsq<9v-IsXxAxMZ58C@e0!rzhJ+D@9^3~?~yllY^s$?&oNwyH!#~6x4gUrfxplCvK#!f z$viuszW>MFEcFL?>ux*((!L$;R?xc*myjRIjgnQX79@UPD$6Dz0jutM@7h_pq z0Zr)#O<^y_K6jfY^X%A-ip>P%3saX{!v;fxT-*0C_j4=UMH+Xth(XVkVGiiKE#f)q z%Jp=JT)uy{&}Iq2E*xr4YsJ5>w^=#-mRZ4vPXpI6q~1aFwi+lQcimO45V-JXP;>(Q zo={U`{=_JF`EQj87Wf}{Qy35s8r1*9Mxg({CvOt}?Vh9d&(}iI-quvs-rm~P;eRA@ zG5?1HO}puruc@S{YNAF3vmUc2B4!k*yi))<5BQmvd3tr}cIs#9)*AX>t`=~{f#Uz0 z0&Nk!7sSZwJe}=)-R^$0{yeS!V`Dh7w{w5rZ9ir!Z7Cd7dwZcK;BT#V0bzTt>;@Cl z#|#A!-IL6CZ@eHH!CG>OO8!%G8&8t4)Ro@}USB*k>oEUo0LsljsJ-%5Mo^MJF2I8- z#v7a5VdJ-Cd%(a+y6QwTmi+?f8Nxtm{g-+WGL>t;s#epv7ug>inqimZCVm!uT5Pf6 ziEgQt7^%xJf#!aPWbuC_3Nxfb&CFbQy!(8ANpkWLI4oSnH?Q3f?0k1t$3d+lkQs{~(>06l&v|MpcFsyAv zin6N!-;pggosR*vV=DO(#+}4ps|5$`udE%Kdmp?G7B#y%H`R|i8skKOd9Xzx8xgR$>Zo2R2Ytktq^w#ul4uicxW#{ zFjG_RNlBroV_n;a7U(KIpcp*{M~e~@>Q#Av90Jc5v%0c>egEdY4v3%|K1XvB{O_8G zkTWLC>OZKf;XguMH2-Pw{BKbFzaY;4v2seZV0>^7Q~d4O=AwaPhP3h|!hw5aqOtT@ z!SNz}$of**Bl3TK209@F=Tn1+mgZa8yh(Png%Zd6Mt}^NSjy)etQrF zme*llAW=N_8R*O~d2!apJnF%(JcN??=`$qs3Y+~xs>L9x`0^NIn!8mMRFA_tg`etw z3k{9JAjnl@ygIiJcNHTy02GMAvBVqEss&t2<2mnw!; zU`J)0>lWiqVqo|ex7!+@0i>B~BSU1A_0w#Ee+2pJx0BFiZ7RDHEvE*ptc9md(B{&+ zKE>TM)+Pd>HEmdJao7U@S>nL(qq*A)#eLOuIfAS@j`_sK0UEY6OAJJ-kOrHG zjHx`g!9j*_jRcJ%>CE9K2MVf?BUZKFHY?EpV6ai7sET-tqk=nDFh-(65rhjtlKEY% z@G&cQ<5BKatfdA1FKuB=i>CCC5(|9TMW%K~GbA4}80I5%B}(gck#Wlq@$nO3%@QP_ z8nvPkJFa|znk>V92cA!K1rKtr)skHEJD;k8P|R8RkCq1Rh^&}Evwa4BUJz2f!2=MH zo4j8Y$YL2313}H~F7@J7mh>u%556Hw0VUOz-Un@ZASCL)y8}4XXS`t1AC*^>PLwIc zUQok5PFS=*#)Z!3JZN&eZ6ZDP^-c@StY*t20JhCnbMxXf=LK#;`4KHEqMZ-Ly9KsS zI2VUJGY&PmdbM+iT)zek)#Qc#_i4uH43 z@T5SZBrhNCiK~~esjsO9!qBpaWK<`>!-`b71Y5ReXQ4AJU~T2Njri1CEp5oKw;Lnm)-Y@Z3sEY}XIgSy%xo=uek(kAAH5MsV$V3uTUsoTzxp_rF=tx zV07vlJNKtJhCu`b}*#m&5LV4TAE&%KtHViDAdv#c^x`J7bg z&N;#I2GkF@SIGht6p-V}`!F_~lCXjl1BdTLIjD2hH$J^YFN`7f{Q?OHPFEM$65^!u zNwkelo*5+$ZT|oQ%o%;rBX$+?xhvjb)SHgNHE_yP%wYkkvXHS{Bf$OiKJ5d1gI0j< zF6N}Aq=(WDo(J{e-uOecxPD>XZ@|u-tgTR<972`q8;&ZD!cep^@B5CaqFz|oU!iFj zU0;6fQX&~15E53EW&w1s9gQQ~Zk16X%6 zjG`j0yq}4deX2?Tr(03kg>C(!7a|b9qFI?jcE^Y>-VhudI@&LI6Qa}WQ>4H_!UVyF z((cm&!3gmq@;BD#5P~0;_2qgZhtJS|>WdtjY=q zLnHH~Fm!cxw|Z?Vw8*~?I$g#9j&uvgm7vPr#&iZgPP~v~BI4jOv;*OQ?jYJtzO<^y z7-#C={r7CO810!^s(MT!@@Vz_SVU)7VBi(e1%1rvS!?PTa}Uv`J!EP3s6Y!xUgM^8 z4f!fq<3Wer_#;u!5ECZ|^c1{|q_lh3m^9|nsMR1#Qm|?4Yp5~|er2?W^7~cl;_r4WSme_o68J9p03~Hc%X#VcX!xAu%1`R!dfGJCp zV*&m47>s^%Ib0~-2f$6oSgn3jg8m%UA;ArcdcRyM5;}|r;)?a^D*lel5C`V5G=c~k zy*w_&BfySOxE!(~PI$*dwG><+-%KT5p?whOUMA*k<9*gi#T{h3DAxzAPxN&Xws8o9Cp*`PA5>d9*Z-ynV# z9yY*1WR^D8|C%I@vo+d8r^pjJ$>eo|j>XiLWvTWLl(^;JHCsoPgem6PvegHb-OTf| zvTgsHSa;BkbG=(NgPO|CZu9gUCGr$8*EoH2_Z#^BnxF0yM~t`|9ws_xZ8X8iZYqh! zAh;HXJ)3P&)Q0(&F>!LN0g#bdbis-cQxyGn9Qgh`q+~49Fqd2epikEUw9caM%V6WgP)532RMRW}8gNS%V%Hx7apSz}tn@bQy!<=lbhmAH=FsMD?leawbnP5BWM0 z5{)@EEIYMu5;u)!+HQWhQ;D3_Cm_NADNeb-f56}<{41aYq8p4=93d=-=q0Yx#knGYfXVt z+kMxlus}t2T5FEyCN~!}90O_X@@PQpuy;kuGz@bWft%diBTx?d)_xWd_-(!LmVrh**oKg!1CNF&LX4{*j|) zIvjCR0I2UUuuEXh<9}oT_zT#jOrJAHNLFT~Ilh9hGJPI1<5`C-WA{tUYlyMeoy!+U zhA#=p!u1R7DNg9u4|QfED-2TuKI}>p#2P9--z;Bbf4Op*;Q9LCbO&aL2i<0O$ByoI z!9;Ght733FC>Pz>$_mw(F`zU?`m@>gE`9_p*=7o=7av`-&ifU(^)UU`Kg3Kw`h9-1 z6`e6+im=|m2v`pN(2dE%%n8YyQz;#3Q-|x`91z?gj68cMrHl}C25|6(_dIGk*8cA3 zRHB|Nwv{@sP4W+YZM)VKI>RlB`n=Oj~Rzx~M+Khz$N$45rLn6k1nvvD^&HtsMA4`s=MmuOJID@$s8Ph4E zAmSV^+s-z8cfv~Yd(40Sh4JG#F~aB>WFoX7ykaOr3JaJ&Lb49=B8Vk-SQT9%7TYhv z?-Pprt{|=Y5ZQ1?od|A<_IJU93|l4oAfBm?3-wk{O<8ea+`}u%(kub(LFo2zFtd?4 zwpN|2mBNywv+d^y_8#<$r>*5+$wRTCygFLcrwT(qc^n&@9r+}Kd_u@Ithz(6Qb4}A zWo_HdBj#V$VE#l6pD0a=NfB0l^6W^g`vm^sta>Tly?$E&{F?TTX~DsKF~poFfmN%2 z4x`Dc{u{Lkqz&y!33;X}weD}&;7p>xiI&ZUb1H9iD25a(gI|`|;G^NwJPv=1S5e)j z;U;`?n}jnY6rA{V^ zxTd{bK)Gi^odL3l989DQlN+Zs39Xe&otGeY(b5>rlIqfc7Ap4}EC?j<{M=hlH{1+d zw|c}}yx88_xQr`{98Z!d^FNH77=u(p-L{W6RvIn40f-BldeF-YD>p6#)(Qzf)lfZj z?3wAMtPPp>vMehkT`3gToPd%|D8~4`5WK{`#+}{L{jRUMt zrFz+O$C7y8$M&E4@+p+oV5c%uYzbqd2Y%SSgYy#xh4G3hQv>V*BnuKQhBa#=oZB~w{azUB+q%bRe_R^ z>fHBilnRTUfaJ201czL8^~Ix#+qOHSO)A|xWLqOxB$dT2W~)e-r9;bm=;p;RjYahB z*1hegN(VKK+ztr~h1}YP@6cfj{e#|sS`;3tJhIJK=tVJ-*h-5y9n*&cYCSdg#EHE# zSIx=r#qOaLJoVVf6v;(okg6?*L_55atl^W(gm^yjR?$GplNP>BZsBYEf_>wM0Lc;T zhf&gpzOWNxS>m+mN92N0{;4uw`P+9^*|-1~$uXpggj4- z^SFc4`uzj2OwdEVT@}Q`(^EcQ_5(ZtXTql*yGzdS&vrS_w>~~ra|Nb5abwf}Y!uq6R5f&6g2ge~2p(%c< z@O)cz%%rr4*cRJ5f`n@lvHNk@lE1a*96Kw6lJ~B-XfJW%?&-y?;E&?1AacU@`N`!O z6}V>8^%RZ7SQnZ-z$(jsX`amu*5Fj8g!3RTRwK^`2_QHe;_2y_n|6gSaGyPmI#kA0sYV<_qOZc#-2BO%hX)f$s-Z3xlI!ub z^;3ru11DA`4heAu%}HIXo&ctujzE2!6DIGE{?Zs>2}J+p&C$rc7gJC35gxhflorvsb%sGOxpuWhF)dL_&7&Z99=5M0b~Qa;Mo!j&Ti_kXW!86N%n= zSC@6Lw>UQ__F&+&Rzv?gscwAz8IP!n63>SP)^62(HK98nGjLY2*e^OwOq`3O|C92? z;TVhZ2SK%9AGW4ZavTB9?)mUbOoF`V7S=XM;#3EUpR+^oHtdV!GK^nXzCu>tpR|89 zdD{fnvCaN^^LL%amZ^}-E+214g&^56rpdc@yv0b<3}Ys?)f|fXN4oHf$six)-@<;W&&_kj z-B}M5U*1sb4)77aR=@%I?|Wkn-QJVuA96an25;~!gq(g1@O-5VGo7y&E_srxL6ZfS z*R%$gR}dyONgju*D&?geiSj7SZ@ftyA|}(*Y4KbvU!YLsi1EDQQCnb+-cM=K1io78o!v*);o<XwjaQH%)uIP&Zm?)Nfbfn;jIr z)d#!$gOe3QHp}2NBak@yYv3m(CPKkwI|{;d=gi552u?xj9ObCU^DJFQp4t4e1tPzM zvsRIGZ6VF+{6PvqsplMZWhz10YwS={?`~O0Ec$`-!klNUYtzWA^f9m7tkEzCy<_nS z=&<(awFeZvt51>@o_~>PLs05CY)$;}Oo$VDO)?l-{CS1Co=nxjqben*O1BR>#9`0^ zkwk^k-wcLCLGh|XLjdWv0_Hg54B&OzCE^3NCP}~OajK-LuRW53CkV~Su0U>zN%yQP zH8UH#W5P3-!ToO-2k&)}nFe`t+mdqCxxAHgcifup^gKpMObbox9LFK;LP3}0dP-UW z?Zo*^nrQ6*$FtZ(>kLCc2LY*|{!dUn$^RW~m9leoF|@Jy|M5p-G~j%+P0_#orRKf8 zvuu5<*XO!B?1E}-*SY~MOa$6c%2cM+xa8}_8x*aVn~57v&W(0mqN1W`5a7*VN{SUH zXz98DDyCnX2EPl-`Lesf`=AQT%YSDb`$%;(jUTrNen$NPJrlpPDP}prI>Ml!r6bCT;mjsg@X^#&<}CGf0JtR{Ecwd&)2zuhr#nqdgHj+g2n}GK9CHuwO zk>oZxy{vcOL)$8-}L^iVfJHAGfwN$prHjYV0ju}8%jWquw>}_W6j~m<}Jf!G?~r5&Rx)!9JNX!ts#SGe2HzobV5); zpj@&`cNcO&q+%*<%D7za|?m5qlmFK$=MJ_iv{aRs+BGVrs)98BlN^nMr{V_fcl_;jkzRju+c-y?gqBC_@J0dFLq-D9@VN&-`R9U;nv$Hg?>$oe4N&Ht$V_(JR3TG^! zzJsbQbi zFE6-{#9{G{+Z}ww!ycl*7rRdmU#_&|DqPfX3CR1I{Kk;bHwF6jh0opI`UV2W{*|nn zf_Y@%wW6APb&9RrbEN=PQRBEpM(N1w`81s=(xQj6 z-eO0k9=Al|>Ej|Mw&G`%q8e$2xVz1v4DXAi8G};R$y)ww638Y=9y$ZYFDM$}vzusg zUf+~BPX>(SjA|tgaFZr_e0{)+z9i6G#lgt=F_n$d=beAt0Sa0a7>z-?vcjl3e+W}+ z1&9=|vC=$co}-Zh*%3588G?v&U7%N1Qf-wNWJ)(v`iO5KHSkC5&g7CrKu8V}uQGcfcz zmBz#Lbqwqy#Z~UzHgOQ;Q-rPxrRNvl(&u6ts4~0=KkeS;zqURz%!-ERppmd%0v>iRlEf+H$yl{_8TMJzo0 z>n)`On|7=WQdsqhXI?#V{>+~}qt-cQbokEbgwV3QvSP7&hK4R{Z{aGHVS3;+h{|Hz z6$Js}_AJr383c_+6sNR|$qu6dqHXQTc6?(XWPCVZv=)D#6_;D_8P-=zOGEN5&?~8S zl5jQ?NL$c%O)*bOohdNwGIKM#jSAC?BVY={@A#c9GmX0=T(0G}xs`-%f3r=m6-cpK z!%waekyAvm9C3%>sixdZj+I(wQlbB4wv9xKI*T13DYG^T%}zZYJ|0$Oj^YtY+d$V$ zAVudSc-)FMl|54n=N{BnZTM|!>=bhaja?o7s+v1*U$!v!qQ%`T-6fBvmdPbVmro&d zk07TOp*KuxRUSTLRrBj{mjsnF8`d}rMViY8j`jo~Hp$fkv9F_g(jUo#Arp;Xw0M$~ zRIN!B22~$kx;QYmOkos@%|5k)!QypDMVe}1M9tZfkpXKGOxvKXB!=lo`p?|R1l=tA zp(1}c6T3Fwj_CPJwVsYtgeRKg?9?}%oRq0F+r+kdB=bFUdVDRPa;E~~>2$w}>O>v=?|e>#(-Lyx?nbg=ckJ#5U6;RT zNvHhXk$P}m9wSvFyU3}=7!y?Y z=fg$PbV8d7g25&-jOcs{%}wTDKm>!Vk);&rr;O1nvO0VrU&Q?TtYVU=ir`te8SLlS zKSNmV=+vF|ATGg`4$N1uS|n??f}C_4Sz!f|4Ly8#yTW-FBfvS48Tef|-46C(wEO_%pPhUC5$-~Y?!0vFZ^Gu`x=m7X99_?C-`|h zfmMM&Y@zdfitA@KPw4Mc(YHcY1)3*1xvW9V-r4n-9ZuBpFcf{yz+SR{ zo$ZSU_|fgwF~aakGr(9Be`~A|3)B=9`$M-TWKipq-NqRDRQc}ABo*s_5kV%doIX7LRLRau_gd@Rd_aLFXGSU+U?uAqh z8qusWWcvgQ&wu{|sRXmv?sl=xc<$6AR$+cl& zFNh5q1~kffG{3lDUdvEZu5c(aAG~+64FxdlfwY^*;JSS|m~CJusvi-!$XR`6@XtY2 znDHSz7}_Bx7zGq-^5{stTRy|I@N=>*y$zz>m^}^{d&~h;0kYiq8<^Wq7Dz0w31ShO^~LUfW6rfitR0(=3;Uue`Y%y@ex#eKPOW zO~V?)M#AeHB2kovn1v=n^D?2{2jhIQd9t|_Q+c|ZFaWt+r&#yrOu-!4pXAJuxM+Cx z*H&>eZ0v8Y`t}8{TV6smOj=__gFC=eah)mZt9gwz>>W$!>b3O;Rm^Ig*POZP8Rl0f zT~o=Nu1J|lO>}xX&#P58%Yl z83`HRs5#32Qm9mdCrMlV|NKNC+Z~ z9OB8xk5HJ>gBLi+m@(pvpw)1(OaVJKs*$Ou#@Knd#bk+V@y;YXT?)4eP9E5{J%KGtYinNYJUH9PU3A}66c>Xn zZ{Bn0<;8$WCOAL$^NqTjwM?5d=RHgw3!72WRo0c;+houoUA@HWLZM;^U$&sycWrFd zE7ekt9;kb0`lps{>R(}YnXlyGY}5pPd9zBpgXeJTY_jwaJGSJQC#-KJqmh-;ad&F- z-Y)E>!&`Rz!HtCz>%yOJ|v(u7P*I$jqEY3}(Z-orn4 zlI?CYKNl`6I){#2P1h)y(6?i;^z`N3bxTV%wNvQW+eu|x=kbj~s8rhCR*0H=iGkSj zk23lr9kr|p7#qKL=UjgO`@UnvzU)`&fI>1Qs7ubq{@+lK{hH* zvl6eSb9%yngRn^T<;jG1SVa)eA>T^XX=yUS@NCKpk?ovCW1D@!=@kn;l_BrG;hOTC z6K&H{<8K#dI(A+zw-MWxS+~{g$tI7|SfP$EYKxA}LlVO^sT#Oby^grkdZ^^lA}uEF zBSj$weBJG{+Bh@Yffzsw=HyChS(dtLE3i*}Zj@~!_T-Ay7z=B)+*~3|?w`Zd)Co2t zC&4DyB!o&YgSw+fJn6`sn$e)29`kUwAc+1MND7YjV%lO;H2}fNy>hD#=gT ze+-aFNpyKIoXY~Vq-}OWPBe?Rfu^{ps8>Xy%42r@RV#*QV~P83jdlFNgkPN=T|Kt7 zV*M`Rh*30&AWlb$;ae130e@}Tqi3zx2^JQHpM>j$6x`#{mu%tZlwx9Gj@Hc92IuY* zarmT|*d0E~vt6<+r?W^UW0&#U&)8B6+1+;k^2|FWBRP9?C4Rk)HAh&=AS8FS|NQaZ z2j!iZ)nbEyg4ZTp-zHwVlfLC~tXIrv(xrP8PAtR{*c;T24ycA-;auWsya-!kF~CWZ zw_uZ|%urXgUbc@x=L=_g@QJ@m#5beS@6W195Hn7>_}z@Xt{DIEA`A&V82bc^#!q8$ zFh?z_Vn|ozJ;NPd^5uu(9tspo8t%&-U9Ckay-s@DnM*R5rtu|4)~e)`z0P-sy?)kc zs_k&J@0&0!q4~%cKL)2l;N*T&0;mqX5T{Qy60%JtKTQZ-xb%KOcgqwJmb%MOOKk7N zgq})R_6**{8A|6H?fO+2`#QU)p$Ei2&nbj6TpLSIT^D$|`TcSeh+)}VMb}LmvZ{O| ze*1IdCt3+yhdYVxcM)Q_V0bIXLgr6~%JS<<&dxIgfL=Vnx4YHuU@I34JXA|+$_S3~ zy~X#gO_X!cSs^XM{yzDGNM>?v(+sF#<0;AH^YrE8smx<36bUsHbN#y57K8WEu(`qHvQ6cAZPo=J5C(lSmUCZ57Rj6cx!e^rfaI5%w}unz}4 zoX=nt)FVNV%QDJH`o!u9olLD4O5fl)xp+#RloZlaA92o3x4->?rB4`gS$;WO{R;Z3>cG3IgFX2EA?PK^M}@%1%A;?f6}s&CV$cIyEr#q5;yHdNZ9h{| z-=dX+a5elJoDo?Eq&Og!nN6A)5yYpnGEp}?=!C-V)(*~z-+?kY1Q7qs#Rsy%hu_60rdbB+QQNr?S1 z?;xtjUv|*E3}HmuNyB9aFL5H~3Ho0UsmuMZELp1a#CA1g`P{-mT?BchuLEtK}!QZ=3AWakRu~?f9V~3F;TV`5%9Pcs_$gq&CcU}r8gOO zC2&SWPsSG{&o-LIGTBqp6SLQZPvYKp$$7L4WRRZ0BR$Kf0I0SCFkqveCp@f)o8W)! z$%7D1R`&j7W9Q9CGus_)b%+B#J2G;l*FLz#s$hw{BHS~WNLODV#(!u_2Pe&tMsq={ zdm7>_WecWF#D=?eMjLj=-_z`aHMZ=3_-&E8;ibPmM}61i6J3is*=dKf%HC>=xbj4$ zS|Q-hWQ8T5mWde6h@;mS+?k=89?1FU<%qH9B(l&O>k|u_aD|DY*@~(`_pb|B#rJ&g zR0(~(68fpUPz6TdS@4JT5MOPrqDh5_H(eX1$P2SQrkvN8sTxwV>l0)Qq z0pzTuvtEAKRDkKGhhv^jk%|HQ1DdF%5oKq5BS>szk-CIke{%js?~%@$uaN3^Uz6Wf z_iyx{bZ(;9y4X&>LPV=L=d+A}7I4GkK0c1Xts{rrW1Q7apHf-))`BgC^0^F(>At1* za@e7{lq%yAkn*NH8Q1{@{lKhRg*^TfGvv!Sn*ed*x@6>M%aaqySxR|oNadYt1mpUZ z6H(rupHYf&Z z29$5g#|0MX#aR6TZ$@eGxxABRKakDYtD%5BmKp;HbG_ZbT+=81E&=XRk6m_3t9PvD zr5Cqy(v?gHcYvYvXkNH@S#Po~q(_7MOuCAB8G$a9BC##gw^5mW16cML=T=ERL7wsk zzNEayTG?mtB=x*wc@ifBCJ|irFVMOvH)AFRW8WE~U()QT=HBCe@s$dA9O!@`zAAT) zaOZ7l6vyR+Nk_OOF!ZlZmjoImKh)dxFbbR~z(cMhfeX1l7S_`;h|v3gI}n9$sSQ>+3@AFAy9=B_y$)q;Wdl|C-X|VV3w8 z2S#>|5dGA8^9%Bu&fhmVRrTX>Z7{~3V&0UpJNEl0=N32euvDGCJ>#6dUSi&PxFW*s zS`}TB>?}H(T2lxBJ!V#2taV;q%zd6fOr=SGHpoSG*4PDaiG0pdb5`jelVipkEk%FV zThLc@Hc_AL1#D&T4D=w@UezYNJ%0=f3iVRuVL5H?eeZM}4W*bomebEU@e2d`M<~uW zf#Bugwf`VezG|^Qbt6R_=U0}|=k;mIIakz99*>FrsQR{0aQRP6ko?5<7bkDN8evZ& zB@_KqQG?ErKL=1*ZM9_5?Pq%lcS4uLSzN(Mr5=t6xHLS~Ym`UgM@D&VNu8e?_=nSFtF$u@hpPSmI4Vo_t&v?>$~K4y(O~Rb*(MFy_igM7 z*~yYUyR6yQgzWnWMUgDov!!g=lInM+=lOmOk4L`O?{i&qxy&D*_qorRbDwj6?)!ef z#JLd7F6Z2I$S0iYI={rZNk*<{HtIl^mx=h>Cim*04K4+Z4IJtd*-)%6XV2(MCscPiw_a+y*?BKbTS@BZ3AUao^%Zi#PhoY9Vib4N>SE%4>=Jco0v zH_Miey{E;FkdlZSq)e<{`+S3W=*ttvD#hB8w=|2aV*D=yOV}(&p%0LbEWH$&@$X3x~CiF-?ejQ*N+-M zc8zT@3iwkdRT2t(XS`d7`tJQAjRmKAhiw{WOqpuvFp`i@Q@!KMhwKgsA}%@sw8Xo5Y=F zhRJZg)O4uqNWj?V&&vth*H#je6T}}p_<>!Dr#89q@uSjWv~JuW(>FqoJ5^ho0%K?E z9?x_Q;kmcsQ@5=}z@tdljMSt9-Z3xn$k)kEjK|qXS>EfuDmu(Z8|(W?gY6-l z@R_#M8=vxKMAoi&PwnaIYw2COJM@atcgfr=zK1bvjW?9B`-+Voe$Q+H$j!1$Tjn+* z&LY<%)L@;zhnJlB^Og6I&BOR-m?{IW;tyYC%FZ!&Z>kGjHJ6cqM-F z&19n+e1=9AH1VrVeHrIzqlC`w9=*zfmrerF?JMzO&|Mmv;!4DKc(sp+jy^Dx?(8>1 zH&yS_4yL7m&GWX~mdfgH*AB4{CKo;+egw=PrvkTaoBU+P-4u?E|&!c z)DKc;>$$B6u*Zr1SjUh2)FeuWLWHl5TH(UHWkf zLs>7px!c5n;rbe^lO@qlYLzlDVp(z?6rPZel=YB)Uv&n!2{+Mb$-vQl=xKw( zve&>xYx+jW_NJh!FV||r?;hdP*jOXYcLCp>DOtJ?2S^)DkM{{Eb zS$!L$e_o0(^}n3tA1R3-$SNvgBq;DOEo}fNc|tB%%#g4RA3{|euq)p+xd3I8^4E&m zFrD%}nvG^HUAIKe9_{tXB;tl|G<%>yk6R;8L2)KUJw4yHJXUOPM>(-+jxq4R;z8H#>rnJy*)8N+$wA$^F zN+H*3t)eFEgxLw+Nw3};4WV$qj&_D`%ADV2%r zJCPCo%{=z7;`F98(us5JnT(G@sKTZ^;2FVitXyLe-S5(hV&Ium+1pIUB(CZ#h|g)u zSLJJ<@HgrDiA-}V_6B^x1>c9B6%~847JkQ!^KLZ2skm;q*edo;UA)~?SghG8;QbHh z_6M;ouo_1rq9=x$<`Y@EA{C%6-pEV}B(1#sDoe_e1s3^Y>n#1Sw;N|}8D|s|VPd+g z-_$QhCz`vLxxrVMx3ape1xu3*wjx=yKSlM~nFgkNWb4?DDr*!?U)L_VeffF<+!j|b zZ$Wn2$TDv3C3V@BHpSgv3JUif8%hk%OsGZ=OxH@8&4`bbf$`aAMchl^qN>Eyu3JH} z9-S!x8-s4fE=lad%Pkp8hAs~u?|uRnL48O|;*DEU! zuS0{cpk%1E0nc__2%;apFsTm0bKtd&A0~S3Cj^?72-*Owk3V!ZG*PswDfS~}2<8le z5+W^`Y(&R)yVF*tU_s!XMcJS`;(Tr`J0%>p=Z&InR%D3@KEzzI+-2)HK zuoNZ&o=wUC&+*?ofPb0a(E6(<2Amd6%uSu_^-<1?hsxs~0K5^f(LsGqgEF^+0_H=uNk9S0bb!|O8d?m5gQjUKevPaO+*VfSn^2892K~%crWM8+6 z25@V?Y@J<9w%@NXh-2!}SK_(X)O4AM1-WTg>sj1{lj5@=q&dxE^9xng1_z9w9DK>| z6Iybcd0e zyi;Ew!KBRIfGPGytQ6}z}MeXCfLY0?9%RiyagSp_D1?N&c{ zyo>VbJ4Gy`@Fv+5cKgUgs~na$>BV{*em7PU3%lloy_aEovR+J7TfQKh8BJXyL6|P8un-Jnq(ghd!_HEOh$zlv2$~y3krgeH;9zC}V3f`uDtW(%mT#944DQa~^8ZI+zAUu4U(j0YcDfKR$bK#gvn_{JZ>|gZ5+)u?T$w7Q%F^;!Wk?G z(le7r!ufT*cxS}PR6hIVtXa)i`d$-_1KkyBU>qmgz-=T};uxx&sKgv48akIWQ89F{ z0XiY?WM^~;|T8zBOr zs#zuOONzH?svv*jokd5SK8wG>+yMC)LYL|vLqm^PMHcT=`}V$=nIRHe2?h)8WQa6O zPAU}d`1y(>kZiP~Gr=mtJLMu`i<2CspL|q2DqAgAD^7*$xzM`PU4^ga`ilE134XBQ z99P(LhHU@7qvl9Yzg$M`+dlS=x^(m-_3t|h>S}E0bcFMn=C|KamQ)=w2^e)35p`zY zRV8X?d;s^>Cof2SPR&nP3E+-LCkS0J$H!eh8~k0qo$}00b=7!H_I2O+Ro@3O$nPdm ztmbOO^B+IHzQ5w>@@@J4cKw5&^_w6s!s=H%&byAbUtczPQ7}wfTqxxtQNfn*u73Qw zGuWsrky_ajPx-5`R<)6xHf>C(oqGf_Fw|-U*GfS?xLML$kv;h_pZ@Kk$y0X(S+K80 z6^|z)*`5VUkawg}=z`S;VhZhxyDfrE0$(PMurAxl~<>lfZa>JZ288ULK7D` zl9|#L^JL}Y$j*j`0-K6kH#?bRmg#5L3iB4Z)%iF@SqT+Lp|{i`m%R-|ZE94Np7Pa5 zCqC^V3}B(FR340pmF*qaa}M}+h6}mqE~7Sh!9bDv9YRT|>vBNAqv09zXHMlcuhKD| zcjjA(b*XCIwJ33?CB!+;{)vX@9xns_b-VO{i0y?}{!sdXj1GM8+$#v>W7nw;+O_9B z_{4L;C6ol?(?W0<6taGEn1^uG=?Q3i29sE`RfYCaV$3DKc_;?HsL?D_fSYg}SuO5U zOB_f4^vZ_x%o`5|C@9C5+o=mFy@au{s)sKw!UgC&L35aH(sgDxRE2De%(%OT=VUdN ziVLEmdOvJ&5*tCMKRyXctCwQu_RH%;m*$YK&m;jtbdH#Ak~13T1^f89tn`A%QEHWs~jnY~E}p_Z$XC z=?YXLCkzVSK+Id`xZYTegb@W8_baLt-Fq`Tv|=)JPbFsKRm)4UW;yT+J`<)%#ue9DPOkje)YF2fsCilK9MIIK>p*`fkoD5nGfmLwt)!KOT+> zOFq*VZktDDyM3P5UOg`~XL#cbzC}eL%qMB=Q5$d89MKuN#$6|4gx_Jt0Gfn8w&q}%lq4QU%6#jT*MRT% zrLz~C8FYKHawn-EQWN1B75O&quS+Z81(zN)G>~vN8VwC+e+y(`>HcxC{MrJ;H1Z4k zZWuv$w_F0-Ub%MVcpIc){4PGL^I7M{>;hS?;eH!;gmcOE66z3;Z1Phqo(t zVP(Hg6q#0gIKgsg7L7WE!{Y#1nI(45tx2{$34dDd#!Z0NIyrm)HOn5W#7;f4pQci# zDW!FI(g4e668kI9{2+mLwB+=#9bfqgX%!B34V-$wwSN(_cm*^{y0jQtv*4}eO^sOV z*9xoNvX)c9isB}Tgx&ZRjp3kwhTVK?r9;n!x>^XYT z@Q^7zp{rkIs{2mUSE^2!Gf6$6;j~&4=-0cSJJDizZp6LTe8b45;{AKM%v99}{{FfC zz709%u0mC=1KXTo(=TqmZQ;c?$M3z(!xah>aywrj40sc2y3rKFw4jCq+Y+u=CH@_V zxz|qeTwa>+<|H%8Dz5u>ZI5MmjTFwXS-Fv!TDd*`>3{krWoNVx$<133`(ftS?ZPyY z&4@ah^3^i`vL$BZa>O|Nt?ucewzsF)0zX3qmM^|waXr=T0pfIb0*$AwU=?Ipl|1Y; z*Pk6{C-p4MY;j@IJ|DW>QHZQJcp;Z~?8(Q+Kk3^0qJ}SCk^*n4W zu9ZFwLHUx-$6xvaQ)SUQcYd6fF8&x)V`1bIuX@>{mE$b|Yd(qomn3;bPwnDUc0F=; zh*6_((%bqAYQWQ~odER?h>1mkL4kpb3s7`0m@rDKGU*oyF)$j~Ffd4fXV$?`f~rHf zB%Y)@5SXZvfwm10RY5X?TEo)PK_`L6qgBp=#>fO49$D zDq8Ozj0q6213tV5Qq=;fZ0$|KroY{Dz=l@lU^J)?Ko@ti20TRplXzphBi>XGx4bou zEWrkNjz0t5j!_ke{g5I#PUlEU$Km8g8TE|XK=MkU@PT4T><2OVamoK;wJ}3X0L$vX zgd7gNa359*nc)R-0!`2X@FOTB`+oETOPc=ubp5R)VQgY+5BTZZJ2?9QwnO=dnulIUF3gFn;BODC2)65)HeVd%t86sL7Rv^Y+nbn+&l z6BAJY(ETvwI)Ts$aiE8rht4KD*qNyE{8{x6R|%akbTBzw;2+6Echkt+W+`u^XX z_z&x%nzbL-0C3_3~ zRZ#mYf6f1oqJoH`jHHCB8l!^by~4z}yc`4LEP@;Z?bO6{g9`Hk+s@(L1jC5Tq{1Yf z4E;CQvrx0-gF+peRxFC*gF=&$zNYjO?K|gN=WqXMz`tYs@0o%B{dRD+{C_6(f9t^g zhmNJQv6-#;f2)f2uc{u-#*U8W&i{|ewYN^n_1~cv|1J!}zc&$eaBy{T{cEpa46s*q zHFkD2cV;xTHFj}{*3kBt*FgS4A5SI|$F%$gB@It9FlC}D3y`sbZG{2P6gGwC$U`6O zb_cId9AhQl#A<&=x>-xDD%=Ppt$;y71@Lwsl{x943#T@8*?cbR<~d`@@}4V${+r$jICUIOzgZJy_9I zu*eA(F)$~J07zX%tmQN}1^wj+RM|9bbwhQA=xrPE*{vB_P!pPYT5{Or^m*;Qz#@Bl zRywCG_RDyM6bf~=xn}FtiFAw|rrUxa1+z^H`j6e|GwKDuq}P)z&@J>MEhsVBvnF|O zOEm)dADU1wi8~mX(j_8`DwMT_OUAnjbWYer;P*^Uku_qMu3}qJU zTAkza-K9aj&wcsGuhQ>RQoD?gz~L8RwCHOZDzhBD$az*$TQ3!uygnx_rsXG`#_x5t zn*lb(%JI3%G^MpYp-Y(KI4@_!&kBRa3q z|Fzn&3R%ZsoMNEn4pN3-BSw2S_{IB8RzRv(eQ1X zyBQZHJ<(~PfUZ~EoI!Aj`9k<+Cy z2DtI<+9sXQu!6&-Sk4SW3oz}?Q~mFvy(urUy<)x!KQ>#7yIPC)(ORhKl7k)4eSy~} z7#H3KG<|lt68$tk^`=yjev%^usOfpQ#+Tqyx|b#dVA(>fPlGuS@9ydo z!Cs#hse9nUETfGX-7lg;F>9)+ml@M8OO^q|W~NiysX2N|2dH>qj%NM`=*d3GvES_# zyLEHw&1Fx<-dYxCQbk_wk^CI?W44%Q9!!9aJKZW-bGVhK?N;q`+Cgc*WqyXcxZ%U5QXKu!Xn)u_dxeQ z;uw9Vysk!3OFzUmVoe)qt3ifPin0h25TU zrG*03L~0|aaBg7^YPEW^Yq3>mSNQgk-o^CEH?wXZ^QiPiuH}jGk;75PUMNquJjm$3 zLcXN*uDRf$Jukqg3;046b;3s8zkxa_6yAlG{+7{81O3w96i_A$KcJhD&+oz1<>?lun#C3+X0q zO4JxN{qZ!e#FCl@e_3G?0I^$CX6e$cy7$BL#4<`AA)Lw+k`^15pmb-447~5lkSMZ` z>Ce|adKhb-F%yy!vx>yQbXFgHyl(an=x^zi(!-~|k;G1=E(e@JgqbAF{;nv`3i)oi zDeT*Q+Mp{+NkURoabYb9@#Bi5FMQnBFEU?H{~9c;g3K%m{+^hNe}(MdpPb?j9`?2l z#%AO!|2QxGq7-2Jn2|%atvGb(+?j&lmP509i5y87`9*BSY++<%%DXb)kaqG0(4Eft zj|2!Od~2TfVTi^0dazAIeVe&b#{J4DjN6;4W;M{yWj7#+oLhJyqeRaO;>?%mX>Ec{Mp~;`bo}p;`)@5dA8fNQ38FyMf;wUPOdZS{U*8SN6xa z-kq3>*Zos!2`FMA7qjhw-`^3ci%c91Lh`;h{qX1r;x1}eW2hYaE*3lTk4GwenoxQ1kHt1Lw!*N8Z%DdZSGg5~Bw}+L!1#d$u+S=Bzo7gi zqGsBV29i)Jw(vix>De)H&PC; z-t2OX_ak#~eSJ?Xq=q9A#0oaP*dO7*MqV;dJv|aUG00UX=cIhdaet|YEIhv6AUuyM zH1h7fK9-AV)k8sr#POIhl+?Z^r?wI^GE)ZI=H!WR<|UI(3_YUaD#TYV$Fxd015^mT zpy&#-IK>ahfBlJm-J(n(A%cKV;)8&Y{P!E|AHPtRHk=XqvYUX?+9po4B$0-6t74UUef${01V{QLEE8gzw* z5nFnvJ|T4dlRiW9;Ed_yB{R@)fC=zo4hCtD?TPW*WJmMXYxN_&@YQYg zBQ$XRHa&EE;YJrS{bn7q?}Y&DH*h;){5MmE(9A6aSU|W?{3Ox%5fHLFScv7O-txuRbPG1KQtI`Oay=IcEG=+hPhlnYC;`wSHeo|XGio0aTS6&W($E$ z?N&?TK*l8;Y^-xPl-WVZwrfdiQv10KdsAb9u-*1co*0-Z(h#H)k{Vc5CT!708cs%sExvPC+7-^UY~jTfFq=cj z!Dmy<+NtKp&}}$}rD{l?%MwHdpE(cPCd;-QFPk1`E5EVNY2i6E`;^aBlx4}h*l42z zpY#2cYzC1l6EDrOY*ccb%kP;k8LHE3tP>l3iK?XZ%FI<3666yPw1rM%>eCgnv^JS_ zK7c~;g7yXt9fz@(49}Dj7VO%+P!eEm& z;z8UXs%NsQ%@2S5nve)@;yT^61BpVlc}=+i6{ZZ9r7<({yUYqe==9*Z+HguP3`sA& z{`inI4G)eLieUQ*pH9M@)u7yVnWTQva;|xq&-B<>MoP(|xP(HqeCk1&h>DHNLT>Zi zQ$uH%s6GoPAi0~)sC;`;ngsk+StYL9NFzhFEoT&Hzfma1f|tEnL0 zMWdX4(@Y*?*tM2@H<#^_l}BC&;PYJl%~E#veQ61{wG6!~nyop<^e)scV5#VkGjYc2 z$u)AW-NmMm%T7WschOnQ!Hbbw&?`oMZrJ&%dVlN3VNra1d0TKfbOz{dHfrCmJ2Jj= zS#Gr}JQcVD?S9X!u|oQ7LZ+qcq{$40 ziG5=X^+WqeqxU00YuftU7o;db=K+Tq!y^daCZgQ)O=M} zK>j*<3oxs=Rcr&W2h%w?0Cn3);~vqG>JO_tTOzuom^g&^vzlEjkx>Sv!@NNX%_C!v zaMpB>%yVb}&ND9b*O>?HxQ$5-%@xMGe4XKjWh7X>CYoRI2^JIwi&3Q5UM)?G^k8;8 zmY$u;(KjZx>vb3fe2zgD7V;T2_|1KZQW$Yq%y5Ioxmna9#xktcgVitv7Sb3SlLd6D zfmBM9Vs4rt1s0M}c_&%iP5O{Dnyp|g1(cLYz^qLqTfN6`+o}59Zlu%~oR3Q3?{Bnr zkx+wTpeag^G12fb_%SghFcl|p2~<)Av?Agumf@v7y-)ecVs`US=q~=QG%(_RTsqQi z%B&JdbOBOmoywgDW|DKR5>l$1^FPhxsBrja<&}*pfvE|5dQ7j-wV|ur%QUCRCzBR3q*X`05O3U@?#$<>@e+Zh&Z&`KfuM!0XL& zI$gc@ZpM4o>d&5)mg7+-Mmp98K^b*28(|Ew8kW}XEV7k^vnX-$onm9OtaO@NU9a|as7iA%5Wrw9*%UtJYacltplA5}gx^YQM` zVkn`TIw~avq)mIQO0F0xg)w$c)=8~6Jl|gdqnO6<5XD)&e7z7ypd3HOIR+ss0ikSVrWar?548HFQ*+hC)NPCq*;cG#B$7 z!n?{e9`&Nh-y}v=nK&PR>PFdut*q&i81Id`Z<0vXUPEbbJ|<~_D!)DJMqSF~ly$tN zygoa)um~xdYT<7%%m!K8+V(&%83{758b0}`b&=`))Tuv_)OL6pf=XOdFk&Mfx9y{! z6nL>V?t=#eFfM$GgGT8DgbGRCF@0ZcWaNs_#yl+6&sK~(JFwJmN-aHX{#Xkpmg;!} zgNyYYrtZdLzW1tN#QZAh!z5>h|At3m+ryJ-DFl%V>w?cmVTxt^DsCi1ZwPaCe*D{) z?#AZV6Debz{*D#C2>44Czy^yT3y92AYDcIXtZrK{L-XacVl$4i=X2|K=Fy5vAzhk{ zu3qG=qSb_YYh^HirWf~n!_Hn;TwV8FU9H8+=BO)XVFV`nt)b>5yACVr!b98QlLOBDY=^KS<*m9@_h3;64VhBQzb_QI)gbM zSDto2i*iFrvxSmAIrePB3i`Ib>LdM8wXq8(R{-)P6DjUi{2;?}9S7l7bND4w%L2!; zUh~sJ(?Yp}o!q6)2CwG*mgUUWlZ;xJZo`U`tiqa)H4j>QVC_dE7ha0)nP5mWGB268 zn~MVG<#fP#R%F=Ic@(&Va4dMk$ysM$^Avr1&hS!p=-7F>UMzd(M^N9Ijb|364}qcj zcIIh7suk$fQE3?Z^W4XKIPh~|+3(@{8*dSo&+Kr(J4^VtC{z*_{2}ld<`+mDE2)S| zQ}G#Q0@ffZCw!%ZGc@kNoMIdQ?1db%N1O0{IPPesUHI;(h8I}ETudk5ESK#boZgln z(0kvE`&6z1xH!s&={%wQe;{^&5e@N0s7IqR?L*x%iXM_czI5R1aU?!bA7)#c4UN2u zc_LZU+@elD5iZ=4*X&8%7~mA;SA$SJ-8q^tL6y)d150iM)!-ry@TI<=cnS#$kJAS# zq%eK**T*Wi2OlJ#w+d_}4=VN^A%1O+{?`BK00wkm)g8;u?vM;RR+F1G?}({ENT3i= zQsjJkp-dmJ&3-jMNo)wrz0!g*1z!V7D(StmL(A}gr^H-CZ~G9u?*Uhcx|x7rb`v^X z9~QGx;wdF4VcxCmEBp$F#sms@MR?CF67)rlpMxvwhEZLgp2?wQq|ci#rLtrYRV~iR zN?UrkDDTu114&d~Utjcyh#tXE_1x%!dY?G>qb81pWWH)Ku@Kxbnq0=zL#x@sCB(gs zm}COI(!{6-XO5li0>1n}Wz?w7AT-Sp+=NQ1aV@fM$`PGZjs*L+H^EW&s!XafStI!S zzgdntht=*p#R*o8-ZiSb5zf6z?TZr$^BtmIfGAGK;cdg=EyEG)fc*E<*T=#a?l=R5 zv#J;6C(umoSfc)W*EODW4z6czg3tXIm?x8{+8i^b;$|w~k)KLhJQnNW7kWXcR^sol z1GYOp?)a+}9Dg*nJ4fy*_riThdkbHO37^csfZRGN;CvQOtRacu6uoh^gg%_oEZKDd z?X_k67s$`|Q&huidfEonytrq!wOg07H&z@`&BU6D114p!rtT2|iukF}>k?71-3Hk< zs6yvmsMRO%KBQ44X4_FEYW~$yx@Y9tKrQ|rC1%W$6w}-9!2%4Zk%NycTzCB=nb)r6*92_Dg+c0;a%l1 zsJ$X)iyYR2iSh|%pIzYV1OUWER&np{w1+RXb~ zMUMRymjAw*{M)UtbT)T!kq5ZAn%n=gq3ssk3mYViE^$paZ;c^7{vXDJ`)q<}QKd2?{r9`X3mpZ{AW^UaRe2^wWxIZ$tuyKzp#!X-hXkHwfD zj@2tA--vFi3o_6B?|I%uwD~emwn0a z+?2Lc1xs(`H{Xu>IHXpz=@-84uw%dNV;{|c&ub|nFz(=W-t4|MME(dE4tZQi?0CE|4_?O_dyZj1)r zBcqB8I^Lt*#)ABdw#yq{OtNgf240Jvjm8^zdSf40 z;H)cp*rj>WhGSy|RC5A@mwnmQ`y4{O*SJ&S@UFbvLWyPdh)QnM=(+m3p;0&$^ysbZ zJt!ZkNQ%3hOY*sF2_~-*`aP|3Jq7_<18PX*MEUH*)t{eIx%#ibC|d&^L5FwoBN}Oe z?!)9RS@Zz%X1mqpHgym75{_BM4g)k1!L{$r4(2kL<#Oh$Ei7koqoccI3(MN1+6cDJ zp=xQhmilz1?+ZjkX%kfn4{_6K_D{wb~rdbkh!!k!Z@cE z^&jz55*QtsuNSlGPrU=R?}{*_8?4L7(+?>?(^3Ss)f!ou&{6<9QgH>#2$?-HfmDPN z6oIJ$lRbDZb)h-fFEm^1-v?Slb8udG{7GhbaGD_JJ8a9f{6{TqQN;m@$&)t81k77A z?{{)61za|e2GEq2)-OqcEjP`fhIlUs_Es-dfgX-3{S08g`w=wGj2{?`k^GD8d$}6Z zBT0T1lNw~fuwjO5BurKM593NGYGWAK%UCYiq{$p^GoYz^Uq0$YQ$j5CBXyog8(p_E znTC+$D`*^PFNc3Ih3b!2Lu|OOH6@46D)bbvaZHy%-9=$cz}V^|VPBpmPB6Ivzlu&c zPq6s7(2c4=1M;xlr}bkSmo9P`DAF>?Y*K%VPsY`cVZ{mN&0I=jagJ?GA!I;R)i&@{ z0Gl^%TLf_N`)`WKs?zlWolWvEM_?{vVyo(!taG$`FH2bqB`(o50pA=W34kl-qI62lt z1~4LG_j%sR2tBFteI{&mOTRVU7AH>>-4ZCD_p6;-J<=qrod`YFBwJz(Siu(`S}&}1 z6&OVJS@(O!=HKr-Xyzuhi;swJYK*ums~y1ePdX#~*04=b9)UqHHg;*XJOxnS6XK#j zG|O$>^2eW2ZVczP8#$C`EpcWwPFX4^}$omn{;P(fL z>J~%-r5}*D3$Kii z34r@JmMW2XEa~UV{bYP=F;Y5=9miJ+Jw6tjkR+cUD5+5TuKI`mSnEaYE2=usXNBs9 zac}V13%|q&Yg6**?H9D620qj62dM+&&1&a{NjF}JqmIP1I1RGppZ|oIfR}l1>itC% zl>ed${{_}8^}m2^br*AIX$L!Vc?Sm@H^=|LnpJg`a7EC+B;)j#9#tx-o0_e4!F5-4 zF4gA;#>*qrpow9W%tBzQ89U6hZ9g=-$gQpCh6Nv_I0X7t=th2ajJ8dBbh{i)Ok4{I z`Gacpl?N$LjC$tp&}7Sm(?A;;Nb0>rAWPN~@3sZ~0_j5bR+dz;Qs|R|k%LdreS3Nn zp*36^t#&ASm=jT)PIjNqaSe4mTjAzlAFr*@nQ~F+Xdh$VjHWZMKaI+s#FF#zjx)BJ zufxkW_JQcPcHa9PviuAu$lhwPR{R{7CzMUi49=MaOA%ElpK;A)6Sgsl7lw)D$8FwE zi(O6g;m*86kcJQ{KIT-Rv&cbv_SY4 zpm1|lSL*o_1LGOlBK0KuU2?vWcEcQ6f4;&K=&?|f`~X+s8H)se?|~2HcJo{M?Ity) zE9U!EKGz2^NgB6Ud;?GcV*1xC^1RYIp&0fr;DrqWLi_Kts()-#&3|wz{wFQsKfnnsC||T?oIgUp z{O(?Df7&vW!i#_~*@naguLLjDAz+)~*_xV2iz2?(N|0y8DMneikrT*dG`mu6vdK`% z=&nX5{F-V!Reau}+w_V3)4?}h@A@O)6GCY7eXC{p-5~p8x{cH=hNR;Sb{*XloSZ_%0ZKYG=w<|!vy?spR4!6mF!sXMUB5S9o_lh^g0!=2m55hGR; z-&*BZ*&;YSo474=SAM!WzrvjmNtq17L`kxbrZ8RN419e=5CiQ-bP1j-C#@@-&5*(8 zRQdU~+e(teUf}I3tu%PB1@Tr{r=?@0KOi3+Dy8}+y#bvgeY(FdN!!`Kb>-nM;7u=6 z;0yBwOJ6OdWn0gnuM{0`*fd=C(f8ASnH5aNYJjpbY1apTAY$-%)uDi$%2)lpH=#)=HH z<9JaYwPKil@QbfGOWvJ?cN6RPBr`f+jBC|-dO|W@x_Vv~)bmY(U(!cs6cnhe0z31O z>yTtL4@KJ*ac85u9|=LFST22~!lb>n7IeHs)_(P_gU}|8G>{D_fJX)8BJ;Se? z67QTTlTzZykb^4!{xF!=C}VeFd@n!9E)JAK4|vWVwWop5vSWcD<;2!88v-lS&ve7C zuYRH^85#hGKX(Mrk};f$j_V&`Nb}MZy1mmfz(e`nnI4Vpq(R}26pZx?fq%^|(n~>* z5a5OFtFJJfrZmgjyHbj1`9||Yp?~`p2?4NCwu_!!*4w8K`&G7U_|np&g7oY*-i;sI zu)~kYH;FddS{7Ri#Z5)U&X3h1$Mj{{yk1Q6bh4!7!)r&rqO6K~{afz@bis?*a56i& zxi#(Ss6tkU5hDQJ0{4sKfM*ah0f$>WvuRL zunQ-eOqa3&(rv4kiQ(N4`FO6w+nko_HggKFWx@5aYr}<~8wuEbD(Icvyl~9QL^MBt zSvD)*C#{2}!Z55k1ukV$kcJLtW2d~%z$t0qMe(%2qG`iF9K_Gsae7OO%Tf8E>ooch ztAw01`WVv6?*14e1w%Wovtj7jz_)4bGAqqo zvTD|B4)Ls8x7-yr6%tYp)A7|A)x{WcI&|&DTQR&2ir(KGR7~_RhNOft)wS<+vQ*|sf;d>s zEfl&B^*ZJp$|N`w**cXOza8(ARhJT{O3np#OlfxP9Nnle4Sto)Fv{w6ifKIN^f1qO*m8+MOgA1^Du!=(@MAh8)@wU8t=Ymh!iuT_lzfm za~xEazL-0xwy9$48!+?^lBwMV{!Gx)N>}CDi?Jwax^YX@_bxl*+4itP;DrTswv~n{ zZ0P>@EB({J9ZJ(^|ptn4ks^Z2UI&87d~J_^z0&vD2yb%*H^AE!w= zm&FiH*c%vvm{v&i3S>_hacFH${|(2+q!`X~zn4$aJDAry>=n|{C7le(0a)nyV{kAD zlud4-6X>1@-XZd`3SKKHm*XNn_zCyKHmf*`C_O509$iy$Wj`Sm3y?nWLCDy>MUx1x zl-sz7^{m(&NUk*%_0(G^>wLDnXW90FzNi$Tu6* z<+{ePBD`%IByu977rI^x;gO5M)Tfa-l*A2mU-#IL2?+NXK-?np<&2rlF;5kaGGrx2 zy8Xrz`kHtTVlSSlC=nlV4_oCsbwyVHG4@Adb6RWzd|Otr!LU=% zEjM5sZ#Ib4#jF(l!)8Na%$5VK#tzS>=05GpV?&o* z3goH1co0YR=)98rPJ~PuHvkA59KUi#i(Mq_$rApn1o&n1mUuZfFLjx@3;h`0^|S##QiTP8rD`r8P+#D@gvDJh>amMIl065I)PxT6Hg(lJ?X7*|XF2Le zv36p8dWHCo)f#C&(|@i1RAag->5ch8TY!LJ3(+KBmLxyMA%8*X%_ARR*!$AL66nF= z=D}uH)D)dKGZ5AG)8N-;Il*-QJ&d8u30&$_Q0n1B58S0ykyDAyGa+BZ>FkiOHm1*& zNOVH;#>Hg5p?3f(7#q*dL74;$4!t?a#6cfy#}9H3IFGiCmevir5@zXQj6~)@zYrWZ zRl*e66rjwksx-)Flr|Kzd#Bg>We+a&E{h7bKSae9P~ z(g|zuXmZ zD?R*MlmoZ##+0c|cJ(O{*h(JtRdA#lChYhfsx25(Z`@AK?Q-S8_PQqk z>|Z@Ki1=wL1_c6giS%E4YVYD|Y-{^ZzFwB*yN8-4#+TxeQ`jhks7|SBu7X|g=!_XL z`mY=0^chZfXm%2DYHJ4z#soO7=NONxn^K3WX={dV>$CTWSZe@<81-8DVtJEw#Uhd3 zxZx+($6%4a&y_rD8a&E`4$pD6-_zZJ%LEE*1|!9uOm!kYXW< zOBXZAowsX-&$5C`xgWkC43GcnY)UQt2Qkib4!!8Mh-Q!_M%5{EC=Gim@_;0+lP%O^ zG~Q$QmatQk{Mu&l{q~#kOD;T-{b1P5u7)o-QPPnqi?7~5?7%IIFKdj{;3~Hu#iS|j z)Zoo2wjf%+rRj?vzWz(6JU`=7H}WxLF*|?WE)ci7aK?SCmd}pMW<{#1Z!_7BmVP{w zSrG>?t}yNyCR%ZFP?;}e8_ zRy67~&u11TN4UlopWGj6IokS{vB!v!n~TJYD6k?~XQkpiPMUGLG2j;lh>Eb5bLTkX zx>CZlXdoJsiPx=E48a4Fkla>8dZYB%^;Xkd(BZK$z3J&@({A`aspC6$qnK`BWL;*O z-nRF{XRS`3Y&b+}G&|pE1K-Ll_NpT!%4@7~l=-TtYRW0JJ!s2C-_UsRBQ=v@VQ+4> z*6jF0;R@5XLHO^&PFyaMDvyo?-lAD(@H61l-No#t@at@Le9xOgTFqkc%07KL^&iss z!S2Ghm)u#26D(e1Q7E;L`rxOy-N{kJ zTgfw}az9=9Su?NEMMtpRlYwDxUAUr8F+P=+9pkX4%iA4&&D<|=B|~s*-U+q6cq`y* zIE+;2rD7&D5X;VAv=5rC5&nP$E9Z3HKTqIFCEV%V;b)Y|dY?8ySn|FD?s3IO>VZ&&f)idp_7AGnwVd1Z znBUOBA}~wogNpEWTt^1Rm-(YLftB=SU|#o&pT7vTr`bQo;=ZqJHIj2MP{JuXQPV7% z0k$5Ha6##aGly<}u>d&d{Hkpu?ZQeL_*M%A8IaXq2SQl35yW9zs4^CZheVgHF`%r= zs(Z|N!gU5gj-B^5{*sF>;~fauKVTq-Ml2>t>E0xl9wywD&nVYZfs1F9Lq}(clpNLz z4O(gm_i}!k`wUoKr|H#j#@XOXQ<#eDGJ=eRJjhOUtiKOG;hym-1Hu)1JYj+Kl*To<8( za1Kf4_Y@Cy>eoC59HZ4o&xY@!G(2p^=wTCV>?rQE`Upo^pbhWdM$WP4HFdDy$HiZ~ zRUJFWTII{J$GLVWR?miDjowFk<1#foE3}C2AKTNFku+BhLUuT>?PATB?WVLzEYyu+ zM*x((pGdotzLJ{}R=OD*jUexKi`mb1MaN0Hr(Wk8-Uj0zA;^1w2rmxLI$qq68D>^$ zj@)~T1l@K|~@YJ6+@1vlWl zHg5g%F{@fW5K!u>4LX8W;ua(t6YCCO_oNu}IIvI6>Fo@MilYuwUR?9p)rKNzDmTAN zzN2d>=Za&?Z!rJFV*;mJ&-sBV80%<-HN1;ciLb*Jk^p?u<~T25%7jjFnorfr={+wm zzl5Q6O>tsN8q*?>uSU6#xG}FpAVEQ_++@}G$?;S7owlK~@trhc#C)TeIYj^N(R&a} zypm~c=fIs;M!YQrL}5{xl=tUU-Tfc0ZfhQuA-u5(*w5RXg!2kChQRd$Fa8xQ0CQIU zC`cZ*!!|O!*y1k1J^m8IIi|Sl3R}gm@CC&;4840^9_bb9%&IZTRk#=^H0w%`5pMDCUef5 zYt-KpWp2ijh+FM`!zZ35>+7eLN;s3*P!bp%-oSx34fdTZ14Tsf2v7ZrP+mitUx$rS zW(sOi^CFxe$g3$x45snQwPV5wpf}>5OB?}&Gh<~i(mU&ss#7;utaLZ!|KaTHniGO9 zVC9OTzuMKz)afey_{93x5S*Hfp$+r*W>O^$2ng|ik!<`U1pkxm3*)PH*d#>7md1y} zs7u^a8zW8bvl92iN;*hfOc-=P7{lJeJ|3=NfX{(XRXr;*W3j845SKG&%N zuBqCtDWj*>KooINK1 zFPCsCWr!-8G}G)X*QM~34R*k zmRmDGF*QE?jCeNfc?k{w<}@29e}W|qKJ1K|AX!htt2|B`nL=HkC4?1bEaHtGBg}V( zl(A`6z*tck_F$4;kz-TNF%7?=20iqQo&ohf@S{_!TTXnVh}FaW2jxAh(DI0f*SDG- z7tqf5X@p#l?7pUNI(BGi>n_phw=lDm>2OgHx-{`T>KP2YH9Gm5ma zb{>7>`tZ>0d5K$j|s2!{^sFWQo3+xDb~#=9-jp(1ydI3_&RXGB~rxWSMgDCGQG)oNoc#>)td zqE|X->35U?_M6{^lB4l(HSN|`TC2U*-`1jSQeiXPtvVXdN-?i1?d#;pw%RfQuKJ|e zjg75M+Q4F0p@8I3ECpBhGs^kK;^0;7O@MV=sX^EJLVJf>L;GmO z3}EbTcoom7QbI(N8ad!z(!6$!MzKaajSRb0c+ZDQ($kFT&&?GvXmu7+V3^_(VJx1z zP-1kW_AB&_A;cxm*g`$ z#Pl@Cg{siF0ST2-w)zJkzi@X)5i@)Z;7M5ewX+xcY36IaE0#flASPY2WmF8St0am{ zV|P|j9wqcMi%r-TaU>(l*=HxnrN?&qAyzimA@wtf;#^%{$G7i4nXu=Pp2#r@O~wi)zB>@25A*|axl zEclXBlXx1LP3x0yrSx@s-kVW4qlF+idF+{M7RG54CgA&soDU-3SfHW@-6_ z+*;{n_SixmGCeZjHmEE!IF}!#aswth_{zm5Qhj0z-@I}pR?cu=P)HJUBClC;U+9;$#@xia30o$% zDw%BgOl>%vRenxL#|M$s^9X}diJ9q7wI1-0n2#6>@q}rK@ng(4M68(t52H_Jc{f&M9NPxRr->vj-88hoI?pvpn}llcv_r0`;uN>wuE{ z&TOx_i4==o;)>V4vCqG)A!mW>dI^Ql8BmhOy$6^>OaUAnI3>mN!Zr#qo4A>BegYj` zNG_)2Nvy2Cqxs1SF9A5HHhL7sai#Umw%K@+riaF+q)7&MUJvA&;$`(w)+B@c6!kX@ zzuY;LGu6|Q2eu^06PzSLspV2v4E?IPf`?Su_g8CX!75l)PCvyWKi4YRoRThB!-BhG zubQ#<7oCvj@z`^y&mPhSlbMf0<;0D z?5&!I?nV-jh-j1g~&R(YL@c=KB_gNup$8abPzXZN`N|WLqxlN)ZJ+#k4UWq#WqvVD z^|j+8f5uxTJtgcUscKTqKcr?5g-Ih3nmbvWvvEk})u-O}h$=-p4WE^qq7Z|rLas0$ zh0j&lhm@Rk(6ZF0_6^>Rd?Ni-#u1y`;$9tS;~!ph8T7fLlYE{P=XtWfV0Ql z#z{_;A%p|8+LhbZT0D_1!b}}MBx9`R9uM|+*`4l3^O(>Mk%@ha>VDY=nZMMb2TnJ= zGlQ+#+pmE98zuFxwAQcVkH1M887y;Bz&EJ7chIQQe!pgWX>(2ruI(emhz@_6t@k8Z zqFEyJFX2PO`$gJ6p$=ku{7!vR#u+$qo|1r;orjtp9FP^o2`2_vV;W&OT)acRXLN^m zY8a;geAxg!nbVu|uS8>@Gvf@JoL&GP`2v4s$Y^5vE32&l;2)`S%e#AnFI-YY7_>d#IKJI!oL6e z_7W3e=-0iz{bmuB*HP+D{Nb;rn+RyimTFqNV9Bzpa0?l`pWmR0yQOu&9c0S*1EPr1 zdoHMYlr>BycjTm%WeVuFd|QF8I{NPT&`fm=dITj&3(M^q ze2J{_2zB;wDME%}SzVWSW6)>1QtiX)Iiy^p2eT}Ii$E9w$5m)kv(3wSCNWq=#DaKZ zs%P`#^b7F-J0DgQ1?~2M`5ClYtYN{AlU|v4pEg4z03=g6nqH`JjQuM{k`!6jaIL_F zC;sn?1x?~uMo_DFg#ypNeie{3udcm~M&bYJ1LI zE%y}P9oCX3I1Y9yhF(y9Ix_=8L(p)EYr&|XZWCOb$7f2qX|A4aJ9bl7pt40Xr zXUT#NMBB8I@xoIGSHAZkYdCj>eEd#>a;W-?v4k%CwBaR5N>e3IFLRbDQTH#m_H+4b zk2UHVymC`%IqwtHUmpS1!1p-uQB`CW1Y!+VD!N4TT}D8(V0IOL|&R&)Rwj@n8g@=`h&z9YTPDT+R9agnwPuM!JW~=_ya~% zIJ*>$Fl;y7_`B7G4*P!kcy=MnNmR`(WS5_sRsvHF42NJ;EaDram5HwQ4Aw*qbYn0j;#)bh1lyKLg#dYjN*BMlh+fxmCL~?zB;HBWho;20WA==ci0mAqMfyG>1!HW zO7rOga-I9bvut1Ke_1eFo9tbzsoPTXDW1Si4}w3fq^Z|5LGf&egnw%DV=b11$F=P~ z(aV+j8S}m=CkI*8=RcrT>GmuYifP%hCoKY22Z4 zmu}o08h3YhcXx-v-QC??8mDn<+}+*X{+gZH-I;G^|7=1fBveS?J$27H&wV5^V^P$! z84?{UeYSmZ3M!@>UFoIN?GJT@IroYr;X@H~ax*CQ>b5|Xi9FXt5j`AwUPBq`0sWEJ z3O|k+g^JKMl}L(wfCqyMdRj9yS8ncE7nI14Tv#&(?}Q7oZpti{Q{Hw&5rN-&i|=fWH`XTQSu~1jx(hqm$Ibv zRzFW9$xf@oZAxL~wpj<0ZJ3rdPAE=0B>G+495QJ7D>=A&v^zXC9)2$$EnxQJ<^WlV zYKCHb1ZzzB!mBEW2WE|QG@&k?VXarY?umPPQ|kziS4{EqlIxqYHP!HN!ncw6BKQzKjqk!M&IiOJ9M^wc~ZQ1xoaI z;4je%ern~?qi&J?eD!vTl__*kd*nFF0n6mGEwI7%dI9rzCe~8vU1=nE&n4d&8}pdL zaz`QAY?6K@{s2x%Sx%#(y+t6qLw==>2(gb>AksEebXv=@ht>NBpqw=mkJR(c?l7vo z&cV)hxNoYPGqUh9KAKT)kc(NqekzE6(wjjotP(ac?`DJF=Sb7^Xet-A3PRl%n&zKk zruT9cS~vV1{%p>OVm1-miuKr<@rotj*5gd$?K`oteNibI&K?D63RoBjw)SommJ5<4 zus$!C8aCP{JHiFn2>XpX&l&jI7E7DcTjzuLYvON2{rz<)#$HNu(;ie-5$G<%eLKnTK7QXfn(UR(n+vX%aeS6!q6kv z!3nzY76-pdJp339zsl_%EI|;ic_m56({wdc(0C5LvLULW=&tWc5PW-4;&n+hm1m`f zzQV0T>OPSTjw=Ox&UF^y< zarsYKY8}YZF+~k70=olu$b$zdLaozBE|QE@H{_R21QlD5BilYBTOyv$D5DQZ8b1r- zIpSKX!SbA0Pb5#cT)L5!KpxX+x+8DRy&`o-nj+nmgV6-Gm%Fe91R1ca3`nt*hRS|^ z<&we;TJcUuPDqkM7k0S~cR%t7a`YP#80{BI$e=E!pY}am)2v3-Iqk2qvuAa1YM>xj#bh+H2V z{b#St2<;Gg>$orQ)c2a4AwD5iPcgZ7o_}7xhO86(JSJ(q(EWKTJDl|iBjGEMbX8|P z4PQHi+n(wZ_5QrX0?X_J)e_yGcTM#E#R^u_n8pK@l5416`c9S=q-e!%0RjoPyTliO zkp{OC@Ep^#Ig-n!C)K0Cy%8~**Vci8F1U(viN{==KU0nAg2(+K+GD_Gu#Bx!{tmUm zCwTrT(tCr6X8j43_n96H9%>>?4akSGMvgd+krS4wRexwZ1JxrJy!Uhz#yt$-=aq?A z@?*)bRZxjG9OF~7d$J0cwE_^CLceRK=LvjfH-~{S><^D;6B2&p-02?cl?|$@>`Qt$ zP*iaOxg<+(rbk>34VQDQpNQ|a9*)wScu!}<{oXC87hRPqyrNWpo?#=;1%^D2n2+C* zKKQH;?rWn-@%Y9g%NHG&lHwK9pBfV1a`!TqeU_Fv8s6_(@=RHua7`VYO|!W&WL*x= zIWE9eQaPq3zMaXuf)D0$V`RIZ74f)0P73xpeyk4)-?8j;|K%pD$eq4j2%tL=;&+E91O(2p91K|85b)GQcbRe&u6Ilu@SnE={^{Ix1Eqgv8D z4=w65+&36|;5WhBm$!n*!)ACCwT9Sip#1_z&g~E1kB=AlEhO0lu`Ls@6gw*a)lzc# zKx!fFP%eSBBs)U>xIcQKF(r_$SWD3TD@^^2Ylm=kC*tR+I@X>&SoPZdJ2fT!ysjH% z-U%|SznY8Fhsq7Vau%{Ad^Pvbf3IqVk{M2oD+w>MWimJA@VSZC$QooAO3 zC=DplXdkyl>mSp^$zk7&2+eoGQ6VVh_^E#Z3>tX7Dmi<2aqlM&YBmK&U}m>a%8)LQ z8v+c}a0QtXmyd%Kc2QNGf8TK?_EK4wtRUQ*VDnf5jHa?VvH2K(FDZOjAqYufW8oIZ z31|o~MR~T;ZS!Lz%8M0*iVARJ>_G2BXEF8(}6Dmn_rFV~5NI`lJjp`Mi~g7~P%H zO`S&-)Fngo3VXDMo7ImlaZxY^s!>2|csKca6!|m7)l^M0SQT1_L~K29%x4KV8*xiu zwP=GlyIE9YPSTC0BV`6|#)30=hJ~^aYeq7d6TNfoYUkk-^k0!(3qp(7Mo-$|48d8Z2d zrsfsRM)y$5)0G`fNq!V?qQ+nh0xwFbcp{nhW%vZ?h);=LxvM(pWd9FG$Bg1;@Bv)mKDW>AP{ol zD(R~mLzdDrBv$OSi{E%OD`Ano=F^vwc)rNb*Bg3-o)bbAgYE=M7Gj2OHY{8#pM${_^ zwkU|tnTKawxUF7vqM9UfcQ`V49zg78V%W)$#5ssR}Rj7E&p(4_ib^?9luZPJ%iJTvW&-U$nFYky>KJwHpEHHx zVEC;!ETdkCnO|${Vj#CY>LLut_+c|(hpWk8HRgMGRY%E--%oKh@{KnbQ~0GZd}{b@ z`J2qHBcqqjfHk^q=uQL!>6HSSF3LXL*cCd%opM|k#=xTShX~qcxpHTW*BI!c3`)hQq{@!7^mdUaG7sFsFYnl1%blslM;?B8Q zuifKqUAmR=>33g~#>EMNfdye#rz@IHgpM$~Z7c5@bO@S>MyFE3_F}HVNLnG0TjtXU zJeRWH^j5w_qXb$IGs+E>daTa}XPtrUnnpTRO9NEx4g6uaFEfHP9gW;xZnJi{oqAH~ z5dHS(ch3^hbvkv@u3QPLuWa}ImaElDrmIc%5HN<^bwej}3+?g) z-ai7D&6Iq_P(}k`i^4l?hRLbCb>X9iq2UYMl=`9U9Rf=3Y!gnJbr?eJqy>Zpp)m>Ae zcQ4Qfs&AaE?UDTODcEj#$_n4KeERZHx-I+E5I~E#L_T3WI3cj$5EYR75H7hy%80a8Ej?Y6hv+fR6wHN%_0$-xL!eI}fdjOK7(GdFD%`f%-qY@-i@fTAS&ETI99jUVg8 zslPSl#d4zbOcrgvopvB2c2A6r^pEr&Sa5I5%@1~BpGq`Wo|x=&)WnnQjE+)$^U-wW zr2Kv?XJby(8fcn z8JgPn)2_#-OhZ+;72R6PspMfCVvtLxFHeb7d}fo(GRjm_+R(*?9QRBr+yPF(iPO~ zA4Tp1<0}#fa{v0CU6jz}q9;!3Pew>ikG1qh$5WPRTQZ~ExQH}b1hDuzRS1}65uydS z~Te*3@?o8fih=mZ`iI!hL5iv3?VUBLQv0X zLtu58MIE7Jbm?)NFUZuMN2_~eh_Sqq*56yIo!+d_zr@^c@UwR&*j!fati$W<=rGGN zD$X`$lI%8Qe+KzBU*y3O+;f-Csr4$?3_l+uJ=K@dxOfZ?3APc5_x2R=a^kLFoxt*_ z4)nvvP+(zwlT5WYi!4l7+HKqzmXKYyM9kL5wX$dTSFSN&)*-&8Q{Q$K-})rWMin8S zy*5G*tRYNqk7&+v;@+>~EIQgf_SB;VxRTQFcm5VtqtKZ)x=?-f+%OY(VLrXb^6*aP zP&0Nu@~l2L!aF8i2!N~fJiHyxRl?I1QNjB)`uP_DuaU?2W;{?0#RGKTr2qH5QqdhK zP__ojm4WV^PUgmrV)`~f>(769t3|13DrzdDeXxqN6XA|_GK*;zHU()a(20>X{y-x| z2P6Ahq;o=)Nge`l+!+xEwY`7Q(8V=93A9C+WS^W%p&yR)eiSX+lp)?*7&WSYSh4i> zJa6i5T9o;Cd5z%%?FhB?J{l+t_)c&_f86gZMU{HpOA=-KoU5lIL#*&CZ_66O5$3?# ztgjGLo`Y7bj&eYnK#5x1trB_6tpu4$EomotZLb*9l6P(JmqG`{z$?lNKgq?GAVhkA zvw!oFhLyX=$K=jTAMwDQ)E-8ZW5$X%P2$YB5aq!VAnhwGv$VR&;Ix#fu%xlG{|j_K zbEYL&bx%*YpXcaGZj<{Y{k@rsrFKh7(|saspt?OxQ~oj_6En(&!rTZPa7fLCEU~mA zB7tbVs=-;cnzv*#INgF_9f3OZhp8c5yk!Dy1+`uA7@eJfvd~g34~wKI1PW%h(y&nA zRwMni12AHEw36)C4Tr-pt6s82EJa^8N#bjy??F*rg4fS@?6^MbiY3;7x=gd~G|Hi& zwmG+pAn!aV>>nNfP7-Zn8BLbJm&7}&ZX+$|z5*5{{F}BRSxN=JKZTa#{ut$v0Z0Fs za@UjXo#3!wACv+p9k*^9^n+(0(YKIUFo`@ib@bjz?Mh8*+V$`c%`Q>mrc5bs4aEf4 zh0qtL1qNE|xQ9JrM}qE>X>Y@dQ?%` zBx(*|1FMzVY&~|dE^}gHJ37O9bjnk$d8vKipgcf+As(kt2cbxAR3^4d0?`}}hYO*O z{+L&>G>AYaauAxE8=#F&u#1YGv%`d*v+EyDcU2TnqvRE33l1r}p#Vmcl%n>NrYOqV z2Car_^^NsZ&K=a~bj%SZlfxzHAxX$>=Q|Zi;E0oyfhgGgqe1Sd5-E$8KV9=`!3jWZCb2crb;rvQ##iw}xm7Da za!H${ls5Ihwxkh^D)M<4Yy3bp<-0a+&KfV@CVd9X6Q?v)$R3*rfT@jsedSEhoV(vqv?R1E8oWV;_{l_+_6= zLjV^-bZU$D_ocfSpRxDGk*J>n4G6s-e>D8JK6-gA>aM^Hv8@)txvKMi7Pi#DS5Y?r zK0%+L;QJdrIPXS2 ztjWAxkSwt2xG$L)Zb7F??cjs!KCTF+D{mZ5e0^8bdu_NLgFHTnO*wx!_8#}NO^mu{FaYeCXGjnUgt_+B-Ru!2_Ue-0UPg2Y)K3phLmR<4 zqUCWYX!KDU!jYF6c?k;;vF@Qh^q(PWwp1ez#I+0>d7V(u_h|L+kX+MN1f5WqMLn!L z!c(pozt7tRQi&duH8n=t-|d)c^;%K~6Kpyz(o53IQ_J+aCapAif$Ek#i0F9U>i+94 zFb=OH5(fk-o`L(o|DyQ(hlozl*2cu#)Y(D*zgNMi1Z!DTex#w#)x(8A-T=S+eByJW z%-k&|XhdZOWjJ&(FTrZNWRm^pHEot_MRQ_?>tKQ&MB~g(&D_e>-)u|`Ot(4j=UT6? zQ&YMi2UnCKlBpwltP!}8a2NJ`LlfL=k8SQf69U)~=G;bq9<2GU&Q#cHwL|o4?ah1` z;fG)%t0wMC;DR?^!jCoKib_iiIjsxCSxRUgJDCE%0P;4JZhJCy)vR1%zRl>K?V6#) z2lDi*W3q9rA zo;yvMujs+)a&00~W<-MNj=dJ@4%tccwT<@+c$#CPR%#aE#Dra+-5eSDl^E>is2v^~ z8lgRwkpeU$|1LW4yFwA{PQ^A{5JY!N5PCZ=hog~|FyPPK0-i;fCl4a%1 z?&@&E-)b4cK)wjXGq|?Kqv0s7y~xqvSj-NpOImt{Riam*Z!wz-coZIMuQU>M%6ben z>P@#o^W;fizVd#?`eeEPs#Gz^ySqJn+~`Pq%-Ee6*X+E>!PJGU#rs6qu0z5{+?`-N zxf1#+JNk7e6AoJTdQwxs&GMTq?Djch_8^xL^A;9XggtGL>!@0|BRuIdE&j$tzvt7I zr@I@0<0io%lpF697s1|qNS|BsA>!>-9DVlgGgw2;;k;=7)3+&t!);W3ulPgR>#JiV zUerO;WxuJqr$ghj-veVGfKF?O7si#mzX@GVt+F&atsB@NmBoV4dK|!owGP005$7LN7AqCG(S+={YA- zn#I{UoP_$~Epc=j78{(!2NLN)3qSm-1&{F&1z4Dz&7Mj_+SdlR^Q5{J=r822d4A@?Rj~xATaWewHUOus{*C|KoH`G zHB8SUT06GpSt)}cFJ18!$Kp@r+V3tE_L^^J%9$&fcyd_AHB)WBghwqBEWW!oh@StV zDrC?ttu4#?Aun!PhC4_KF1s2#kvIh~zds!y9#PIrnk9BWkJpq}{Hlqi+xPOR&A1oP zB0~1tV$Zt1pQuHpJw1TAOS=3$Jl&n{n!a+&SgYVe%igUtvE>eHqKY0`e5lwAf}2x( zP>9Wz+9uirp7<7kK0m2&Y*mzArUx%$CkV661=AIAS=V=|xY{;$B7cS5q0)=oq0uXU z_roo90&gHSfM6@6kmB_FJZ)3y_tt0}7#PA&pWo@_qzdIMRa-;U*Dy>Oo#S_n61Fn! z%mrH%tRmvQvg%UqN_2(C#LSxgQ>m}FKLGG=uqJQuSkk=S@c~QLi4N+>lr}QcOuP&% zQCP^cRk&rk-@lpa0^Lcvdu`F*qE)-0$TnxJlwZf|dP~s8cjhL%>^+L~{umxl5Xr6@ z^7zVKiN1Xg;-h+kr4Yt2BzjZs-Mo54`pDbLc}fWq{34=6>U9@sBP~iWZE`+FhtU|x zTV}ajn*Hc}Y?3agQ+bV@oIRm=qAu%|zE;hBw7kCcDx{pm!_qCxfPX3sh5^B$k_2d` z6#rAeUZC;e-LuMZ-f?gHeZogOa*mE>ffs+waQ+fQl4YKoAyZii_!O0;h55EMzD{;) z8lSJvv((#UqgJ?SCQFqJ-UU?2(0V{;7zT3TW`u6GH6h4m3}SuAAj_K(raGBu>|S&Q zZGL?r9@caTbmRm7p=&Tv?Y1)60*9At38w)$(1c?4cpFY2RLyw9c<{OwQE{b@WI}FQ zTT<2HOF4222d%k70yL~x_d#6SNz`*%@4++8gYQ8?yq0T@w~bF@aOHL2)T4xj`AVps9k z?m;<2ClJh$B6~fOYTWIV*T9y1BpB1*C?dgE{%lVtIjw>4MK{wP6OKTb znbPWrkZjYCbr`GGa%Xo0h;iFPNJBI3fK5`wtJV?wq_G<_PZ<`eiKtvN$IKfyju*^t zXc}HNg>^PPZ16m6bfTpmaW5=qoSsj>3)HS}teRa~qj+Y}mGRE?cH!qMDBJ8 zJB!&-=MG8Tb;V4cZjI_#{>ca0VhG_P=j0kcXVX5)^Sdpk+LKNv#yhpwC$k@v^Am&! z_cz2^4Cc{_BC!K#zN!KEkPzviUFPJ^N_L-kHG6}(X#$>Q=9?!{$A(=B3)P?PkxG9gs#l! zo6TOHo$F|IvjTC3MW%XrDoc7;m-6wb9mL(^2(>PQXY53hE?%4FW$rTHtN`!VgH72U zRY)#?Y*pMA<)x3B-&fgWQ(TQ6S6nUeSY{9)XOo_k=j$<*mA=f+ghSALYwBw~!Egn!jtjubOh?6Cb-Zi3IYn*fYl()^3u zRiX0I{5QaNPJ9w{yh4(o#$geO7b5lSh<5ZaRg9_=aFdZjxjXv(_SCv^v-{ZKQFtAA}kw=GPC7l81GY zeP@0Da{aR#{6`lbI0ON0y#K=t|L*}MG_HSl$e{U;v=BSs{SU3(e*qa(l%rD;(zM^3 zrRgN3M#Sf(Cr9>v{FtB`8JBK?_zO+~{H_0$lLA!l{YOs9KQd4Zt<3*Ns7dVbT{1Ut z?N9{XkN(96?r(4BH~3qeiJ_CAt+h1}O_4IUF$S(5EyTyo=`{^16P z=VhDY!NxkDukQz>T`0*H=(D3G7Np*2P`s(6M*(*ZJa;?@JYj&_z`d5bap=KK37p3I zr5#`%aC)7fUo#;*X5k7g&gQjxlC9CF{0dz*m2&+mf$Sc1LnyXn9lpZ!!Bl!@hnsE5px};b-b-`qne0Kh;hziNC zXV|zH%+PE!2@-IrIq!HM2+ld;VyNUZiDc@Tjt|-1&kq}>muY;TA3#Oy zWdYGP3NOZWSWtx6?S6ES@>)_Yz%%nLG3P>Z7`SrhkZ?shTfrHkYI;2zAn8h65wV3r z^{4izW-c9!MTge3eN=~r5aTnz6*6l#sD68kJ7Nv2wMbL~Ojj0H;M`mAvk*`Q!`KI? z7nCYBqbu$@MSNd+O&_oWdX()8Eh|Z&v&dJPg*o-sOBb2hriny)< zd(o&&kZM^NDtV=hufp8L zCkKu7)k`+czHaAU567$?GPRGdkb4$37zlIuS&<&1pgArURzoWCbyTEl9OiXZBn4p<$48-Gekh7>e)v*?{9xBt z=|Rx!@Y3N@ffW5*5!bio$jhJ7&{!B&SkAaN`w+&3x|D^o@s{ZAuqNss8K;211tUWIi1B!%-ViYX+Ys6w)Q z^o1{V=hK#+tt&aC(g+^bt-J9zNRdv>ZYm9KV^L0y-yoY7QVZJ_ivBS02I|mGD2;9c zR%+KD&jdXjPiUv#t1VmFOM&=OUE2`SNm4jm&a<;ZH`cYqBZoAglCyixC?+I+}*ScG#;?SEAFob{v0ZKw{`zw*tX}<2k zoH(fNh!>b5w8SWSV}rQ*E24cO=_eQHWy8J!5;Y>Bh|p;|nWH|nK9+ol$k`A*u*Y^Uz^%|h4Owu}Cb$zhIxlVJ8XJ0xtrErT zcK;34CB;ohd|^NfmVIF=XlmB5raI}nXjFz;ObQ4Mpl_`$dUe7sj!P3_WIC~I`_Xy@ z>P5*QE{RSPpuV=3z4p3}dh>Dp0=We@fdaF{sJ|+_E*#jyaTrj-6Y!GfD@#y@DUa;& zu4Iqw5(5AamgF!2SI&WT$rvChhIB$RFFF|W6A>(L9XT{0%DM{L`knIQPC$4F`8FWb zGlem_>>JK-Fib;g*xd<-9^&_ue95grYH>5OvTiM;#uT^LVmNXM-n8chJBD2KeDV7t zbnv3CaiyN>w(HfGv86K5MEM{?f#BTR7**smpNZ}ftm+gafRSt=6fN$(&?#6m3hF!>e$X)hFyCF++Qvx(<~q3esTI zH#8Sv!WIl2<&~=B)#sz1x2=+KTHj=0v&}iAi8eD=M->H|a@Qm|CSSzH#eVIR3_Tvu zG8S**NFbz%*X?DbDuP(oNv2;Lo@#_y4k$W+r^#TtJ8NyL&&Rk;@Q}~24`BB)bgwcp z=a^r(K_NEukZ*|*7c2JKrm&h&NP)9<($f)eTN}3|Rt`$5uB0|!$Xr4Vn#i;muSljn zxG?zbRD(M6+8MzGhbOn%C`M#OcRK!&ZHihwl{F+OAnR>cyg~No44>vliu$8^T!>>*vYQJCJg=EF^lJ*3M^=nGCw`Yg@hCmP(Gq^=eCEE1!t-2>%Al{w@*c% zUK{maww*>K$tu;~I@ERb9*uU@LsIJ|&@qcb!&b zsWIvDo4#9Qbvc#IS%sV1_4>^`newSxEcE08c9?rHY2%TRJfK2}-I=Fq-C)jc`gzV( zCn?^noD(9pAf2MP$>ur0;da`>Hr>o>N@8M;X@&mkf;%2A*2CmQBXirsJLY zlX21ma}mKH_LgYUM-->;tt;6F?E5=fUWDwQhp*drQ%hH0<5t2m)rFP%=6aPIC0j$R znGI0hcV~}vk?^&G`v~YCKc7#DrdMM3TcPBmxx#XUC_JVEt@k=%3-+7<3*fTcQ>f~?TdLjv96nb66xj=wVQfpuCD(?kzs~dUV<}P+Fpd)BOTO^<*E#H zeE80(b~h<*Qgez(iFFOkl!G!6#9NZAnsxghe$L=Twi^(Q&48 zD0ohTj)kGLD){xu%pm|}f#ZaFPYpHtg!HB30>F1c=cP)RqzK2co`01O5qwAP zUJm0jS0#mci>|Nu4#MF@u-%-4t>oUTnn_#3K09Hrwnw13HO@9L;wFJ*Z@=gCgpA@p zMswqk;)PTXWuMC-^MQxyNu8_G-i3W9!MLd2>;cM+;Hf&w| zLv{p*hArp9+h2wsMqT5WVqkkc0>1uokMox{AgAvDG^YJebD-czexMB!lJKWllLoBI zetW2;;FKI1xNtA(ZWys!_un~+834+6y|uV&Lo%dKwhcoDzRADYM*peh{o`-tHvwWIBIXW`PKwS3|M>CW37Z2dr!uJWNFS5UwY4;I zNIy1^sr+@8Fob%DHRNa&G{lm?KWU7sV2x9(Ft5?QKsLXi!v6@n&Iyaz5&U*|hCz+d z9vu60IG<v6+^ZmBs_aN!}p|{f(ikVl&LcB+UY;PPz* zj84Tm>g5~-X=GF_4JrVmtEtm=3mMEL1#z+pc~t^Iify^ft~cE=R0TymXu*iQL+XLX zdSK$~5pglr3f@Lrcp`>==b5Z6r7c=p=@A5nXNacsPfr(5m;~ks@*Wu7A z%WyY$Pt*RAKHz_7cghHuQqdU>hq$vD?plol_1EU(Fkgyo&Q2&2e?FT3;H%!|bhU~D z>VX4-6}JLQz8g3%Bq}n^NhfJur~v5H0dbB^$~+7lY{f3ES}E?|JnoLsAG%l^%eu_PM zEl0W(sbMRB3rFeYG&tR~(i2J0)RjngE`N_Jvxx!UAA1mc7J>9)`c=`}4bVbm8&{A` z3sMPU-!r-8de=P(C@7-{GgB<5I%)x{WfzJwEvG#hn3ict8@mexdoTz*(XX!C&~}L* z^%3eYQ8{Smsmq(GIM4d5ilDUk{t@2@*-aevxhy7yk(wH?8yFz%gOAXRbCYzm)=AsM z?~+vo2;{-jkA%Pqwq&co;|m{=y}y2lN$QPK>G_+jP`&?U&Ubq~T`BzAj1TlC`%8+$ zzdwNf<3suPnbh&`AI7RAYuQ<#!sD|A=ky2?hca{uHsB|0VqShI1G3lG5g}9~WSvy4 zX3p~Us^f5AfXlBZ0hA;mR6aj~Q8yb^QDaS*LFQwg!!<|W!%WX9Yu}HThc7>oC9##H zEW`}UQ%JQ38UdsxEUBrA@=6R-v1P6IoIw8$8fw6F{OSC7`cOr*u?p_0*Jvj|S)1cd z-9T);F8F-Y_*+h-Yt9cQQq{E|y^b@r&6=Cd9j0EZL}Pj*RdyxgJentY49AyC@PM<< zl&*aq_ubX%*pqUkQ^Zsi@DqhIeR&Ad)slJ2g zmeo&+(g!tg$z1ao1a#Qq1J022mH4}y?AvWboI4H028;trScqDQrB36t!gs|uZS9}KG0}DD$ zf2xF}M*@VJSzEJ5>ucf+L_AtN-Ht=34g&C?oPP>W^bwoigIncKUyf61!ce!2zpcNT zj&;rPGI~q2!Sy>Q7_lRX*DoIs-1Cei=Cd=+Xv4=%bn#Yqo@C=V`|QwlF0Y- zONtrwpHQ##4}VCL-1ol(e<~KU9-ja^kryz!g!})y-2S5z2^gE$Isj8l{%tF=Rzy`r z^RcP7vu`jHgHLKUE957n3j+BeE(bf;f)Zw($XaU6rZ26Upl#Yv28=8Y`hew{MbH>* z-sGI6dnb5D&dUCUBS`NLAIBP!Vi!2+~=AU+)^X^IpOEAn#+ab=`7c z%7B|mZ>wU+L;^&abXKan&N)O;=XI#dTV|9OMYxYqLbtT#GY8PP$45Rm2~of+J>>HIKIVn(uQf-rp09_MwOVIp@6!8bKV(C#(KxcW z;Pesq(wSafCc>iJNV8sg&`!g&G55<06{_1pIoL`2<7hPvAzR1+>H6Rx0Ra%4j7H-<-fnivydlm{TBr06;J-Bq8GdE^Amo)ptV>kS!Kyp*`wUx=K@{3cGZnz53`+C zLco1jxLkLNgbEdU)pRKB#Pq(#(Jt>)Yh8M?j^w&RPUueC)X(6`@@2R~PV@G(8xPwO z^B8^+`qZnQr$8AJ7<06J**+T8xIs)XCV6E_3W+al18!ycMqCfV>=rW0KBRjC* zuJkvrv;t&xBpl?OB3+Li(vQsS(-TPZ)Pw2>s8(3eF3=n*i0uqv@RM^T#Ql7(Em{(~%f2Fw|Reg@eSCey~P zBQlW)_DioA*yxxDcER@_=C1MC{UswPMLr5BQ~T6AcRyt0W44ffJG#T~Fk}wU^aYoF zYTayu-s?)<`2H(w+1(6X&I4?m3&8sok^jpXBB<|ZENso#?v@R1^DdVvKoD?}3%@{}}_E7;wt9USgrfR3(wabPRhJ{#1es81yP!o4)n~CGsh2_Yj2F^z|t zk((i&%nDLA%4KFdG96pQR26W>R2^?C1X4+a*hIzL$L=n4M7r$NOTQEo+k|2~SUI{XL{ynLSCPe%gWMMPFLO{&VN2pom zBUCQ(30qj=YtD_6H0-ZrJ46~YY*A;?tmaGvHvS^H&FXUG4)%-a1K~ly6LYaIn+4lG zt=wuGLw!%h=Pyz?TP=?6O-K-sT4W%_|Nl~;k~YA^_`gqfe{Xw=PWn#9f1mNz)sFuL zJbrevo(DPgpirvGMb6ByuEPd=Rgn}fYXqeUKyM+!n(cKeo|IY%p!#va6`D8?A*{u3 zEeWw0*oylJ1X!L#OCKktX2|>-z3#>`9xr~azOH+2dXHRwdfnpri9|xmK^Q~AuY!Fg z`9Xx?hxkJge~)NVkPQ(VaW(Ce2pXEtgY*cL8i4E)mM(iz_vdm|f@%cSb*Lw{WbShh41VGuplex9E^VvW}irx|;_{VK=N_WF39^ zH4<*peWzgc)0UQi4fBk2{FEzldDh5+KlRd!$_*@eYRMMRb1gU~9lSO_>Vh-~q|NTD zL}X*~hgMj$*Gp5AEs~>Bbjjq7G>}>ki1VxA>@kIhLe+(EQS0mjNEP&eXs5)I;7m1a zmK0Ly*!d~Dk4uxRIO%iZ!1-ztZxOG#W!Q_$M7_DKND0OwI+uC;PQCbQ#k#Y=^zQve zTZVepdX>5{JSJb;DX3%3g42Wz2D@%rhIhLBaFmx#ZV8mhya}jo1u{t^tzoiQy=jJp zjY2b7D2f$ZzJx)8fknqdD6fd5-iF8e(V}(@xe)N=fvS%{X$BRvW!N3TS8jn=P%;5j zShSbzsLs3uqycFi3=iSvqH~}bQn1WQGOL4?trj(kl?+q2R23I42!ipQ&`I*&?G#i9 zWvNh8xoGKDt>%@i0+}j?Ykw&_2C4!aYEW0^7)h2Hi7$;qgF3;Go?bs=v)kHmvd|`R z%(n94LdfxxZ)zh$ET8dH1F&J#O5&IcPH3=8o;%>OIT6w$P1Yz4S!}kJHNhMQ1(prc zM-jSA-7Iq=PiqxKSWb+YbLB-)lSkD6=!`4VL~`ExISOh2ud=TI&SKfR4J08Bad&rj zcXxMpcNgOB?w$~L7l^wPcXxw$0=$oV?)`I44)}b#ChS`_lBQhvb6ks?HDr3tFgkg&td19?b8=!sETXtp=&+3T$cCwZe z0nAET-7561gsbBws$TVjP7QxY(NuBYXVn9~9%vyN-B#&tJhWgtL1B<%BTS*-2$xB` zO)cMDHoWsm%JACZF--Pa7oP;f!n%p`*trlpvZ!HKoB={l+-(8O;;eYv2A=ra z3U7rSMCkP_6wAy`l|Se(&5|AefXvV1E#XA(LT!% zjj4|~xlZ-kPLNeQLFyXb%$K}YEfCBvHA-Znw#dZSI6V%3YD{Wj2@utT5Hieyofp6Qi+lz!u)htnI1GWzvQsA)baEuw9|+&(E@p8M+#&fsX@Kf`_YQ>VM+40YLv`3-(!Z7HKYg@+l00WGr779i-%t`kid%e zDtbh8UfBVT3|=8FrNian@aR3*DTUy&u&05x%(Lm3yNoBZXMHWS7OjdqHp>cD>g!wK z#~R{1`%v$IP;rBoP0B0P><;dxN9Xr+fp*s_EK3{EZ94{AV0#Mtv?;$1YaAdEiq5)g zYME;XN9cZs$;*2p63Q9^x&>PaA1p^5m7|W?hrXp2^m;B@xg0bD?J;wIbm6O~Nq^^K z2AYQs@7k)L#tgUkTOUHsh&*6b*EjYmwngU}qesKYPWxU-z_D> zDWr|K)XLf_3#k_9Rd;(@=P^S^?Wqlwert#9(A$*Y$s-Hy)BA0U0+Y58zs~h=YtDKxY0~BO^0&9{?6Nny;3=l59(6ec9j(79M?P1cE zex!T%$Ta-KhjFZLHjmPl_D=NhJULC}i$}9Qt?nm6K6-i8&X_P+i(c*LI3mtl3 z*B+F+7pnAZ5}UU_eImDj(et;Khf-z^4uHwrA7dwAm-e4 zwP1$Ov3NP5ts+e(SvM)u!3aZMuFQq@KE-W;K6 zag=H~vzsua&4Sb$4ja>&cSJ)jjVebuj+?ivYqrwp3!5>ul`B*4hJGrF;!`FaE+wKo z#};5)euvxC1zX0-G;AV@R(ZMl=q_~u8mQ5OYl;@BAkt)~#PynFX#c1K zUQ1^_N8g+IZwUl*n0Bb-vvliVtM=zuMGU-4a8|_8f|2GEd(2zSV?aSHUN9X^GDA8M zgTZW06m*iAy@7l>F3!7+_Y3mj^vjBsAux3$%U#d$BT^fTf-7{Y z_W0l=7$ro5IDt7jp;^cWh^Zl3Ga1qFNrprdu#g=n9=KH!CjLF#ucU5gy6*uASO~|b z7gcqm90K@rqe({P>;ww_q%4}@bq`ST8!0{V08YXY)5&V!>Td)?j7#K}HVaN4FU4DZ z%|7OppQq-h`HJ;rw-BAfH* z1H$ufM~W{%+b@9NK?RAp-$(P0N=b<(;wFbBN0{u5vc+>aoZ|3&^a866X@el7E8!E7 z=9V(Ma**m_{DKZit2k;ZOINI~E$|wO99by=HO{GNc1t?nl8soP@gxk8)WfxhIoxTP zoO`RA0VCaq)&iRDN9yh_@|zqF+f07Esbhe!e-j$^PS57%mq2p=+C%0KiwV#t^%_hH zoO?{^_yk5x~S)haR6akK6d|#2TN& zfWcN zc7QAWl)E9`!KlY>7^DNw$=yYmmRto>w0L(~fe?|n6k2TBsyG@sI)goigj=mn)E)I* z4_AGyEL7?(_+2z=1N@D}9$7FYdTu;%MFGP_mEJXc2OuXEcY1-$fpt8m_r2B|<~Xfs zX@3RQi`E-1}^9N{$(|YS@#{ZWuCxo)91{k>ESD54g_LYhm~vlOK_CAJHeYFfuIVB^%cqCfvpy#sU8Do8u}# z>>%PLKOZ^+$H54o@brtL-hHorSKcsjk_ZibBKBgyHt~L z=T6?e0oLX|h!Z3lbkPMO27MM?xn|uZAJwvmX?Yvp#lE3sQFY)xqet>`S2Y@1t)Z*& z;*I3;Ha8DFhk=YBt~{zp=%%*fEC}_8?9=(-k7HfFeN^GrhNw4e?vx*#oMztnO*&zY zmRT9dGI@O)t^=Wj&Og1R3b%(m*kb&yc;i`^-tqY9(0t!eyOkH<$@~1lXmm!SJllE_ zr~{a&w|8*LI>Z^h!m%YLgKv06Js7j7RaoX}ZJGYirR<#4Mghd{#;38j3|V+&=ZUq#1$ zgZb-7kV)WJUko?{R`hpSrC;w2{qa`(Z4gM5*ZL`|#8szO=PV^vpSI-^K_*OQji^J2 zZ_1142N}zG$1E0fI%uqHOhV+7%Tp{9$bAR=kRRs4{0a`r%o%$;vu!_Xgv;go)3!B#;hC5qD-bcUrKR&Sc%Zb1Y($r78T z=eG`X#IpBzmXm(o6NVmZdCQf6wzqawqI63v@e%3TKuF!cQ#NQbZ^?6K-3`_b=?ztW zA>^?F#dvVH=H-r3;;5%6hTN_KVZ=ps4^YtRk>P1i>uLZ)Ii2G7V5vy;OJ0}0!g>j^ z&TY&E2!|BDIf1}U(+4G5L~X6sQ_e7In0qJmWYpn!5j|2V{1zhjZt9cdKm!we6|Pp$ z07E+C8=tOwF<<}11VgVMzV8tCg+cD_z?u+$sBjwPXl^(Ge7y8-=c=fgNg@FxI1i5Y-HYQMEH z_($je;nw`Otdhd1G{Vn*w*u@j8&T=xnL;X?H6;{=WaFY+NJfB2(xN`G)LW?4u39;x z6?eSh3Wc@LR&yA2tJj;0{+h6rxF zKyHo}N}@004HA(adG~0solJ(7>?LoXKoH0~bm+xItnZ;3)VJt!?ue|~2C=ylHbPP7 zv2{DH()FXXS_ho-sbto)gk|2V#;BThoE}b1EkNYGT8U#0ItdHG>vOZx8JYN*5jUh5Fdr9#12^ zsEyffqFEQD(u&76zA^9Jklbiz#S|o1EET$ujLJAVDYF znX&4%;vPm-rT<8fDutDIPC@L=zskw49`G%}q#l$1G3atT(w70lgCyfYkg7-=+r7$%E`G?1NjiH)MvnKMWo-ivPSQHbk&_l5tedNp|3NbU^wk0SSXF9ohtM zUqXiOg*8ERKx{wO%BimK)=g^?w=pxB1Vu_x<9jKOcU7N;(!o3~UxyO+*ZCw|jy2}V*Z22~KhmvxoTszc+#EMWXTM6QF*ks% zW47#2B~?wS)6>_ciKe1Fu!@Tc6oN7e+6nriSU;qT7}f@DJiDF@P2jXUv|o|Wh1QPf zLG31d>@CpThA+Ex#y)ny8wkC4x-ELYCXGm1rFI=1C4`I5qboYgDf322B_Nk@#eMZ% znluCKW2GZ{r9HR@VY`>sNgy~s+D_GkqFyz6jgXKD)U|*eKBkJRRIz{gm3tUd*yXmR z(O4&#ZA*us6!^O*TzpKAZ#}B5@}?f=vdnqnRmG}xyt=)2o%<9jj>-4wLP1X-bI{(n zD9#|rN#J;G%LJ&$+Gl2eTRPx6BQC6Uc~YK?nMmktvy^E8#Y*6ZJVZ>Y(cgsVnd!tV z!%twMNznd)?}YCWyy1-#P|2Fu%~}hcTGoy>_uawRTVl=(xo5!%F#A38L109wyh@wm zdy+S8E_&$Gjm=7va-b7@Hv=*sNo0{i8B7=n4ex-mfg`$!n#)v@xxyQCr3m&O1Jxg! z+FXX^jtlw=utuQ+>Yj$`9!E<5-c!|FX(~q`mvt6i*K!L(MHaqZBTtuSA9V~V9Q$G? zC8wAV|#XY=;TQD#H;;dcHVb9I7Vu2nI0hHo)!_{qIa@|2}9d ztpC*Q{4Py~2;~6URN^4FBCBip`QDf|O_Y%iZyA0R`^MQf$ce0JuaV(_=YA`knEMXw zP6TbjYSGXi#B4eX=QiWqb3bEw-N*a;Yg?dsVPpeYFS*&AsqtW1j2D$h$*ZOdEb$8n0 zGET4Igs^cMTXWG{2#A7w_usx=KMmNfi4oAk8!MA8Y=Rh9^*r>jEV(-{I0=rc);`Y) zm+6KHz-;MIy|@2todN&F+Yv1e&b&ZvycbTHpDoZ>FIiUn+M-=%A2C(I*^Yx@VKf(Z zxJOny&WoWcyKodkeN^5))aV|-UBFw{?AGo?;NNFFcKzk+6|gYfA#FR=y@?;3IoQ zUMI=7lwo9gV9fRvYi}Nd)&gQw7(K3=a0#p27u6Q)7JlP#A)piUUF8B3Li&38Xk$@| z9OR+tU~qgd3T3322E))eV)hAAHYIj$TmhH#R+C-&E-}5Qd{3B}gD{MXnsrS;{Erv1 z6IyQ=S2qD>Weqqj#Pd65rDSdK54%boN+a?=CkR|agnIP6;INm0A*4gF;G4PlA^3%b zN{H%#wYu|!3fl*UL1~f+Iu|;cqDax?DBkZWSUQodSDL4Es@u6zA>sIm>^Aq-&X#X8 zI=#-ucD|iAodfOIY4AaBL$cFO@s(xJ#&_@ZbtU+jjSAW^g;_w`FK%aH_hAY=!MTjI zwh_OEJ_25zTQv$#9&u0A11x_cGd92E74AbOrD`~f6Ir9ENNQAV2_J2Ig~mHWhaO5a zc>fYG$zke^S+fBupw+klDkiljJAha z6DnTemhkf>hv`8J*W_#wBj-2w(cVtXbkWWtE(3j@!A-IfF?`r$MhVknTs3D1N`rYN zKth9jZtX#>v#%U@^DVN!;ni#n1)U&H_uB{6pcq7$TqXJX!Q0P7U*JUZyclb~)l*DS zOLpoQfW_3;a0S$#V0SOwVeeqE$Hd^L`$;l_~2giLYd?7!gUYIpOs!jqSL~pI)4`YuB_692~A z^T#YYQ_W3Rakk}$SL&{`H8mc{>j+3eKprw6BK`$vSSIn;s31M~YlJLApJ)+Gi1{^- zw96WnT9M0Vr_D=e=a}${raR{(35Q!g+8`}vOFj1e&Or(_wp2U2aVQP0_jP57 z2(R4E(E$n!xl<}Zx38wO;27wuQ`P#_j!}L2 z2qr;As4D4n2X$-Jd_-!fsbu_D(64i;c4cJnP576x_>Q4WNushFwkBV!kVd(AYFXe{ zaqO5`Qfr!#ETmE(B;u_&FITotv~W}QYFCI!&ENKIb1p4fg*Yv1)EDMb==EjHHWM#{ zGMpqb2-LXdHB@D~pE3|+B392Gh4q)y9jBd$a^&cJM60VEUnLtHQD5i-X6PVF>9m_k zDvG3P(?CzdaIrC8s4cu~N9MEb!Tt(g*GK~gIp1Gyeaw3b7#YPx_1T6i zRi#pAMr~PJKe9P~I+ARa$a!K~)t(4LaVbjva1yd;b1Yz2$7MMc`aLmMl(a^DgN(u? zq2o9&Gif@Tq~Yq+qDfx^F*nCnpuPv%hRFc$I!p74*quLt^M}D_rwl10uMTr!)(*=7 zSC5ea@#;l(h87k4T4x)(o^#l76P-GYJA(pOa&F9YT=fS<*O{4agzba^dIrh0hjls<~APlIz9{ zgRY{OMv2s|`;VCoYVj?InYoq^QWuA&*VDyOn@pPvK8l~g#1~~MGVVvtLDt}>id_Z` zn(ihfL?Y}Y4YX335m*Xx(y+bbukchHrM zycIGp#1*K3$!(tgTsMD2VyUSg^yvCwB8*V~sACE(yq2!MS6f+gsxv^GR|Q7R_euYx z&X+@@H?_oQddGxJYS&ZG-9O(X+l{wcw;W7srpYjZZvanY(>Q1utSiyuuonkjh5J0q zGz6`&meSuxixIPt{UoHVupUbFKIA+3V5(?ijn}(C(v>=v?L*lJF8|yRjl-m#^|krg zLVbFV6+VkoEGNz6he;EkP!Z6|a@n8?yCzX9>FEzLnp21JpU0x!Qee}lwVKA})LZJq zlI|C??|;gZ8#fC3`gzDU%7R87KZyd)H__0c^T^$zo@TBKTP*i{)Gp3E0TZ}s3mKSY zix@atp^j#QnSc5K&LsU38#{lUdwj%xF zcx&l^?95uq9on1m*0gp$ruu||5MQo)XaN>|ngV5Jb#^wWH^5AdYcn_1>H~XtNwJd3 zd9&?orMSSuj=lhO?6)Ay7;gdU#E}pTBa5wFu`nejq##Xd71BHzH2XqLA5 zeLEo;9$}~u0pEu@(?hXB_l;{jQ=7m?~mwj-ME~Tw-OHPrR7K2Xq9eCNwQO$hR z3_A?=`FJctNXA#yQEorVoh{RWxJbdQga zU%K##XEPgy?E|K(=o#IPgnbk7E&5%J=VHube|2%!Qp}@LznjE%VQhJ?L(XJOmFVY~ zo-az+^5!Ck7Lo<7b~XC6JFk>17*_dY;=z!<0eSdFD2L?CSp_XB+?;N+(5;@=_Ss3& zXse>@sA7hpq;IAeIp3hTe9^$DVYf&?)={zc9*hZAV)|UgKoD!1w{UVo8D)Htwi8*P z%#NAn+8sd@b{h=O)dy9EGKbpyDtl@NBZw0}+Wd=@65JyQ2QgU}q2ii;ot1OsAj zUI&+Pz+NvuRv#8ugesT<<@l4L$zso0AQMh{we$tkeG*mpLmOTiy8|dNYhsqhp+q*yfZA`Z)UC*(oxTNPfOFk3RXkbzAEPofVUy zZ3A%mO?WyTRh@WdXz+zD!ogo}gbUMV!YtTNhr zrt@3PcP%5F;_SQ>Ui`Gq-lUe&taU4*h2)6RDh@8G1$o!){k~3)DT87%tQeHYdO?B` zAmoJvG6wWS?=0(Cj?Aqj59`p(SIEvYyPGJ^reI z`Hr?3#U2zI7k0=UmqMD35l`>3xMcWlDv$oo6;b`dZq3d!~)W z=4Qk)lE8&>#HV>?kRLOHZYz83{u7?^KoXmM^pazj8`7OwQ=5I!==; zA!uN`Q#n=Drmzg}@^nG!mJp9ml3ukWk96^6*us*;&>s+7hWfLXtl?a}(|-#=P12>A zon1}yqh^?9!;on?tRd6Fk0knQSLl4vBGb87A_kJNDGyrnpmn48lz_%P{* z_G*3D#IR<2SS54L5^h*%=)4D9NPpji7DZ5&lHD|99W86QN_(|aJ<5C~PX%YB`Qt_W z>jF_Os@kI6R!ub4n-!orS(G6~mKL7()1g=Lf~{D!LR7#wRHfLxTjYr{*c{neyhz#U zbm@WBKozE+kTd+h-mgF+ELWqTKin57P;0b){ zii5=(B%S(N!Z=rAFGnM6iePtvpxB_Q9-oq_xH!URn2_d-H~i;lro8r{-g!k-Ydb6_w5K@FOV?zPF_hi z%rlxBv$lQi%bjsu^7KT~@u#*c$2-;AkuP)hVEN?W5MO8C9snj*EC&|M!aK6o12q3+ z8e?+dH17E!A$tRlbJW~GtMDkMPT=m1g-v67q{sznnWOI$`g(8E!Pf!#KpO?FETxLK z2b^8^@mE#AR1z(DT~R3!nnvq}LG2zDGoE1URR=A2SA z%lN$#V@#E&ip_KZL}Q6mvm(dsS?oHoRf8TWL~1)4^5<3JvvVbEsQqSa3(lF*_mA$g zv`LWarC79G)zR0J+#=6kB`SgjQZ2460W zN%lZt%M@=EN>Wz4I;eH>C0VnDyFe)DBS_2{h6=0ZJ*w%s)QFxLq+%L%e~UQ0mM9ud zm&|r){_<*Om%vlT(K9>dE(3AHjSYro5Y1I?ZjMqWyHzuCE0nyCn`6eq%MEt(aY=M2rIzHeMds)4^Aub^iTIT|%*izG4YH;sT`D9MR(eND-SB+e66LZT z2VX)RJsn${O{D48aUBl|(>ocol$1@glsxisc#GE*=DXHXA?|hJT#{;X{i$XibrA}X zFHJa+ssa2$F_UC(o2k2Z0vwx%Wb(<6_bdDO#=a$0gK2NoscCr;vyx?#cF)JjM%;a| z$^GIlIzvz%Hx3WVU481}_e4~aWcyC|j&BZ@uWW1`bH1y9EWXOxd~f-VE5DpueNofN zv7vZeV<*!A^|36hUE;`#x%MHhL(~?eZ5fhA9Ql3KHTWoAeO-^7&|2)$IcD1r5X#-u zN~N0$6pHPhop@t1_d`dO3#TC0>y5jm>8;$F5_A2& zt#=^IDfYv?JjPPTPNx2TL-Lrl82VClQSLWW_$3=XPbH}xM34)cyW5@lnxy=&h%eRq zv29&h^fMoxjsDnmua(>~OnX{Cq!7vM0M4Mr@_18|YuSKPBKUTV$s^So zc}JlAW&bVz|JY#Eyup6Ny{|P_s0Pq;5*tinH+>5Xa--{ z2;?2PBs((S4{g=G`S?B3Ien`o#5DmUVwzpGuABthYG~OKIY`2ms;33SN9u^I8i_H5`BQ%yOfW+N3r|ufHS_;U;TWT5z;b14n1gX%Pn`uuO z6#>Vl)L0*8yl|#mICWQUtgzeFp9$puHl~m&O+vj3Ox#SxQUa?fY*uK?A;00RiFg(G zK?g=7b5~U4QIK`C*um%=Sw=OJ1eeaV@WZ%hh-3<=lR#(Xesk%?)l4p(EpTwPvN99V@TT)!A8SeFTV+frN=r|5l?K#odjijx2nFgc3kI zC$hVs1S-!z9>xn9MZcRk0YXdYlf~8*LfH$IHKD59H&gLz%6 z#mAYSRJufbRi~LRadwM*G!O2>&U<^d`@<)otXZJJxT@G}4kTx0zPDVhVXwiU)$}5Y z`0iV`8EEh&GlUk&VY9m0Mqr*U&|^Bc?FB`<%{x-o0ATntwIA%(YDcxWs$C)%a%d_@ z?fx!Co+@3p7ha$|pWYD}p6#(PG%_h8K7sQjT_P~|3ZEH0DRxa3~bP&&lPMj3C~!H2QD zq>(f^RUFSqf6K3BMBFy$jiuoSE+DhEq$xLDb7{57 z0B|1pSjYJ5F@cHG%qDZ{ogL$P!BK&sR%zD`gbK#9gRZX17EtAJxN% zys^gb2=X9=7HP}N(iRqt(tot2yyeE%s;L}AcMh;~-W~s_eAe!gIUYdQz5j~T)0trh z>#1U$uOyyl%!Pi(gD&)uHe9Q^27_kHyFCC}n^-KL(=OxHqUfex1YS__RJh0m-S>eM zqAk`aSev*z1lI&-?CycgDm=bdQCp}RqS0_d-4Mf&>u2KyGFxKe8JM1N{GNWw0n$FL z1UDp(h0(1I2Jh9I`?IS}h4R~n zRwRz>8?$fFMB2{UPe^$Ifl;Oc>}@Q9`|8DCeR{?LUQLPfaMsxs8ps=D_aAXORZH~< zdcIOca-F;+D3~M+)Vi4h)I4O3<)$65yI)goQ_vk#fb;Uim>UI4Dv9#2b1;N_Wg>-F zNwKeMKY+su#~NL0uE%_$mw1%ddX2Qs2P!ncM+>wnz}OCQX1!q~oS?OqYU;&ESAAwP z452QWL0&u^mraF#=j_ZeBWhm&F|d!QjwRl^7=Bl7@(43=BkN=3{BRv#QHIk>Umc_w zvP>q|q{lJ=zs|W9%a@8%W>C@MYN1D5{(=Af31+pR#kB`cd0-YlQQTg}+ zL|_h=F9JQ|Gux5c0ehaffHNYLf8VwF+qnM6IjBEI_eceee;o;FY@#~FFVsZjBSp!j z8V*Bgmn{RK!!zqGc;jy)z@Zjo>5{%m1?K}fLEL$l6Dl4f=ye0wNI#)2L=^K(&18Gb zJoj8@WBB;P^T#V)I0`aDSy?$rJU{+-5472NyFp>;Vw43j@3Z=;D2eSfyw5*0Q+&ML zsV&&*3c3$pa`qcaGbEB0*CA~Wp3%PkF?B87FV&rWNb|@GU$LB;l|;YutU*k za1hjUL_BX%G^s;BuzRi4Hl?eqC2z&ZrKh1tZDwnufG$g$LX(j!h%F5(n8D@in3lnX z(*8+3ZT6TVYRcSpM1eMeCps=Fz8q%gyM&B=a7(Vf`4k3dN$IM+`BO^_7HZq4BR|7w z+5kOJ;9_$X%-~arA@qmXSzD|+NMh--%5-9u6t(M=f%&z$<_V#Y_lzn{E$MZZG)+A> zu2E`_Y(MBJ2l*AqvCUmU;yBT}#oQ{V=((mC-QGJwsCOH*a;{1JRTKv7DBNG+M!XL7(^jbv&Qy-o9HNFrmN)-`D3WFtXs>1vBOJpI(=x; zKhJlFdfMf^G#oU(w1+ucMKYPZaDp>$kt=wiYsBCjUY-uz<4JziB>6fXDSLH*2Y z&Px5y`#3!fF=c4>fCMdg-tX582pemU@ZxyFbznL8-=TTo1Sybg9>7h*J^9^~XxXJO z`k9v~=4amxl<;FCV9h2k%?^-ZUzQy^#{JleyH23o1S{r<+t#z6jKS<9rbAM96^1iY zi6{IjauB)UwBhC-_L(MzGCxhhv`?ryc zja_Uwi7$8l!}*vjJppGyp#Wz=*?;jC*xQ&J894rql5A$2giJRtV&DWQh#(+Vs3-5_ z69_tj(>8%z1VtVp>a74r5}j2rG%&;uaTQ|fr&r%ew-HO}76i8`&ki%#)~}q4Y|d$_ zfNp9uc#$#OEca>>MaY6rF`dB|5#S)bghf>>TmmE&S~IFw;PF0UztO6+R-0!TSC?QP z{b(RA_;q3QAPW^XN?qQqu{h<}Vfiv}Rr!lA$C79^1=U>+ng9Dh>v{`?AOZt>CrQ=o zI}=mSnR))8fJpO->rcX?H);oqSQUZ?sR!fH2SoFdcPm5*2y<_u;4h;BqcF*XbwWSv zcJN%!g|L(22Xp!^1?c;T&qm%rpkP&2EQC3JF+SENm$+@7#e!UKD1uQ{TDw43?!b!3 zUooS_rt=xJfa&h?c^hfV>YwQXre3qosz_^c#)FO~d!<)2o}Oxz5HWtr<)1Yw012v4 zhv0w(RfJspDnA^-6Jmr;GkWt%{mAYOm6yPb&Vl&rv@D^K&;#?=X{kaK5FhScNJ_3> z#5u(Saisq2(~pVlrfG#@kLM#Ot~5rZZc%B&h1=gen?R+#t^1bYKf zVvtefX=D$*)39e^2@!~A_}9c${Gf0?1;dk=!Itp#s%0>Io%k`9(bDeI-udd&E6Zfu zcaiv(h`DM3W3Mfda)fYwhB=8RAPkotVt5-z21Ij~Ot9A^SK-1u*zFVK&mF?q1;|wy zrF+XWs^5Q-%Z6I62gTwrRe#F>riVM#fv_TihxSJ6to1X7NVszgivoTa!fPfBBYj94 zuc2m zL_k-<1FoORng1aL{Zx(P7JmUiH zlmTHdzkn75=mS{V=o$V;gzhEaunoJzJ3uq>0_w~77eID^U*w+v0po_N8=sS-DL~!V z%-~rL<0V7PCEWPCpNgpfsein`Fr)+8=N}mUn2x=K`z%efnhSs#23&N1fjdO`M>s%z zP3(;v93%lLq>ZfqBi#QI-aCXAP8-may8x5s`G)KA;{HSYe2szWINWf^b*fc{jl0KecD zRTle?)%_YzJJcVb>;VJ>P?3Lu2S)vCJZlF>Jxj~~X2U5-NNNy(H?8%XD~yFUxNKs&hwWx^)iF@ zGmEv<|7Q7hGrY_+`iz+d_=^9c(_c}UCzq2#%A0|5WjzCXjZUOxOX zU&-^smw$iwKPe;r`&{rP{L35^&+wk6f2-Sn;D2Ww@sjAJj{Gwbp4H!o{#5_}qALFq z{-q%LGklZvKf%A4D!+t%sRRBDi(>mvuz&V4yu^GdD*KFy?fg%ef5ZU%w=d&M`POGt zNSEJ0{qJI~FRTAjlJc1-+x>Tm{%D?m3sk-&cq#w)OpxI98wCF#2KbWcrAXK_(}M4B zF#VQf*h|irx=+uXZUMi+`A;fPFR5M%Wjs^Wh5rWCKgedhWO^w|@XS;b^&3oom;>K0 zB??|ry^IBarYem6Z7RU`#rDs-ZZAn*hSollv?csD$sh0QpTtI9vb>Dpd}e7*`fZj! zM|8d{~YM@vfW-r0z8vJ z<^6B6Ur(}L?ms_c9@hO0^Iy&J_uc51^?d33e#Y!-``?)VG)BGjCq5$&0G8A*r!2qk zUHscGc;VxE=1KqbH=dW%&Ogl({>L!>((m$2W8M9KQ@a1=h51jN|KoG{v(x0K&*iy% e1c3cF4~(n?C}6GmGu)3JNC)6=LGAhZ*Z%`+-T+_# diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ffed3a2..b82aa23 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index cccdd3d..1aa94a4 100755 --- a/gradlew +++ b/gradlew @@ -1,78 +1,127 @@ -#!/usr/bin/env sh +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -81,92 +130,120 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=$((i+1)) + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" fi +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index e95643d..7101f8e 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,4 +1,20 @@ -@if "%DEBUG%" == "" @echo off +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -9,25 +25,29 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -35,48 +55,36 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal