From 9546e28487fc10dfced23a006b5c398442f2b540 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Thu, 19 Dec 2024 09:19:41 -0800 Subject: [PATCH 1/4] rate-limiting: propagate back-pressure from queue as HTTP 429's (#179) Adds a proactive handler that rejects new requests with HTTP 429's when the queue has been blocking for more than 10 consecutive seconds, allowing back- pressure to propagate in advance of filling up the connection backlog queue. --- .gitignore | 6 + CHANGELOG.md | 3 + VERSION | 2 +- spec/inputs/helpers.rb | 6 +- spec/inputs/http_spec.rb | 74 +++++- .../plugins/inputs/http/HttpInitializer.java | 10 +- .../inputs/http/util/ExecutionObserver.java | 186 ++++++++++++++ .../ExecutionObservingMessageHandler.java | 48 ++++ .../util/RejectWhenBlockedInboundHandler.java | 66 +++++ .../http/util/ExecutionObserverTest.java | 230 ++++++++++++++++++ 10 files changed, 622 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/logstash/plugins/inputs/http/util/ExecutionObserver.java create mode 100644 src/main/java/org/logstash/plugins/inputs/http/util/ExecutionObservingMessageHandler.java create mode 100644 src/main/java/org/logstash/plugins/inputs/http/util/RejectWhenBlockedInboundHandler.java create mode 100644 src/test/java/org/logstash/plugins/inputs/http/util/ExecutionObserverTest.java diff --git a/.gitignore b/.gitignore index 7327a2dc..80bc6363 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,9 @@ Gemfile.bak .bundle vendor .idea +.ci +build/* +.ci/* +.gradle/* +lib/logstash-input-http_jars.rb +logstash-input-http.iml diff --git a/CHANGELOG.md b/CHANGELOG.md index bb559315..6134eb8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.10.0 + - add improved proactive rate-limiting, rejecting new requests when queue has been actively blocking for more than 10 seconds [#179](https://github.com/logstash-plugins/logstash-input-http/pull/179) + ## 3.9.2 - Upgrade netty to 4.1.115 [#183](https://github.com/logstash-plugins/logstash-input-http/pull/183) diff --git a/VERSION b/VERSION index 2009c7df..30291cba 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.9.2 +3.10.0 diff --git a/spec/inputs/helpers.rb b/spec/inputs/helpers.rb index 5f65761c..c9da41e3 100644 --- a/spec/inputs/helpers.rb +++ b/spec/inputs/helpers.rb @@ -3,4 +3,8 @@ def certificate_path(filename) File.join(CERTS_DIR, filename) -end \ No newline at end of file +end + +RSpec.configure do |config| + config.formatter = :documentation +end diff --git a/spec/inputs/http_spec.rb b/spec/inputs/http_spec.rb index cc262275..bc44bd68 100644 --- a/spec/inputs/http_spec.rb +++ b/spec/inputs/http_spec.rb @@ -57,7 +57,7 @@ let(:config) { { "port" => port, "threads" => threads, "max_pending_requests" => max_pending_requests } } context "when sending more requests than queue slots" do - it "should block when the queue is full" do + it "rejects additional incoming requests with HTTP 429" do # these will queue and return 200 logstash_queue_size.times.each do |i| response = client.post("http://127.0.0.1:#{port}", :body => '{}').call @@ -65,15 +65,77 @@ end # these will block - (threads + max_pending_requests).times.each do |i| - expect { - client.post("http://127.0.0.1:#{port}", :body => '{}').call - }.to raise_error(Manticore::SocketTimeout) + blocked_calls = (threads + max_pending_requests).times.map do + Thread.new do + begin + {:result => client.post("http://127.0.0.1:#{port}", :body => '{}').call} + rescue Manticore::SocketException, Manticore::SocketTimeout => e + {:exception => e} + end + end + end + + sleep 1 # let those requests go, but not so long that our block-detector starts emitting 429's + + # by now we should be rejecting with 429 since the backlog is full + response = client.post("http://127.0.0.1:#{port}", :body => '{}').call + expect(response.code).to eq(429) + + # ensure that our blocked connections did block + aggregate_failures do + blocked_calls.map(&:value).each do |blocked| + expect(blocked[:result]).to be_nil + expect(blocked[:exception]).to be_a_kind_of Manticore::SocketTimeout + end + end + end + end + end + + describe "observing queue back-pressure" do + let(:logstash_queue_size) { rand(10) + 1 } + let(:max_pending_requests) { rand(5) + 1 } + let(:threads) { rand(4) + 1 } + let(:logstash_queue) { SizedQueue.new(logstash_queue_size) } + let(:client_options) { { + "request_timeout" => 0.1, + "connect_timeout" => 3, + "socket_timeout" => 0.1 + } } + + let(:config) { { "port" => port, "threads" => threads, "max_pending_requests" => max_pending_requests } } + + context "when sending request to an input that has blocked connections" do + it "rejects incoming requests with HTTP 429" do + # these will queue and return 200 + logstash_queue_size.times.each do |i| + response = client.post("http://127.0.0.1:#{port}", :body => '{}').call + expect(response.code).to eq(200) end - # by now we should be rejecting with 429 + # these will block + blocked_call = Thread.new do + begin + {:result => client.post("http://127.0.0.1:#{port}", :body => '{}').call} + rescue Manticore::SocketException, Manticore::SocketTimeout => e + {:exception => e} + end + end + + sleep 12 # let that requests go, and ensure it is blocking long enough to be problematic + + # by now we should be rejecting with 429 since at least one existing request is blocked + # for more than 10s. response = client.post("http://127.0.0.1:#{port}", :body => '{}').call expect(response.code).to eq(429) + + # ensure that our blocked connections did block + aggregate_failures do + blocked_call.value.tap do |blocked| + expect(blocked[:result]).to be_nil + expect(blocked[:exception]).to be_a_kind_of Manticore::SocketTimeout + end + end end end end diff --git a/src/main/java/org/logstash/plugins/inputs/http/HttpInitializer.java b/src/main/java/org/logstash/plugins/inputs/http/HttpInitializer.java index 27f21500..04a70642 100644 --- a/src/main/java/org/logstash/plugins/inputs/http/HttpInitializer.java +++ b/src/main/java/org/logstash/plugins/inputs/http/HttpInitializer.java @@ -8,8 +8,12 @@ import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpServerCodec; import io.netty.handler.ssl.SslHandler; +import org.logstash.plugins.inputs.http.util.ExecutionObserver; +import org.logstash.plugins.inputs.http.util.ExecutionObservingMessageHandler; +import org.logstash.plugins.inputs.http.util.RejectWhenBlockedInboundHandler; import org.logstash.plugins.inputs.http.util.SslHandlerProvider; +import java.time.Duration; import java.util.concurrent.ThreadPoolExecutor; /** @@ -22,9 +26,11 @@ public class HttpInitializer extends ChannelInitializer { private final HttpResponseStatus responseStatus; private final ThreadPoolExecutor executorGroup; + private final ExecutionObserver executionObserver = new ExecutionObserver(); + public HttpInitializer(IMessageHandler messageHandler, ThreadPoolExecutor executorGroup, int maxContentLength, HttpResponseStatus responseStatus) { - this.messageHandler = messageHandler; + this.messageHandler = new ExecutionObservingMessageHandler(executionObserver, messageHandler); this.executorGroup = executorGroup; this.maxContentLength = maxContentLength; this.responseStatus = responseStatus; @@ -37,7 +43,9 @@ protected void initChannel(SocketChannel socketChannel) throws Exception { SslHandler sslHandler = sslHandlerProvider.getSslHandler(socketChannel.alloc()); pipeline.addLast(sslHandler); } + pipeline.addLast(new HttpServerCodec()); + pipeline.addLast(new RejectWhenBlockedInboundHandler(executionObserver, Duration.ofSeconds(10))); pipeline.addLast(new HttpContentDecompressor()); pipeline.addLast(new HttpObjectAggregator(maxContentLength)); pipeline.addLast(new HttpServerHandler(messageHandler.copy(), executorGroup, responseStatus)); diff --git a/src/main/java/org/logstash/plugins/inputs/http/util/ExecutionObserver.java b/src/main/java/org/logstash/plugins/inputs/http/util/ExecutionObserver.java new file mode 100644 index 00000000..408f346f --- /dev/null +++ b/src/main/java/org/logstash/plugins/inputs/http/util/ExecutionObserver.java @@ -0,0 +1,186 @@ +package org.logstash.plugins.inputs.http.util; + +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; + +/** + * An {@code ExecutionObserver} observes possibly-concurrent execution, and provides information about the + * longest-running observed execution. + * + *

+ * It is concurrency-safe and non-blocking, and uses plain memory access where practical. + *

+ */ +public class ExecutionObserver { + private final AtomicReference tail; // newest execution + private final AtomicReference head; // oldest execution + + private final LongSupplier nanosSupplier; + + public ExecutionObserver() { + this(System::nanoTime); + } + + ExecutionObserver(final LongSupplier nanosSupplier) { + this.nanosSupplier = nanosSupplier; + final Execution anchor = new Execution(nanosSupplier.getAsLong(), true); + this.tail = new AtomicReference<>(anchor); + this.head = new AtomicReference<>(anchor); + } + + /** + * @see ExecutionObserver#anyExecuting(Duration) + * @return true if there are any active executions. + */ + public boolean anyExecuting() { + return this.anyExecuting(Duration.ZERO); + } + + /** + * @param minimumDuration a threshold to exclude young executions + * @return true if any active execution has been running for at least the provided {@code Duration} + */ + public boolean anyExecuting(final Duration minimumDuration) { + final Execution headExecution = compactHead(); + if (headExecution.isComplete) { + return false; + } else { + return nanosSupplier.getAsLong() - headExecution.startNanos >= minimumDuration.toNanos(); + } + } + + // visible for test + Optional longestExecuting() { + final Execution headExecution = compactHead(); + if (headExecution.isComplete) { + return Optional.empty(); + } else { + return Optional.of(Duration.ofNanos(nanosSupplier.getAsLong() - headExecution.startNanos)); + } + } + + // test inspections + Stats stats() { + int nodes = 0; + int executing = 0; + + Execution candidate = this.head.get(); + while (candidate != null) { + nodes += 1; + if (!candidate.isComplete) { + executing += 1; + } + candidate = candidate.next.get(); + } + return new Stats(nodes, executing); + } + + static class Stats { + final int nodes; + final int executing; + + Stats(int nodes, int executing) { + this.nodes = nodes; + this.executing = executing; + } + } + + @FunctionalInterface + public interface ExceptionalSupplier { + T get() throws E; + } + + public T observeExecution(final ExceptionalSupplier supplier) throws E { + final Execution execution = startExecution(); + try { + return supplier.get(); + } finally { + final boolean isCompact = execution.markComplete(); + if (!isCompact) { + this.compactHead(); + } + } + } + + @FunctionalInterface + public interface ExceptionalRunnable { + void run() throws E; + } + + public void observeExecution(final ExceptionalRunnable runnable) throws E { + observeExecution(() -> { runnable.run(); return null; }); + } + + // visible for test + Execution startExecution() { + final Execution newTail = new Execution(nanosSupplier.getAsLong()); + + // atomically attach the new execution as a new (detached) tail + final Execution oldTail = this.tail.getAndSet(newTail); + // attach our new tail to the old one + oldTail.linkNext(newTail); + + return newTail; + } + + private Execution compactHead() { + return this.head.updateAndGet(Execution::seekHead); + } + + static class Execution { + + private final long startNanos; + + private volatile boolean isComplete; + private final AtomicReference next = new AtomicReference<>(); + + Execution(long startNanos) { + this(startNanos, false); + } + + Execution(final long startNanos, + final boolean isComplete) { + this.startNanos = startNanos; + this.isComplete = isComplete; + } + + /** + * marks this execution as complete + * @return true if the completion resulted in a compaction + */ + boolean markComplete() { + isComplete = true; + + final Execution preCompletionNext = this.next.get(); + if (preCompletionNext != null) { + final Execution result = this.next.updateAndGet(Execution::seekHead); + return result != preCompletionNext; + } + + return false; + } + + private void linkNext(final Execution proposedNext) { + final Execution result = next.updateAndGet((ex) -> ex == null ? proposedNext : ex); + if (result != proposedNext) { + throw new IllegalStateException(); + } + } + + /** + * @return the first {@code Execution} that is either not yet complete + * or is the current tail, possibly itself. + */ + private Execution seekHead() { + Execution compactedHead = this; + Execution candidate = this.next.get(); + while (candidate != null && compactedHead.isComplete) { + compactedHead = candidate; + candidate = candidate.next.get(); + } + return compactedHead; + } + } +} diff --git a/src/main/java/org/logstash/plugins/inputs/http/util/ExecutionObservingMessageHandler.java b/src/main/java/org/logstash/plugins/inputs/http/util/ExecutionObservingMessageHandler.java new file mode 100644 index 00000000..fcb11be6 --- /dev/null +++ b/src/main/java/org/logstash/plugins/inputs/http/util/ExecutionObservingMessageHandler.java @@ -0,0 +1,48 @@ +package org.logstash.plugins.inputs.http.util; + +import org.logstash.plugins.inputs.http.IMessageHandler; + +import java.util.Map; + +/** + * An implementation of {@link IMessageHandler} that wraps another {@link IMessageHandler} with an + * {@link ExecutionObserver}, ensuring that the delegate's {@link IMessageHandler#onNewMessage} is + * observed. + */ +public class ExecutionObservingMessageHandler implements IMessageHandler { + private final ExecutionObserver executionObserver; + private final IMessageHandler delegate; + + public ExecutionObservingMessageHandler(final ExecutionObserver executionObserver, + final IMessageHandler delegate) { + this.executionObserver = executionObserver; + this.delegate = delegate; + } + + @Override + public boolean onNewMessage(final String remoteAddress, + final Map headers, + final String body) { + return executionObserver.observeExecution(() -> delegate.onNewMessage(remoteAddress, headers, body)); + } + + @Override + public boolean validatesToken(final String token) { + return delegate.validatesToken(token); + } + + @Override + public boolean requiresToken() { + return delegate.requiresToken(); + } + + @Override + public IMessageHandler copy() { + return new ExecutionObservingMessageHandler(this.executionObserver, delegate.copy()); + } + + @Override + public Map responseHeaders() { + return delegate.responseHeaders(); + } +} diff --git a/src/main/java/org/logstash/plugins/inputs/http/util/RejectWhenBlockedInboundHandler.java b/src/main/java/org/logstash/plugins/inputs/http/util/RejectWhenBlockedInboundHandler.java new file mode 100644 index 00000000..974bdb40 --- /dev/null +++ b/src/main/java/org/logstash/plugins/inputs/http/util/RejectWhenBlockedInboundHandler.java @@ -0,0 +1,66 @@ +package org.logstash.plugins.inputs.http.util; + +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.util.ReferenceCountUtil; + +import java.time.Duration; + +/** + * A {@link RejectWhenBlockedInboundHandler} is a {@link io.netty.channel.ChannelInboundHandler} that rejects incoming + * http requests with HTTP 429 when its {@link ExecutionObserver} reports that one or more active executions have been + * running for more than a configurable {@link Duration}. It must be injected into a pipeline after the + * {@link io.netty.handler.codec.http.HttpServerCodec} that decodes the incoming byte-stream into {@link HttpRequest}s. + * + *

+ * This implementation is keep-alive friendly, but will close the connection if the current request didn't + * include an {@code Expect: 100-continue} to indicate that they won't pre-send the payload. + *

+ */ +public class RejectWhenBlockedInboundHandler extends ChannelInboundHandlerAdapter { + + private final ExecutionObserver executionObserver; + private final FullHttpResponse REJECT_RESPONSE = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.TOO_MANY_REQUESTS, Unpooled.EMPTY_BUFFER); + + private final Duration blockThreshold; + + public RejectWhenBlockedInboundHandler(final ExecutionObserver executionObserver, + final Duration blockThreshold) { + this.executionObserver = executionObserver; + this.blockThreshold = blockThreshold; + } + + @Override + public void channelRead(final ChannelHandlerContext ctx, + final Object msg) throws Exception { + if (msg instanceof HttpRequest) { + final HttpRequest req = (HttpRequest) msg; + if (executionObserver.anyExecuting(this.blockThreshold)) { + final HttpResponse rejection = REJECT_RESPONSE.retainedDuplicate(); + ReferenceCountUtil.release(msg); + ChannelFuture channelFuture = ctx.writeAndFlush(rejection); + + // If the client started to send data already, close because it's impossible to recover. + // If keep-alive is on and 'Expect: 100-continue' is present, it is safe to leave the connection open. + if (HttpUtil.isKeepAlive(req) && HttpUtil.is100ContinueExpected(req)) { + channelFuture.addListener(ChannelFutureListener.CLOSE_ON_FAILURE); + } else { + channelFuture.addListener(ChannelFutureListener.CLOSE); + } + + return; + } + } + super.channelRead(ctx, msg); + } +} diff --git a/src/test/java/org/logstash/plugins/inputs/http/util/ExecutionObserverTest.java b/src/test/java/org/logstash/plugins/inputs/http/util/ExecutionObserverTest.java new file mode 100644 index 00000000..fda18093 --- /dev/null +++ b/src/test/java/org/logstash/plugins/inputs/http/util/ExecutionObserverTest.java @@ -0,0 +1,230 @@ +package org.logstash.plugins.inputs.http.util; + +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Optional; +import java.util.PrimitiveIterator; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.LongAdder; +import java.util.stream.IntStream; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +class ExecutionObserverTest { + @Test + void testBasicFunctionality() { + final LongAdder nanos = new LongAdder(); + final ExecutionObserver observer = new ExecutionObserver(nanos::longValue); + + assertThat(observer.anyExecuting(), is(false)); + assertThat(observer.anyExecuting(Duration.ofSeconds(1)), is(false)); + + nanos.add(Duration.ofSeconds(10).toNanos()); + assertThat(observer.anyExecuting(), is(false)); + assertThat(observer.anyExecuting(Duration.ofSeconds(1)), is(false)); + + final ExecutionObserver.Execution exe1 = observer.startExecution(); + assertThat(observer.anyExecuting(), is(true)); + assertThat(observer.anyExecuting(Duration.ofNanos(1)), is(false)); + assertThat(observer.anyExecuting(Duration.ofSeconds(1)), is(false)); + + nanos.add(Duration.ofSeconds(10).toNanos()); + assertThat(observer.anyExecuting(), is(true)); + assertThat(observer.anyExecuting(Duration.ofNanos(1)), is(true)); + assertThat(observer.anyExecuting(Duration.ofSeconds(1)), is(true)); + assertThat(observer.anyExecuting(Duration.ofSeconds(10).minus(Duration.ofNanos(1))), is(true)); + assertThat(observer.anyExecuting(Duration.ofSeconds(10)), is(true)); + assertThat(observer.anyExecuting(Duration.ofSeconds(10).plus(Duration.ofNanos(1))), is(false)); + + exe1.markComplete(); + assertThat(observer.anyExecuting(), is(false)); + assertThat(observer.anyExecuting(Duration.ofSeconds(1)), is(false)); + } + + @Test + void testManyConcurrentExecutions() { + final LongAdder nanos = new LongAdder(); + final ExecutionObserver observer = new ExecutionObserver(nanos::longValue); + final PrimitiveIterator.OfLong randomLong = ThreadLocalRandom.current().longs(1L, 1_000_000_000L).iterator(); + + // mark the beginning of several executions, advancing the nano clock in randomized increments + nanos.add(randomLong.next()); + final Handle exe1 = new Handle(nanos.longValue(), observer.startExecution()); + nanos.add(randomLong.next()); + final Handle exe2 = new Handle(nanos.longValue(), observer.startExecution()); + nanos.add(randomLong.next()); + final Handle exe3 = new Handle(nanos.longValue(), observer.startExecution()); + nanos.add(randomLong.next()); // because we're recording two for the same nanotime + final Handle exe4a = new Handle(nanos.longValue(), observer.startExecution()); + final Handle exe4b = new Handle(nanos.longValue(), observer.startExecution()); + nanos.add(randomLong.next()); + final Handle exe5 = new Handle(nanos.longValue(), observer.startExecution()); + nanos.add(randomLong.next()); + final Handle exe6 = new Handle(nanos.longValue(), observer.startExecution()); + nanos.add(randomLong.next()); + final Handle exe7 = new Handle(nanos.longValue(), observer.startExecution()); + nanos.add(randomLong.next()); + final Handle exe8 = new Handle(nanos.longValue(), observer.startExecution()); + nanos.add(randomLong.next()); + final Handle exe9 = new Handle(nanos.longValue(), observer.startExecution()); + + // all are still running, so exe1 is the longest-running + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe1.nanoTime))); + + // mark several intermediates complete, and ensure exe1 is still the longest-running + exe2.execution.markComplete(); + exe3.execution.markComplete(); + exe7.execution.markComplete(); + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe1.nanoTime))); + + // mark exe1 complete, and ensure that exe4(a/b) are the longest-running + exe1.execution.markComplete(); + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe4a.nanoTime))); + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe4b.nanoTime))); + + // mark exe4a complete, but exe4b is still the oldest + exe4a.execution.markComplete(); + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe4a.nanoTime))); + + // mark exe5 complete, but exe4b is still the oldest + exe5.execution.markComplete(); + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe4a.nanoTime))); + + // mark exe4b complete, so now exe6 is the oldest + exe4b.execution.markComplete(); + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe6.nanoTime))); + + // mark exe9 complete, but exe6 is still the oldest + exe9.execution.markComplete(); + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe6.nanoTime))); + + // advance the clock again, adding another + nanos.add(1000000000L); + final Handle exe10 = new Handle(nanos.longValue(), observer.startExecution()); + + nanos.add(10000000000L); + + // mark exe10 complete, but exe6 is still the oldest + exe10.execution.markComplete(); + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe6.nanoTime))); + + // mark exe6 complete, now exe8 is the oldest + exe6.execution.markComplete(); + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe8.nanoTime))); + + // mark exe8 complete, now there are none waiting + exe8.execution.markComplete(); + assertThat(observer.anyExecuting(), is(false)); + + // start two more + final Handle exe11 = new Handle(nanos.longValue(), observer.startExecution()); + nanos.add(randomLong.next()); + final Handle exe12 = new Handle(nanos.longValue(), observer.startExecution()); + nanos.add(randomLong.next()); + + // exe11 is our oldest + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe11.nanoTime))); + + // mark exe11 complete; exe12 is now our oldest + exe11.execution.markComplete(); + validateLongestExecuting(observer, Duration.ofNanos(nanos.longValue()).minus(Duration.ofNanos(exe12.nanoTime))); + + // mark exe12 complete; none executing + exe12.execution.markComplete(); + assertThat(observer.anyExecuting(), is(false)); + + ExecutionObserver.Stats stats = observer.stats(); + assertThat(stats.nodes, is(both(greaterThan(0)).and(lessThanOrEqualTo(3)))); + assertThat(stats.executing, is(0)); + } + + private void validateLongestExecuting(final ExecutionObserver observer, final Duration expectedLongestExecutionDuration) { + assertThat(observer.anyExecuting(), is(true)); + assertThat(observer.longestExecuting(), equalTo(Optional.of(expectedLongestExecutionDuration))); + assertThat(observer.anyExecuting(expectedLongestExecutionDuration), is(true)); + assertThat(observer.anyExecuting(expectedLongestExecutionDuration.plus(Duration.ofNanos(1))), is(false)); + } + + + @RepeatedTest(value = 10) + void testThreadSafetyBruteForce() { + final int scale = 1000; + final int concurrency = 100; + final ExecutorService executorService = Executors.newFixedThreadPool(concurrency); + final ExecutionObserver observer = new ExecutionObserver(); + + final AtomicInteger maxConcurrency = new AtomicInteger(0); + final AtomicInteger maxNodes = new AtomicInteger(0); + + try { + // submit $scale tasks to the executor, each of which sleeps a variable amount + // before triggering an observed execution that lasts a random length. The goal + // is to have many concurrent and interleaved executions. + CompletableFuture allRun = CompletableFuture.allOf(IntStream.range(0, scale) + .mapToObj((idx) -> + CompletableFuture.runAsync(() -> { + interruptibleSleep(ThreadLocalRandom.current().nextInt(5)); + observer.observeExecution(() -> { + final ExecutionObserver.Stats stats = observer.stats(); + maxConcurrency.accumulateAndGet(stats.executing, Math::max); + maxNodes.accumulateAndGet(stats.nodes, Math::max); + interruptibleSleep(ThreadLocalRandom.current().nextInt(1, 100)); + }); + }, executorService)) + .toArray(CompletableFuture[]::new)); + + // wait until all have run + allRun.get(30, TimeUnit.SECONDS); + + // at some point in the execution, we want there to be many things running concurrently, + // but we also want the max uncompacted nodes to never get too high + assertThat(maxConcurrency.get(), is(greaterThan(1))); + assertThat(maxNodes.get(), is(lessThanOrEqualTo(concurrency * 2))); + + // without queries, we should at least have some compaction + final ExecutionObserver.Stats preCompactionStats = observer.stats(); + assertThat(preCompactionStats.executing, is(0)); + assertThat(preCompactionStats.nodes, is(both(greaterThan(0)).and(lessThan((int) Math.sqrt(concurrency))))); + + // query triggers tail compaction, leaving 2 or fewer nodes. + assertThat(observer.anyExecuting(), is(false)); + final ExecutionObserver.Stats postCompactionStats = observer.stats(); + assertThat(postCompactionStats.executing, is(0)); + assertThat(postCompactionStats.nodes, is(both(greaterThan(0)).and(lessThanOrEqualTo(2)))); + + + } catch (ExecutionException | InterruptedException | TimeoutException e) { + throw new RuntimeException(e); + } finally { + executorService.shutdown(); + } + } + + static class Handle { + final long nanoTime; + final ExecutionObserver.Execution execution; + + public Handle(long nanoTime, ExecutionObserver.Execution execution) { + this.nanoTime = nanoTime; + this.execution = execution; + } + } + + static void interruptibleSleep(final int millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file From 98ea405c22220f2db419c4ad65473c296f4b8f87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Tue, 4 Feb 2025 17:40:11 +0000 Subject: [PATCH 2/4] Backport #191: Name netty threads with plugin id and their purpose (#192) Co-authored-by: Mashhur <99575341+mashhurs@users.noreply.github.com> --- CHANGELOG.md | 3 +++ VERSION | 2 +- lib/logstash/inputs/http.rb | 3 ++- .../plugins/inputs/http/NettyHttpServer.java | 12 ++++++------ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6134eb8f..97825e99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.10.1 + - Properly naming netty threads [#191](https://github.com/logstash-plugins/logstash-input-http/pull/191) + ## 3.10.0 - add improved proactive rate-limiting, rejecting new requests when queue has been actively blocking for more than 10 seconds [#179](https://github.com/logstash-plugins/logstash-input-http/pull/179) diff --git a/VERSION b/VERSION index 30291cba..f870be23 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.10.0 +3.10.1 diff --git a/lib/logstash/inputs/http.rb b/lib/logstash/inputs/http.rb index ef64507d..4a9f1cb6 100644 --- a/lib/logstash/inputs/http.rb +++ b/lib/logstash/inputs/http.rb @@ -391,7 +391,8 @@ def normalize_ssl_client_authentication_value!(verify_mode, ssl_verify_mode) def create_http_server(message_handler) org.logstash.plugins.inputs.http.NettyHttpServer.new( - @host, @port, message_handler, build_ssl_params, @threads, @max_pending_requests, @max_content_length, @response_code) + @id, @host, @port, message_handler, build_ssl_params, @threads, + @max_pending_requests, @max_content_length, @response_code) end def build_ssl_params diff --git a/src/main/java/org/logstash/plugins/inputs/http/NettyHttpServer.java b/src/main/java/org/logstash/plugins/inputs/http/NettyHttpServer.java index f578cef6..c7498ff2 100644 --- a/src/main/java/org/logstash/plugins/inputs/http/NettyHttpServer.java +++ b/src/main/java/org/logstash/plugins/inputs/http/NettyHttpServer.java @@ -31,9 +31,9 @@ public class NettyHttpServer implements Runnable, Closeable { private final ThreadPoolExecutor executorGroup; private final HttpResponseStatus responseStatus; - public NettyHttpServer(String host, int port, IMessageHandler messageHandler, - SslHandlerProvider sslHandlerProvider, int threads, - int maxPendingRequests, int maxContentLength, int responseCode) + public NettyHttpServer(final String id, final String host, final int port, final IMessageHandler messageHandler, + final SslHandlerProvider sslHandlerProvider, final int threads, + final int maxPendingRequests, final int maxContentLength, final int responseCode) { this.host = host; this.port = port; @@ -42,12 +42,12 @@ public NettyHttpServer(String host, int port, IMessageHandler messageHandler, // boss group is responsible for accepting incoming connections and sending to worker loop // process group is channel handler, see the https://github.com/netty/netty/discussions/13305 // see the https://github.com/netty/netty/discussions/11808#discussioncomment-1610918 for why separation is good - bossGroup = new NioEventLoopGroup(1, daemonThreadFactory("http-input-connector")); - processorGroup = new NioEventLoopGroup(threads, daemonThreadFactory("http-input-processor")); + bossGroup = new NioEventLoopGroup(1, daemonThreadFactory(id + "-bossGroup")); + processorGroup = new NioEventLoopGroup(threads, daemonThreadFactory(id + "-processorGroup")); // event handler group executorGroup = new ThreadPoolExecutor(threads, threads, 0, TimeUnit.MILLISECONDS, - new ArrayBlockingQueue<>(maxPendingRequests), daemonThreadFactory("http-input-handler-executor"), + new ArrayBlockingQueue<>(maxPendingRequests), daemonThreadFactory(id + "-executorGroup"), new CustomRejectedExecutionHandler()); final HttpInitializer httpInitializer = new HttpInitializer(messageHandler, executorGroup, From 5fe44acfaecd2af45b71d7f7ecbfc8eee27905d3 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Wed, 12 Feb 2025 11:45:14 -0800 Subject: [PATCH 3/4] Upgrade netty to 4.1.118 --- CHANGELOG.md | 3 +++ VERSION | 2 +- build.gradle | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97825e99..dfa40ae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.10.2 + - Upgrade netty to 4.1.118 [#194](https://github.com/logstash-plugins/logstash-input-http/pull/194) + ## 3.10.1 - Properly naming netty threads [#191](https://github.com/logstash-plugins/logstash-input-http/pull/191) diff --git a/VERSION b/VERSION index f870be23..7b59a5ca 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.10.1 +3.10.2 diff --git a/build.gradle b/build.gradle index 28329822..8884c8df 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ version rootProject.file('VERSION').text.trim() description = "HTTP Input Netty implementation" String log4jVersion = '2.17.0' -String nettyVersion = '4.1.115.Final' +String nettyVersion = '4.1.118.Final' String junitVersion = '5.9.2' java { From 1632b5200d825db9ab1fd45c8b17d9b087c2ce66 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Thu, 4 Sep 2025 11:21:47 -0700 Subject: [PATCH 4/4] Bump netty and prepare for release --- CHANGELOG.md | 3 +++ VERSION | 2 +- build.gradle | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfa40ae4..130d9404 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.10.3 + - Upgrade netty to 4.1.126 [#199](https://github.com/logstash-plugins/logstash-input-http/pull/199) + ## 3.10.2 - Upgrade netty to 4.1.118 [#194](https://github.com/logstash-plugins/logstash-input-http/pull/194) diff --git a/VERSION b/VERSION index 7b59a5ca..7d4ef04f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.10.2 +3.10.3 diff --git a/build.gradle b/build.gradle index 8884c8df..73cde1bd 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ version rootProject.file('VERSION').text.trim() description = "HTTP Input Netty implementation" String log4jVersion = '2.17.0' -String nettyVersion = '4.1.118.Final' +String nettyVersion = '4.1.126.Final' String junitVersion = '5.9.2' java {