diff --git a/.github/workflows/DependabotReview.yaml b/.github/workflows/DependabotReview.yaml new file mode 100644 index 000000000..347057a2d --- /dev/null +++ b/.github/workflows/DependabotReview.yaml @@ -0,0 +1,14 @@ +name: Dependency Review +on: [pull_request] + +permissions: + contents: read +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@v3 + + - name: 'Dependency Review' + uses: actions/dependency-review-action@v1 diff --git a/.github/workflows/ScanCode.yaml b/.github/workflows/ScanCode.yaml new file mode 100644 index 000000000..0bb6bc8f0 --- /dev/null +++ b/.github/workflows/ScanCode.yaml @@ -0,0 +1,52 @@ +name: "Code Scanning - Action" + +on: + push: + branches: [master] + pull_request: + branches: [master] + schedule: + - cron: '30 1 * * 0' + +jobs: + CodeQL-Build: + # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest + runs-on: ubuntu-latest + + permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories + actions: read + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + # Override language selection by uncommenting this and choosing your languages + # with: + # languages: go, javascript, csharp, python, cpp, java + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below). + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/mac_tarball_notarization.yml b/.github/workflows/mac_tarball_notarization.yml new file mode 100644 index 000000000..054c255c1 --- /dev/null +++ b/.github/workflows/mac_tarball_notarization.yml @@ -0,0 +1,46 @@ +name: Sign Mac OS artifacts +on: + workflow_dispatch: + inputs: + proxy_version: + description: 'proxy version. Example: 11.1.0' + required: true + default: "0.0" + release_type: + description: 'Release type. Example: "proxy-GA" / "proxt-snapshot"' + required: true + default: "proxy-test" +jobs: + sign_proxy_mac_artifact: + # environment with secrets as env vars on wavefront-proxy repo + environment: macos_tarball_notarization + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + APP_SPECIFIC_PW: ${{ secrets.APP_SPECIFIC_PW }} + CERTIFICATE_OSX_P12: ${{ secrets.CERTIFICATE_OSX_P12 }} + CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }} + ESO_DEV_ACCOUNT: ${{ secrets.ESO_DEV_ACCOUNT }} + USERNAME: ${{ secrets.USERNAME }} + WAVEFRONT_TEAM_CERT_P12: ${{ secrets.WAVEFRONT_TEAM_CERT_P12 }} + WAVEFRONT_TEAM_CERT_PASSWORD: ${{ secrets.WAVEFRONT_TEAM_CERT_PASSWORD }} + WF_DEV_ACCOUNT: ${{ secrets.WF_DEV_ACCOUNT }} + + runs-on: macos-latest + steps: + - name: "${{ github.event.inputs.proxy_version }}-${{ github.event.inputs.release_type }}-checkout_proxy_code" + uses: actions/checkout@v3 + + - name: "${{ github.event.inputs.proxy_version }}-${{ github.event.inputs.release_type }}-before_install" + run: | + set -x + ls -la; pwd; pip3 install awscli; chmod +x ./macos_proxy_notarization/create_credentials.sh; ./macos_proxy_notarization/create_credentials.sh; cat ~/.aws/credentials; + set +x + + - name: "${{ github.event.inputs.proxy_version }}-${{ github.event.inputs.release_type }}-notarize" + run: | + set -x + chmod +x ./macos_proxy_notarization/proxy_notarization.sh; ./macos_proxy_notarization/proxy_notarization.sh 'wfproxy-${{ github.event.inputs.proxy_version }}.tar.gz' + set +x + sleep 60 diff --git a/.github/workflows/mac_tarball_notarization_sbhakta_test.yaml b/.github/workflows/mac_tarball_notarization_sbhakta_test.yaml new file mode 100644 index 000000000..9a0cd8259 --- /dev/null +++ b/.github/workflows/mac_tarball_notarization_sbhakta_test.yaml @@ -0,0 +1,45 @@ +name: Sign Mac OS artifacts SBHAKTA TEST +on: + workflow_dispatch: + inputs: + zip_name: + description: 'full name of zip file uploaded to to_be_notarized bucket in s3. Example: wfproxy_macos_11.4_20220907-115828.zip' + required: true + release_type: + description: 'Release type. Example: "proxy-GA" / "proxy-snapshot"' + required: true + default: "proxy-test" +jobs: + sign_proxy_mac_artifact: + # environment with secrets as env vars on wavefront-proxy repo + environment: macos_tarball_notarization + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + APP_SPECIFIC_PW: ${{ secrets.APP_SPECIFIC_PW }} + CERTIFICATE_OSX_P12: ${{ secrets.CERTIFICATE_OSX_P12 }} + CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }} + ESO_DEV_ACCOUNT: ${{ secrets.ESO_DEV_ACCOUNT }} + USERNAME: ${{ secrets.USERNAME }} + WAVEFRONT_TEAM_CERT_P12: ${{ secrets.WAVEFRONT_TEAM_CERT_P12 }} + WAVEFRONT_TEAM_CERT_PASSWORD: ${{ secrets.WAVEFRONT_TEAM_CERT_PASSWORD }} + WF_DEV_ACCOUNT: ${{ secrets.WF_DEV_ACCOUNT }} + + runs-on: macos-latest + steps: + - name: "${{ github.event.inputs.zip_name }}-${{ github.event.inputs.release_type }}-checkout_proxy_code" + uses: actions/checkout@v3 + + - name: "${{ github.event.inputs.zip_name }}-${{ github.event.inputs.release_type }}-before_install" + run: | + set -x + ls -la; pwd; pip3 install awscli; chmod +x ./macos_proxy_notarization/create_credentials.sh; ./macos_proxy_notarization/create_credentials.sh; cat ~/.aws/credentials; + set +x + + - name: "${{ github.event.inputs.zip_name }}-${{ github.event.inputs.release_type }}-notarize" + run: | + set -x + chmod +x ./macos_proxy_notarization/proxy_notarization_sbhakta_test.sh; ./macos_proxy_notarization/proxy_notarization_sbhakta_test.sh '${{ github.event.inputs.zip_name }}' + set +x + sleep 60 diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 000000000..9fcf10d73 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,30 @@ +name: Java CI with Maven + +on: + push: + branches: ["**"] + pull_request: + branches: [master, dev, "release-**"] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + java: ["11"] + # java: ["11", "16", "17"] + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: ${{ matrix.java }} + distribution: "temurin" + cache: maven + - name: versions + run: mvn -v + - name: Check code format + run: mvn -f proxy git-code-format:validate-code-format + - name: Build with Maven + run: mvn -f proxy test -B diff --git a/.gitignore b/.gitignore index 323d04323..9ff7171dc 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,4 @@ test/proxy*.out test/.wavefront_id dependency-reduced-pom.xml - +out/* diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29bb..000000000 diff --git a/.idea/codeStyleSettings.xml b/.idea/codeStyleSettings.xml new file mode 100644 index 000000000..94135f335 --- /dev/null +++ b/.idea/codeStyleSettings.xml @@ -0,0 +1,402 @@ + + + + + + \ No newline at end of file diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml new file mode 100644 index 000000000..cc1791058 --- /dev/null +++ b/.mvn/extensions.xml @@ -0,0 +1,10 @@ + + + + me.qoomon + maven-git-versioning-extension + 7.4.0 + + + \ No newline at end of file diff --git a/.mvn/maven-git-versioning-extension.xml b/.mvn/maven-git-versioning-extension.xml new file mode 100644 index 000000000..05035177d --- /dev/null +++ b/.mvn/maven-git-versioning-extension.xml @@ -0,0 +1,22 @@ + + + + + .+ + ${ref}-SNAPSHOT + + + + .+ + ${ref} + + + + + + ${version} + + + \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c54da0a45..000000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: java -jdk: - - oraclejdk8 - - -addons: - apt: - packages: - - oracle-java8-installer diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..730d4b63d --- /dev/null +++ b/Makefile @@ -0,0 +1,98 @@ +TS := $(shell date +%Y%m%d-%H%M%S) + +VERSION := $(shell mvn -f proxy -q -Dexec.executable=echo -Dexec.args='$${project.version}' --non-recursive exec:exec) +ARTIFACT_ID := $(shell mvn -f proxy -q -Dexec.executable=echo -Dexec.args='$${project.artifactId}' --non-recursive exec:exec) +REVISION ?= ${TS} +USER ?= $(LOGNAME) +REPO ?= proxy-dev +PACKAGECLOUD_USER ?= wavefront +PACKAGECLOUD_REPO ?= proxy-next + +DOCKER_TAG ?= ${VERSION}_${REVISION} + +out = $(shell pwd)/out +$(shell mkdir -p $(out)) + +main: build-jar + +.info: + @echo "\n----------\nBuilding Proxy ${VERSION}\nDocker tag: $(USER)/$(REPO):$(DOCKER_TAG) \n----------\n" + +jenkins: .info build-jar build-linux push-linux docker-multi-arch clean + +##### +# Build Proxy jar file +##### +build-jar: .info + mvn -f proxy --batch-mode clean package ${MVN_ARGS} + cp proxy/target/${ARTIFACT_ID}-${VERSION}-spring-boot.jar ${out} + +##### +# Build single docker image +##### +docker: .info .cp-docker + docker build -t $(USER)/$(REPO):$(DOCKER_TAG) docker/ + +##### +# Build single docker image +##### +docker-RHEL: .info .cp-docker + podman build -t $(USER)/$(REPO):$(DOCKER_TAG) -f ./docker/Dockerfile-rhel docker/ + +##### +# Build multi arch (amd64 & arm64) docker images +##### +docker-multi-arch: .info .cp-docker + docker buildx create --use + docker buildx build --platform linux/amd64,linux/arm64 -t $(USER)/$(REPO):$(DOCKER_TAG) --push docker/ + +docker-multi-arch-with-latest-tag: .info .cp-docker + docker buildx create --use + docker buildx build --platform linux/amd64,linux/arm64 -t $(USER)/$(REPO):$(DOCKER_TAG) -t $(USER)/$(REPO):latest --push docker/ + +##### +# Build rep & deb packages +##### +build-linux: .info .prepare-builder .cp-linux + docker run -v $(shell pwd)/:/proxy proxy-linux-builder /proxy/pkg/build.sh ${VERSION} ${REVISION} + +##### +# Push rep & deb packages +##### +push-linux: .info .prepare-builder + docker run -v $(shell pwd)/:/proxy proxy-linux-builder /proxy/pkg/upload_to_packagecloud.sh ${PACKAGECLOUD_USER}/${PACKAGECLOUD_REPO} /proxy/pkg/package_cloud.conf /proxy/out + +##### +# Package for Macos +##### +pack-macos: + cp ${out}/${ARTIFACT_ID}-${VERSION}-spring-boot.jar macos/wavefront-proxy.jar + cd macos && zip ${out}/wfproxy_macos_${VERSION}_${REVISION}.zip * + unzip -t ${out}/wfproxy_macos_${VERSION}_${REVISION}.zip + + +##### +# Run Proxy complex Tests +##### +tests: .info .cp-docker + $(MAKE) -C tests/chain-checking all + +.prepare-builder: + docker build -t proxy-linux-builder pkg/ + +.cp-docker: + cp ${out}/${ARTIFACT_ID}-${VERSION}-spring-boot.jar docker/wavefront-proxy.jar + ${MAKE} .set_package JAR=docker/wavefront-proxy.jar PKG=docker + +.cp-linux: + cp ${out}/${ARTIFACT_ID}-${VERSION}-spring-boot.jar pkg/wavefront-proxy.jar + # ${MAKE} .set_package JAR=pkg/wavefront-proxy.jar PKG=linux_rpm_deb + +clean: + docker buildx prune -a -f + +.set_package: + jar -xvf ${JAR} BOOT-INF/classes/build.properties + sed 's/\(build.package=\).*/\1${PKG}/' BOOT-INF/classes/build.properties > build.tmp && mv build.tmp BOOT-INF/classes/build.properties + jar -uvf ${JAR} BOOT-INF/classes/build.properties + rm BOOT-INF/classes/build.properties diff --git a/README.md b/README.md index 9fc6b74ec..0916a5e47 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,46 @@ -# Wavefront Java Top-Level Project [![Build Status](https://travis-ci.org/wavefrontHQ/java.svg?branch=master)](https://travis-ci.org/wavefrontHQ/java) +> [!WARNING] +> Wavefront Proxy 14.0 and later no longer use the public `java-lib` implementation. +> This repository is maintained in a **read-only** state and is provided for **reference purposes only**. + +# Security Advisories + +Wavefront proxy version 10.10 and earlier is impacted by a Log4j vulnerability — [CVE-2021-44228](https://github.com/advisories/GHSA-jfh8-c2jp-5v3q). VMware Security Advisory (VMSA): CVE-2021-44228 – [VMSA-2021-0028](https://blogs.vmware.com/security/2021/12/vmsa-2021-0028-log4j-what-you-need-to-know.html) discusses this vulnerability and its impact on VMware products. + +### Patches + +Wavefront proxy version 10.11 and later use a version of Log4j that addresses this vulnerability. + +----- + +# Wavefront Proxy Project [Wavefront](https://docs.wavefront.com/) is a high-performance streaming analytics platform for monitoring and optimizing your environment and applications. -This repository contains several independent Java projects for sending metrics to Wavefront. +The [Wavefront Proxy](https://docs.wavefront.com/proxies.html) is a light-weight Java application that you send your metrics, histograms, logs, and trace data to. It handles batching and transmission of your data to the Wavefront service in a secure, fast, and reliable manner. ## Requirements - * Java >= 1.8 - * Maven + +* Java 8, 9, 10 or 11 (11 recommended) +* Maven ## Overview - * dropwizard-metrics: Wavefront reporter for [DropWizard Metrics](https://metrics.dropwizard.io). - * java-client: Libraries for sending metrics to Wavefront via proxy or direct ingestion. - * java-lib: Common set of Wavefront libraries used by the other java projects. - * pkg: Build and runtime packaging for the Wavefront proxy. - * proxy: [Wavefront Proxy](https://docs.wavefront.com/proxies.html) source code. - * yammer-metrics: Wavefront reporter for Yammer Metrics (predecessor to DropWizard metrics). - * examples: Sample code leveraging the libraries in this repository - Refer the documentation under each project for further details. +* pkg: Build and runtime packaging for the Wavefront proxy. +* proxy: [Wavefront Proxy](https://docs.wavefront.com/proxies.html) source code. + +Please refer to the [project page](https://github.com/wavefrontHQ/wavefront-proxy/tree/master/proxy) for further details. ## To start developing -``` -$ git clone github.com/wavefronthq/java ${directory} -$ cd ${directory} -$ mvn clean install -DskipTests +```bash +git clone https://github.com/wavefronthq/wavefront-proxy +cd wavefront-proxy +mvn -f proxy clean install -DskipTests + +If you want to skip code formatting during build use: +mvn -f proxy clean install -DskipTests -DskipFormatCode ``` ## Contributing + Public contributions are always welcome. Please feel free to report issues or submit pull requests. diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..57806972d --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,45 @@ +FROM eclipse-temurin:11 + +# Wavefront authentication can be configured in three different ways: Customers that have been +# onboarded by CSP can set up CSP api tokens or CSP OAuth apps (CSP_APP_ID, CSP_APP_SECRET). +# Customers of Wavefront can use Wavefront api token. This script may automatically +# configure Wavefront without prompting, based on these variables: +# WAVEFRONT_URL (required) +# WAVEFRONT_TOKEN (not required) +# CSP_API_TOKEN (not required) +# CSP_APP_ID (not required) +# CSP_APP_SECRET (not required) +# CSP_ORG_ID (not required) +# JAVA_HEAP_USAGE (default is 4G) +# WAVEFRONT_HOSTNAME (default is the docker containers hostname) +# WAVEFRONT_PROXY_ARGS (default is none) +# JAVA_ARGS (default is none) + +# Add new group:user "wavefront" +RUN groupadd -g 2000 wavefront +RUN useradd --uid 1001 --gid 2000 -m wavefront +RUN chown -R wavefront:wavefront /opt/java/openjdk/lib/security/cacerts +RUN mkdir -p /var/spool/wavefront-proxy +RUN chown -R wavefront:wavefront /var/spool/wavefront-proxy + +RUN mkdir -p /var/log/wavefront +RUN chown -R wavefront:wavefront /var/log/wavefront + +RUN apt-get update -y && apt-get upgrade -y + +# Temp fix for "MONIT-41551" +RUN apt-get remove -y wget + +# Run the agent +EXPOSE 3878 +EXPOSE 2878 +EXPOSE 4242 + +USER wavefront:wavefront + +ADD wavefront-proxy.jar /opt/wavefront/wavefront-proxy/wavefront-proxy.jar +ADD run.sh /opt/wavefront/wavefront-proxy/run.sh +ADD log4j2.xml /etc/wavefront/wavefront-proxy/log4j2.xml +ADD LICENSE /licenses/LICENSE + +CMD ["/bin/bash", "/opt/wavefront/wavefront-proxy/run.sh"] diff --git a/docker/Dockerfile-rhel b/docker/Dockerfile-rhel new file mode 100644 index 000000000..9b04f5f7b --- /dev/null +++ b/docker/Dockerfile-rhel @@ -0,0 +1,57 @@ +# NOTE: we need this to be a Dockerfile because that's the only option +# when using the Automated Build Service for Red Hat Build Partners. +# see: https://connect.redhat.com/en/blog/automated-build-service-red-hat-build-partners + +FROM registry.access.redhat.com/ubi7 + +MAINTAINER wavefront@vmware.com + +LABEL name="Wavefront Collector" \ + vendor="Wavefront by VMware" \ + version="10.13" \ + release="10.13" \ + summary="The Wavefront Proxy is a light-weight Java application that you send your metrics, histograms, and trace data to. It handles batching and transmission of your data to the Wavefront service in a secure, fast, and reliable manner." \ + description="The Wavefront Proxy is a light-weight Java application that you send your metrics, histograms, and trace data to. It handles batching and transmission of your data to the Wavefront service in a secure, fast, and reliable manner." + +# Wavefront authentication can be configured in three different ways: Customers that have been +# onboarded by CSP can set up CSP api tokens or CSP OAuth apps (CSP_APP_ID, CSP_APP_SECRET). +# Customers of Wavefront can use Wavefront api token. This script may automatically +# configure Wavefront without prompting, based on these variables: +# WAVEFRONT_URL (required) +# WAVEFRONT_TOKEN (not required) +# CSP_API_TOKEN (not required) +# CSP_APP_ID (not required) +# CSP_APP_SECRET (not required) +# CSP_ORG_ID (not required) +# JAVA_HEAP_USAGE (default is 4G) +# WAVEFRONT_HOSTNAME (default is the docker containers hostname) +# WAVEFRONT_PROXY_ARGS (default is none) +# JAVA_ARGS (default is none) + +RUN yum-config-manager --enable rhel-7-server-optional-rpms \ + && yum update --disableplugin=subscription-manager -y \ + && rm -rf /var/cache/yum \ + && yum install -y hostname java-11-openjdk + +# Add new group:user "wavefront" +RUN groupadd -g 2000 wavefront +RUN useradd --uid 1000 --gid 2000 -m wavefront +RUN chmod a+w /usr/lib/jvm/jre/lib/security/cacerts +RUN mkdir -p /var/spool/wavefront-proxy +RUN chown -R wavefront:wavefront /var/spool/wavefront-proxy + +# Run the agent +EXPOSE 3878 +EXPOSE 2878 +EXPOSE 4242 + +USER wavefront:wavefront + +ADD wavefront-proxy.jar /opt/wavefront/wavefront-proxy/wavefront-proxy.jar +ADD run.sh /opt/wavefront/wavefront-proxy/run.sh +ADD log4j2.xml /etc/wavefront/wavefront-proxy/log4j2.xml +ADD LICENSE /licenses/LICENSE + +CMD ["/bin/bash", "/opt/wavefront/wavefront-proxy/run.sh"] + + diff --git a/docker/LICENSE b/docker/LICENSE new file mode 100644 index 000000000..ec524fb20 --- /dev/null +++ b/docker/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Wavefront, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/proxy/docker/README.md b/docker/README.md similarity index 56% rename from proxy/docker/README.md rename to docker/README.md index dce28eb75..068d35c3e 100644 --- a/proxy/docker/README.md +++ b/docker/README.md @@ -1,11 +1,47 @@ +## Build + + docker build -t wavefront-proxy . + +## run + The Proxy will accept Wavefront formatted message on port 2878 (additional listeners can be enabled in WAVEFRONT_PROXY_ARGS, see below). Just run this docker image with the following environment variables defined, e.g. - docker build -t wavefront-proxy . +#### WF Token + docker run \ -e WAVEFRONT_URL=https://you.wavefront.com/api \ -e WAVEFRONT_TOKEN= \ -p 2878:2878 \ wavefront-proxy +#### CSP App ID and App Secret + + docker run -d \ + -e WAVEFRONT_URL=https://you.wavefront.com/api/ \ + -e CSP_APP_ID= \ + -e CSP_APP_SECRET= \ + -p 2878:2878 \ + wavefront-proxy + +#### CSP App ID, App Secret and ORG ID + + docker run -d \ + -e WAVEFRONT_URL=https://you.wavefront.com/api/ \ + -e CSP_APP_ID= \ + -e CSP_APP_SECRET= \ + -e CSP_ORG_ID= \ + -p 2878:2878 \ + wavefront-proxy + +#### CSP Api Token + + docker run -d \ + -e WAVEFRONT_URL=https://you.wavefront.com/api/ \ + -e CSP_API_TOKEN= \ + -p 2878:2878 \ + wavefront-proxy + +## Configuration + All properties that exist in [wavefront.conf](https://github.com/wavefrontHQ/java/blob/master/pkg/etc/wavefront/wavefront-proxy/wavefront.conf.default) can be customized by passing their name as long form arguments within your docker run command in the WAVEFRONT_PROXY_ARGS environment variable. For example, add `-e WAVEFRONT_PROXY_ARGS="--pushRateLimit 1000"` to your docker run command to specify a [rate limit](https://github.com/wavefrontHQ/java/blob/master/pkg/etc/wavefront/wavefront-proxy/wavefront.conf.default#L62) of 1000 pps for the proxy. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 000000000..624fdeb55 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,29 @@ +services: + proxy-1: + build: . + environment: + WAVEFRONT_URL: ${WF_URL} + WAVEFRONT_TOKEN: ${WF_TOKEN} + CSP_API_TOKEN: ${CSP_API_TOKEN} + CSP_APP_ID: ${CSP_APP_ID} + CSP_APP_SECRET: ${CSP_APP_SECRET} + CSP_ORG_ID: ${CSP_ORG_ID} + WAVEFRONT_PROXY_ARGS: --ephemeral false --idFile /var/spool/wavefront-proxy/id-1 + volumes: + - /Users/glaullon/tmp:/var/spool/wavefront-proxy + ports: + - "2878:2878" + proxy-2: + build: . + environment: + WAVEFRONT_URL: ${WF_URL} + WAVEFRONT_TOKEN: ${WF_TOKEN} + CSP_API_TOKEN: ${CSP_API_TOKEN} + CSP_APP_ID: ${CSP_APP_ID} + CSP_APP_SECRET: ${CSP_APP_SECRET} + CSP_ORG_ID: ${CSP_ORG_ID} + WAVEFRONT_PROXY_ARGS: --ephemeral false --idFile /var/spool/wavefront-proxy/id-2 + volumes: + - /Users/glaullon/tmp:/var/spool/wavefront-proxy + ports: + - "2879:2878" diff --git a/docker/log4j2.xml b/docker/log4j2.xml new file mode 100644 index 000000000..5d359824f --- /dev/null +++ b/docker/log4j2.xml @@ -0,0 +1,73 @@ + + + + /var/log/wavefront + + + + + + %d %-5level [%c{1}:%M] %m%n + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/run.sh b/docker/run.sh new file mode 100644 index 000000000..cde459cbf --- /dev/null +++ b/docker/run.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +if [[ -z "$WAVEFRONT_URL" ]]; then + echo "WAVEFRONT_URL environment variable not configured - aborting startup " >&2 + exit 0 +fi + +authType="" +if [[ -n "$CSP_APP_ID" && -n "$CSP_APP_SECRET" ]]; then + if [[ -n "$CSP_ORG_ID" ]]; then + authType="--cspAppId $CSP_APP_ID --cspAppSecret $CSP_APP_SECRET --cspOrgId $CSP_ORG_ID" + else + authType="--cspAppId $CSP_APP_ID --cspAppSecret $CSP_APP_SECRET" + fi +fi +if [[ -n "$CSP_API_TOKEN" ]]; then + authType="--cspAPIToken $CSP_API_TOKEN" +fi +if [[ -n "$WAVEFRONT_TOKEN" ]]; then + authType="-t $WAVEFRONT_TOKEN" +fi + +if [[ -z "$authType" ]]; then + echo "Error: The auth method combination was wrong or no auth method was supplied." + exit 1 +fi + +spool_dir="/var/spool/wavefront-proxy" +mkdir -p $spool_dir + +chown -R wavefront:wavefront $spool_dir + +# Be receptive to core dumps +ulimit -c unlimited + +# Allow high connection count per process (raise file descriptor limit) +ulimit -Sn 65536 +ulimit -Hn 65536 + +java_heap_usage=${JAVA_HEAP_USAGE:-4G} +jvm_initial_ram_percentage=${JVM_INITIAL_RAM_PERCENTAGE:-50.0} +jvm_max_ram_percentage=${JVM_MAX_RAM_PERCENTAGE:-85.0} + +# Use cgroup opts - Note that -XX:UseContainerSupport=true since Java 8u191. +# https://bugs.openjdk.java.net/browse/JDK-8146115 +jvm_container_opts="-XX:InitialRAMPercentage=$jvm_initial_ram_percentage -XX:MaxRAMPercentage=$jvm_max_ram_percentage" +if [ "${JVM_USE_CONTAINER_OPTS}" = false ] ; then + jvm_container_opts="-Xmx$java_heap_usage -Xms$java_heap_usage" +fi + +################### +# import CA certs # +################### +if [ -d "/tmp/ca/" ]; then + files=$(ls /tmp/ca/*.pem) + echo + echo "Adding credentials to JVM store.." + echo + for filename in ${files}; do + alias=$(basename ${filename}) + alias=${alias%.*} + echo "----------- Adding credential file:${filename} alias:${alias}" + keytool -noprompt -cacerts -importcert -storepass changeit -file ${filename} -alias ${alias} + keytool -storepass changeit -list -v -cacerts -alias ${alias} + echo "----------- Done" + echo + done +fi + +############# +# run proxy # +############# +java \ + $jvm_container_opts $JAVA_ARGS \ + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \ + -Dlog4j.configurationFile=/etc/wavefront/wavefront-proxy/log4j2.xml \ + -jar /opt/wavefront/wavefront-proxy/wavefront-proxy.jar \ + -h $WAVEFRONT_URL \ + $authType \ + --ephemeral true \ + --buffer ${spool_dir}/buffer \ + --flushThreads 6 \ + $WAVEFRONT_PROXY_ARGS \ No newline at end of file diff --git a/docker/win-proxy-installer.bat b/docker/win-proxy-installer.bat new file mode 100644 index 000000000..8c0f91cdd --- /dev/null +++ b/docker/win-proxy-installer.bat @@ -0,0 +1,3 @@ +@echo off + +powershell.exe -Command $ErrorActionPreference = 'Stop'; (New-Object System.Net.WebClient).DownloadFile('https://s3-us-west-2.amazonaws.com/wavefront-cdn/windows/wavefront-proxy-setup.exe', 'C:\wavefront-proxy-setup.exe') ; Start-Process C:\wavefront-proxy-setup.exe -ArgumentList @('/server="%WAVEFRONT_URL%" /token="%WAVEFRONT_TOKEN%" /VERYSILENT /SUPPRESSMSGBOXES') -Wait ; Remove-Item C:\wavefront-proxy-setup.exe -Force ; Start-Sleep -s 10 ; $ProgFilePath=${Env:ProgramFiles(x86)} ; Get-Content -Path ${ProgFilePath}\Wavefront\logs\wavefront.log -Wait diff --git a/dropwizard-metrics/README.md b/dropwizard-metrics/README.md deleted file mode 100644 index 9d3de72f2..000000000 --- a/dropwizard-metrics/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Dropwizard Metrics for Wavefront - -Wavefront maintains reporter plugins for Dropwizard Metrics. Source code and more information about each reporter -can be found within these subdirectories: - -- [Dropwizard Metrics](/dropwizard-metrics/dropwizard-metrics) -- [Dropwizard](/dropwizard-metrics/dropwizard-metrics-wavefront) - -The Dropwizard Metrics library is intended for sending metrics from Dropwizard applications that are *not* running -within the Dropwizard Microservice framework. The Dropwizard library, on the other hand, is designed to be linked into -Dropwizard applications that then allows them to be configured with a Wavefront metrics reporter in their YAML -configuration. \ No newline at end of file diff --git a/dropwizard-metrics/dropwizard-metrics-wavefront/README.md b/dropwizard-metrics/dropwizard-metrics-wavefront/README.md deleted file mode 100644 index cc5211182..000000000 --- a/dropwizard-metrics/dropwizard-metrics-wavefront/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Wavefront Reporter - -This is a plug-in Wavefront Reporter for 1.2.x version of [Dropwizard](https://github.com/dropwizard/dropwizard). - -Adding a dependency to your Dropwizard project gives you a new metric reporter type `wavefront` (similar to `graphite`) that can be configured entirely in the application's YAML config file without writing a single line of code. - -## Usage - -This Reporter sends data to Wavefront via a proxy. You can easily install the proxy by following [these instructions](https://docs.wavefront.com/proxies_installing.html). - -To use the Reporter you'll need to know the hostname and port (which by default is localhost:2878) where the Wavefront proxy is running. - -It is designed to be used with the [stable version 1.2.x of Dropwizard](https://github.com/dropwizard/dropwizard). - -### Setting up Maven - -You will also need `org.slf4j` for logging: - -```Maven - - com.wavefront - dropwizard-metrics-wavefront - [LATEST VERSION] - - - org.slf4j - slf4j-simple - 1.7.16 - -``` - -### Example Usage - -The Wavefront Reporter lets you use DropWizard metrics exactly as you normally would. See its [getting started guide](https://dropwizard.github.io/metrics/3.1.0/getting-started/) if you haven't used it before. - -It simply gives you a new Reporter that will seamlessly work with Wavefront. For instance, to create a Reporter which will emit data every 30 seconds to: - -- A Wavefront proxy on `localhost` at port `2878` -- Data that should appear in Wavefront under `source=app-1.company.com` -- Two point tags named `dc` and `service` - -you would add something like this to your application's YAML config file: - -```yaml -metrics: - reporters: - - type: wavefront - proxyHost: localhost - proxyPort: 2878 - metricSource: app1.company.com - prefix: dropwizard - frequency: 30s - pointTags: - dc: dallas - service: query -``` - -If you don't specify `metricSource`, it will try to do a reverse DNS lookup for the local host and use the DNS name. - diff --git a/dropwizard-metrics/dropwizard-metrics-wavefront/pom.xml b/dropwizard-metrics/dropwizard-metrics-wavefront/pom.xml deleted file mode 100644 index 5c2159e9a..000000000 --- a/dropwizard-metrics/dropwizard-metrics-wavefront/pom.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - 4.0.0 - - - com.wavefront - wavefront - 4.35-SNAPSHOT - ../../pom.xml - - - dropwizard-metrics-wavefront - Wavefront Dropwizard Reporter - - - 1.3.5 - - - - - - io.dropwizard - dropwizard-bom - ${dropwizard.version} - pom - import - - - - - - - io.dropwizard - dropwizard-metrics - - - org.eclipse.jetty - jetty-server - - - - - com.wavefront - dropwizard-metrics - - - io.dropwizard - dropwizard-configuration - test - - - org.apache.commons - commons-lang3 - test - - - diff --git a/dropwizard-metrics/dropwizard-metrics-wavefront/src/main/java/com/wavefront/integrations/metrics/WavefrontReporterFactory.java b/dropwizard-metrics/dropwizard-metrics-wavefront/src/main/java/com/wavefront/integrations/metrics/WavefrontReporterFactory.java deleted file mode 100644 index 497ad54e3..000000000 --- a/dropwizard-metrics/dropwizard-metrics-wavefront/src/main/java/com/wavefront/integrations/metrics/WavefrontReporterFactory.java +++ /dev/null @@ -1,146 +0,0 @@ -package com.wavefront.integrations.metrics; - -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.ScheduledReporter; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; -import io.dropwizard.metrics.BaseReporterFactory; -import io.dropwizard.validation.PortRange; -import org.hibernate.validator.constraints.NotEmpty; - -import javax.validation.constraints.NotNull; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Collections; -import java.util.Map; - -/** - * A factory for {@link WavefrontReporter} instances. - *

- * Configuration Parameters: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
NameDefaultDescription
proxyHostlocalhostThe hostname of the Wavefront proxy to report to.
proxyPort2003The port of the Wavefront proxy to report to.
prefixNoneThe prefix for Metric key names to report to Wavefront.
metricSourcecanonical name of localhostThis is the source name all metrics will report to Wavefront under.
pointTagsNoneKey-value pairs for point tags to be added to each point reporting to Wavefront.
- */ -@JsonTypeName("wavefront") -public class WavefrontReporterFactory extends BaseReporterFactory { - @NotEmpty - private String proxyHost = "localhost"; - - @PortRange - private int proxyPort = 2878; - - @NotNull - private String prefix = ""; - - @NotNull - private String metricSource = ""; - - @NotNull - private Map pointTags = Collections.emptyMap(); - - @JsonProperty - public String getProxyHost() { - return proxyHost; - } - - @JsonProperty - public void setProxyHost(String proxyHost) { - this.proxyHost = proxyHost; - } - - @JsonProperty - public int getProxyPort() { - return proxyPort; - } - - @JsonProperty - public void setProxyPort(int proxyPort) { - this.proxyPort = proxyPort; - } - - @JsonProperty - public String getPrefix() { - return prefix; - } - - @JsonProperty - public void setPrefix(String prefix) { - this.prefix = prefix; - } - - @JsonProperty - public String getMetricSource() { - return metricSource; - } - - @JsonProperty - public void setMetricSource(String metricSource) { - this.metricSource = metricSource; - } - - @JsonProperty("pointTags") - public Map getPointTags() { - return pointTags; - } - - @JsonProperty("pointTags") - public void setPointTags(Map pointTags) { - this.pointTags = ImmutableMap.copyOf(pointTags); - } - - - @Override - public ScheduledReporter build(MetricRegistry registry) { - return WavefrontReporter.forRegistry(registry) - .convertDurationsTo(getDurationUnit()) - .convertRatesTo(getRateUnit()) - .filter(getFilter()) - .prefixedWith(getPrefix()) - .withSource(getSource()) - .disabledMetricAttributes(getDisabledAttributes()) - .withPointTags(pointTags) - .build(proxyHost, proxyPort); - } - - private String getSource() { - try { - return Strings.isNullOrEmpty(getMetricSource()) ? - InetAddress.getLocalHost().getCanonicalHostName() : - getMetricSource(); - } catch (UnknownHostException ex) { - return "localhost"; - } - } -} diff --git a/dropwizard-metrics/dropwizard-metrics-wavefront/src/main/resources/META-INF/services/io.dropwizard.metrics.ReporterFactory b/dropwizard-metrics/dropwizard-metrics-wavefront/src/main/resources/META-INF/services/io.dropwizard.metrics.ReporterFactory deleted file mode 100644 index 1bc95098d..000000000 --- a/dropwizard-metrics/dropwizard-metrics-wavefront/src/main/resources/META-INF/services/io.dropwizard.metrics.ReporterFactory +++ /dev/null @@ -1 +0,0 @@ -com.wavefront.integrations.metrics.WavefrontReporterFactory diff --git a/dropwizard-metrics/dropwizard-metrics/README.md b/dropwizard-metrics/dropwizard-metrics/README.md deleted file mode 100644 index 6e8709586..000000000 --- a/dropwizard-metrics/dropwizard-metrics/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# Wavefront Reporter - -This is a Wavefront Reporter compatible with versions 3.1.x, 3.2.x and 4.0.x of [Dropwizard Metrics](https://metrics.dropwizard.io/) (formerly Coda Hale & Yammer Metrics). - -It sends data to the Wavefront service using the [Wavefront proxy](https://docs.wavefront.com/proxies.html) or using [direct ingestion](https://docs.wavefront.com/direct_ingestion.html) and supports point tags being assigned at the Reporter level. - -## Usage - -The Wavefront Reporter lets you use Dropwizard metrics exactly as you normally would. See its [getting started guide](https://dropwizard.github.io/metrics/3.1.0/getting-started/) if you haven't used it before. - -It simply gives you a new Reporter that will work seamlessly with Wavefront. Set up Maven as given below, `import com.wavefront.integrations.metrics.WavefrontReporter;` and then report to a Wavefront proxy or directly to a Wavefront server. - -### Setting up Maven - -You will need both the DropWizard `metrics-core` and the Wavefront `dropwizard-metrics` libraries as dependencies. Logging depends on `org.slf4j`: - -```Maven - - io.dropwizard.metrics - metrics-core - 3.2.5 - - - com.wavefront - dropwizard-metrics - [LATEST VERSION] - - - org.slf4j - slf4j-simple - 1.7.16 - -``` - -Versions `3.1.x`, `3.2.x` and `4.0.x` of `metrics-core` will work. - -### Report to a Wavefront Proxy - -You can install the proxy by following [these instructions](https://docs.wavefront.com/proxies_installing.html). -To use the Reporter you'll need to provide the hostname and port (which by default is 2878) of the Wavefront proxy. - -For example to create a Reporter which will emit data to a Wavefront proxy every 5 seconds: - -```java -MetricRegistry registry = new MetricRegistry(); -Counter evictions = registry.counter("cache-evictions"); - -String hostname = "wavefront.proxy.hostname"; -int port = 2878; - -WavefrontReporter reporter = WavefrontReporter.forRegistry(registry). - withSource("app-1.company.com"). - withPointTag("dc", "us-west-2"). - withPointTag("service", "query"). - build(hostname, port); -reporter.start(5, TimeUnit.SECONDS); -``` - -### Report to a Wavefront Server - -You can send metrics directly to a Wavefront service. To use the Reporter you'll need to provide the Wavefront server URL and a token with direct data ingestion permission. - -For example to create a Reporter which will emit data to a Wavefront server every 5 seconds: - -```java -MetricRegistry registry = new MetricRegistry(); -Counter evictions = registry.counter("cache-evictions"); - -String server = "https://.wavefront.com"; -String token = ""; - -WavefrontReporter reporter = WavefrontReporter.forRegistry(registry). - withSource("app-1.company.com"). - withPointTag("dc", "us-west-2"). - withPointTag("service", "query"). - buildDirect(server, token); -reporter.start(5, TimeUnit.SECONDS); -``` - -### Extended Usage - -The Reporter provides all the same options that the [GraphiteReporter](http://metrics.dropwizard.io/3.1.0/manual/graphite/) does. By default: - -- There is no prefix on the Metrics -- Rates will be converted to Seconds -- Durations will be converted to Milliseconds -- `MetricFilter.ALL` will be used for the Filter -- `Clock.defaultClock()` will be used for the Clock - -In addition you can also: - -- Supply point tags for the Reporter to use. There are two ways to specify point tags at the Reporter level, individually using `.withPointTag(String tagK, String tagV)` or create a `Map` and call `.withPointTags(my-map)` to do many at once. -- Call `.withJvmMetrics()` when building the Reporter if you want it to add some default JVM metrics to the given MetricsRegistry - -If `.withJvmMetrics()` is used the following metrics will be added to the registry: - -```java -registry.register("jvm.uptime", new Gauge() { - @Override - public Long getValue() { - return ManagementFactory.getRuntimeMXBean().getUptime(); - } -}); -registry.register("jvm.current_time", new Gauge() { - @Override - public Long getValue() { - return clock.getTime(); - } -}); - -registry.register("jvm.classes", new ClassLoadingGaugeSet()); -registry.register("jvm.fd_usage", new FileDescriptorRatioGauge()); -registry.register("jvm.buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer())); -registry.register("jvm.gc", new GarbageCollectorMetricSet()); -registry.register("jvm.memory", new MemoryUsageGaugeSet()); -``` diff --git a/dropwizard-metrics/dropwizard-metrics/pom.xml b/dropwizard-metrics/dropwizard-metrics/pom.xml deleted file mode 100644 index 690f42831..000000000 --- a/dropwizard-metrics/dropwizard-metrics/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - 4.0.0 - - - com.wavefront - wavefront - 4.35-SNAPSHOT - ../../pom.xml - - - dropwizard-metrics - 4.35-SNAPSHOT - Wavefront Dropwizard Metrics Reporter - Report metrics via the Wavefront Proxy - - - - com.wavefront - java-client - 4.35-SNAPSHOT - - - io.dropwizard.metrics - metrics-core - 4.0.2 - - - io.dropwizard.metrics - metrics-jvm - 4.0.2 - - - org.json - json - 20160212 - - - - diff --git a/dropwizard-metrics/dropwizard-metrics/src/main/java/com/wavefront/integrations/metrics/WavefrontReporter.java b/dropwizard-metrics/dropwizard-metrics/src/main/java/com/wavefront/integrations/metrics/WavefrontReporter.java deleted file mode 100644 index 7bea452c9..000000000 --- a/dropwizard-metrics/dropwizard-metrics/src/main/java/com/wavefront/integrations/metrics/WavefrontReporter.java +++ /dev/null @@ -1,547 +0,0 @@ -package com.wavefront.integrations.metrics; - -import com.google.common.base.Preconditions; - -import com.codahale.metrics.Clock; -import com.codahale.metrics.Counter; -import com.codahale.metrics.DeltaCounter; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Meter; -import com.codahale.metrics.Metered; -import com.codahale.metrics.MetricAttribute; -import com.codahale.metrics.MetricFilter; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.ScheduledReporter; -import com.codahale.metrics.Snapshot; -import com.codahale.metrics.Timer; -import com.codahale.metrics.jvm.BufferPoolMetricSet; -import com.codahale.metrics.jvm.ClassLoadingGaugeSet; -import com.codahale.metrics.jvm.GarbageCollectorMetricSet; -import com.codahale.metrics.jvm.MemoryUsageGaugeSet; -import com.codahale.metrics.jvm.SafeFileDescriptorRatioGauge; -import com.codahale.metrics.jvm.ThreadStatesGaugeSet; -import com.wavefront.common.MetricConstants; -import com.wavefront.integrations.Wavefront; -import com.wavefront.integrations.WavefrontDirectSender; -import com.wavefront.integrations.WavefrontSender; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; - -import javax.validation.constraints.NotNull; - -/** - * A reporter which publishes metric values to a Wavefront Proxy from a Dropwizard {@link MetricRegistry}. - */ -public class WavefrontReporter extends ScheduledReporter { - - private static final Logger LOGGER = LoggerFactory.getLogger(WavefrontReporter.class); - - /** - * Returns a new {@link Builder} for {@link WavefrontReporter}. - * - * @param registry the registry to report - * @return a {@link Builder} instance for a {@link WavefrontReporter} - */ - public static Builder forRegistry(MetricRegistry registry) { - return new Builder(registry); - } - - /** - * A builder for {@link WavefrontReporter} instances. Defaults to not using a prefix, using the - * default clock, converting rates to events/second, converting durations to milliseconds, a host - * named "unknown", no point Tags, and not filtering any metrics. - */ - public static class Builder { - private final MetricRegistry registry; - private Clock clock; - private String prefix; - private TimeUnit rateUnit; - private TimeUnit durationUnit; - private MetricFilter filter; - private String source; - private Map pointTags; - private boolean includeJvmMetrics; - private Set disabledMetricAttributes; - - private Builder(MetricRegistry registry) { - this.registry = registry; - this.clock = Clock.defaultClock(); - this.prefix = null; - this.rateUnit = TimeUnit.SECONDS; - this.durationUnit = TimeUnit.MILLISECONDS; - this.filter = MetricFilter.ALL; - this.source = "dropwizard-metrics"; - this.pointTags = new HashMap<>(); - this.includeJvmMetrics = false; - this.disabledMetricAttributes = Collections.emptySet(); - } - - /** - * Use the given {@link Clock} instance for the time. Defaults to Clock.defaultClock() - * - * @param clock a {@link Clock} instance - * @return {@code this} - */ - public Builder withClock(Clock clock) { - this.clock = clock; - return this; - } - - /** - * Prefix all metric names with the given string. Defaults to null. - * - * @param prefix the prefix for all metric names - * @return {@code this} - */ - public Builder prefixedWith(String prefix) { - this.prefix = prefix; - return this; - } - - /** - * Set the host for this reporter. This is equivalent to withSource. - * - * @param host the host for all metrics - * @return {@code this} - */ - public Builder withHost(String host) { - this.source = host; - return this; - } - - /** - * Set the source for this reporter. This is equivalent to withHost. - * - * @param source the host for all metrics - * @return {@code this} - */ - public Builder withSource(String source) { - this.source = source; - return this; - } - - /** - * Set the Point Tags for this reporter. - * - * @param pointTags the pointTags Map for all metrics - * @return {@code this} - */ - public Builder withPointTags(Map pointTags) { - this.pointTags.putAll(pointTags); - return this; - } - - /** - * Set a point tag for this reporter. - * - * @param ptagK the key of the Point Tag - * @param ptagV the value of the Point Tag - * @return {@code this} - */ - public Builder withPointTag(String ptagK, String ptagV) { - this.pointTags.put(ptagK, ptagV); - return this; - } - - /** - * Convert rates to the given time unit. Defaults to Seconds. - * - * @param rateUnit a unit of time - * @return {@code this} - */ - public Builder convertRatesTo(TimeUnit rateUnit) { - this.rateUnit = rateUnit; - return this; - } - - /** - * Convert durations to the given time unit. Defaults to Milliseconds. - * - * @param durationUnit a unit of time - * @return {@code this} - */ - public Builder convertDurationsTo(TimeUnit durationUnit) { - this.durationUnit = durationUnit; - return this; - } - - /** - * Only report metrics which match the given filter. Defaults to MetricFilter.ALL - * - * @param filter a {@link MetricFilter} - * @return {@code this} - */ - public Builder filter(MetricFilter filter) { - this.filter = filter; - return this; - } - - /** - * Don't report the passed metric attributes for all metrics (e.g. "p999", "stddev" or "m15"). - * See {@link MetricAttribute}. - * - * @param disabledMetricAttributes a set of {@link MetricAttribute} - * @return {@code this} - */ - public Builder disabledMetricAttributes(Set disabledMetricAttributes) { - this.disabledMetricAttributes = disabledMetricAttributes; - return this; - } - - /** - * Include JVM Metrics from this Reporter. - * - * @return {@code this} - */ - public Builder withJvmMetrics() { - this.includeJvmMetrics = true; - return this; - } - - /** - * Builds a {@link WavefrontReporter} from the VCAP_SERVICES env variable, sending metrics - * using the given {@link WavefrontSender}. This should be used in PCF environment only. It - * uses 'wavefront-proxy' as the name to fetch the proxy details from VCAP_SERVICES. - * - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter bindToCloudFoundryService() { - return bindToCloudFoundryService("wavefront-proxy", false); - } - - /** - * Builds a {@link WavefrontReporter} from the VCAP_SERVICES env variable, sending metrics - * using the given {@link WavefrontSender}. This should be used in PCF environment only. It - * assumes failOnError to be false. - * - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter bindToCloudFoundryService(@NotNull String proxyServiceName) { - return bindToCloudFoundryService(proxyServiceName, false); - } - - /** - * Builds a {@link WavefrontReporter} from the VCAP_SERVICES env variable, sending metrics - * using the given {@link WavefrontSender}. This should be used in PCF environment only. - * - * @param proxyServiceName The name of the wavefront proxy service. If wavefront-tile is used to - * deploy the proxy, then the service name will be 'wavefront-proxy'. - * @param failOnError A flag to determine what to do if the service parameters are not - * available. If 'true' then the method will throw RuntimeException else - * it will log an error message and continue. - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter bindToCloudFoundryService(@NotNull String proxyServiceName, - boolean failOnError) { - - Preconditions.checkNotNull(proxyServiceName, "proxyServiceName arg should not be null"); - - String proxyHostname; - int proxyPort; - // read the env variable VCAP_SERVICES - String services = System.getenv("VCAP_SERVICES"); - if (services == null || services.length() == 0) { - if (failOnError) { - throw new RuntimeException("VCAP_SERVICES environment variable is unavailable."); - } else { - LOGGER.error("Environment variable VCAP_SERVICES is empty. No metrics will be reported " + - "to wavefront proxy."); - // since the wavefront-proxy is not tied to the app, use dummy hostname and port. - proxyHostname = ""; - proxyPort = 2878; - } - } else { - // parse the json to read the hostname and port - JSONObject json = new JSONObject(services); - // When wavefront tile is installed on PCF, it will be automatically named wavefront-proxy - JSONArray jsonArray = json.getJSONArray(proxyServiceName); - if (jsonArray == null || jsonArray.isNull(0)) { - if (failOnError) { - throw new RuntimeException(proxyServiceName + " is not present in the VCAP_SERVICES " + - "env variable. Please verify and provide the wavefront proxy service name."); - } else { - LOGGER.error(proxyServiceName + " is not present in VCAP_SERVICES env variable. No " + - "metrics will be reported to wavefront proxy."); - // since the wavefront-proxy is not tied to the app, use dummy hostname and port. - proxyHostname = ""; - proxyPort = 2878; - } - } else { - JSONObject details = jsonArray.getJSONObject(0); - JSONObject credentials = details.getJSONObject("credentials"); - proxyHostname = credentials.getString("hostname"); - proxyPort = credentials.getInt("port"); - } - } - return new WavefrontReporter(registry, - proxyHostname, - proxyPort, - clock, - prefix, - source, - pointTags, - rateUnit, - durationUnit, - filter, - includeJvmMetrics, - disabledMetricAttributes); - } - - /** - * Builds a {@link WavefrontReporter} with the given properties, sending metrics directly - * to a given Wavefront server using direct ingestion APIs. - * - * @param server Wavefront server hostname of the form "https://serverName.wavefront.com" - * @param token Wavefront API token with direct ingestion permission - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter buildDirect(String server, String token) { - WavefrontSender wavefrontSender = new WavefrontDirectSender(server, token); - return new WavefrontReporter(registry, wavefrontSender, clock, prefix, source, pointTags, rateUnit, - durationUnit, filter, includeJvmMetrics, disabledMetricAttributes); - } - - /** - * Builds a {@link WavefrontReporter} with the given properties, sending metrics using the given - * {@link WavefrontSender}. - * - * @param proxyHostname Wavefront Proxy hostname. - * @param proxyPort Wavefront Proxy port. - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter build(String proxyHostname, int proxyPort) { - return new WavefrontReporter(registry, - proxyHostname, - proxyPort, - clock, - prefix, - source, - pointTags, - rateUnit, - durationUnit, - filter, - includeJvmMetrics, - disabledMetricAttributes); - } - - /** - * Builds a {@link WavefrontReporter} with the given properties, sending metrics using the given - * {@link WavefrontSender}. - * - * @param Wavefront a {@link WavefrontSender}. - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter build(WavefrontSender wavefrontSender) { - return new WavefrontReporter(registry, - wavefrontSender, - clock, - prefix, - source, - pointTags, - rateUnit, - durationUnit, - filter, - includeJvmMetrics, - disabledMetricAttributes); - } -} - - private final WavefrontSender wavefront; - private final Clock clock; - private final String prefix; - private final String source; - private final Map pointTags; - - private WavefrontReporter(MetricRegistry registry, - WavefrontSender wavefrontSender, - final Clock clock, - String prefix, - String source, - Map pointTags, - TimeUnit rateUnit, - TimeUnit durationUnit, - MetricFilter filter, - boolean includeJvmMetrics, - Set disabledMetricAttributes) { - super(registry, "wavefront-reporter", filter, rateUnit, durationUnit, Executors.newSingleThreadScheduledExecutor(), - true, disabledMetricAttributes == null ? Collections.emptySet() : disabledMetricAttributes); - this.wavefront = wavefrontSender; - this.clock = clock; - this.prefix = prefix; - this.source = source; - this.pointTags = pointTags; - - if (includeJvmMetrics) { - registry.register("jvm.uptime", (Gauge) () -> ManagementFactory.getRuntimeMXBean().getUptime()); - registry.register("jvm.current_time", (Gauge) clock::getTime); - registry.register("jvm.classes", new ClassLoadingGaugeSet()); - registry.register("jvm.fd_usage", new SafeFileDescriptorRatioGauge()); - registry.register("jvm.buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer())); - registry.register("jvm.gc", new GarbageCollectorMetricSet()); - registry.register("jvm.memory", new MemoryUsageGaugeSet()); - registry.register("jvm.thread-states", new ThreadStatesGaugeSet()); - } - } - - private WavefrontReporter(MetricRegistry registry, - String proxyHostname, - int proxyPort, - final Clock clock, - String prefix, - String source, - Map pointTags, - TimeUnit rateUnit, - TimeUnit durationUnit, - MetricFilter filter, - boolean includeJvmMetrics, - Set disabledMetricAttributes) { - this(registry, new Wavefront(proxyHostname, proxyPort), clock, prefix, source, pointTags, rateUnit, - durationUnit, filter, includeJvmMetrics, disabledMetricAttributes); - } - - @Override - public void report(SortedMap gauges, - SortedMap counters, - SortedMap histograms, - SortedMap meters, - SortedMap timers) { - - try { - if (!wavefront.isConnected()) { - wavefront.connect(); - } - - for (Map.Entry entry : gauges.entrySet()) { - if (entry.getValue().getValue() instanceof Number) { - reportGauge(entry.getKey(), entry.getValue()); - } - } - - for (Map.Entry entry : counters.entrySet()) { - reportCounter(entry.getKey(), entry.getValue()); - } - - for (Map.Entry entry : histograms.entrySet()) { - reportHistogram(entry.getKey(), entry.getValue()); - } - - for (Map.Entry entry : meters.entrySet()) { - reportMetered(entry.getKey(), entry.getValue()); - } - - for (Map.Entry entry : timers.entrySet()) { - reportTimer(entry.getKey(), entry.getValue()); - } - - wavefront.flush(); - } catch (IOException e) { - LOGGER.warn("Unable to report to Wavefront", wavefront, e); - try { - wavefront.close(); - } catch (IOException e1) { - LOGGER.warn("Error closing Wavefront", wavefront, e); - } - } - } - - @Override - public void stop() { - try { - super.stop(); - } finally { - try { - wavefront.close(); - } catch (IOException e) { - LOGGER.debug("Error disconnecting from Wavefront", wavefront, e); - } - } - } - - private void reportTimer(String name, Timer timer) throws IOException { - final Snapshot snapshot = timer.getSnapshot(); - final long time = clock.getTime() / 1000; - sendIfEnabled(MetricAttribute.MAX, name, convertDuration(snapshot.getMax()), time); - sendIfEnabled(MetricAttribute.MEAN, name, convertDuration(snapshot.getMean()), time); - sendIfEnabled(MetricAttribute.MIN, name, convertDuration(snapshot.getMin()), time); - sendIfEnabled(MetricAttribute.STDDEV, name, convertDuration(snapshot.getStdDev()), time); - sendIfEnabled(MetricAttribute.P50, name, convertDuration(snapshot.getMedian()), time); - sendIfEnabled(MetricAttribute.P75, name, convertDuration(snapshot.get75thPercentile()), time); - sendIfEnabled(MetricAttribute.P95, name, convertDuration(snapshot.get95thPercentile()), time); - sendIfEnabled(MetricAttribute.P98, name, convertDuration(snapshot.get98thPercentile()), time); - sendIfEnabled(MetricAttribute.P99, name, convertDuration(snapshot.get99thPercentile()), time); - sendIfEnabled(MetricAttribute.P999, name, convertDuration(snapshot.get999thPercentile()), time); - - reportMetered(name, timer); - } - - private void reportMetered(String name, Metered meter) throws IOException { - final long time = clock.getTime() / 1000; - sendIfEnabled(MetricAttribute.COUNT, name, meter.getCount(), time); - sendIfEnabled(MetricAttribute.M1_RATE, name, convertRate(meter.getOneMinuteRate()), time); - sendIfEnabled(MetricAttribute.M5_RATE, name, convertRate(meter.getFiveMinuteRate()), time); - sendIfEnabled(MetricAttribute.M15_RATE, name, convertRate(meter.getFifteenMinuteRate()), time); - sendIfEnabled(MetricAttribute.MEAN_RATE, name, convertRate(meter.getMeanRate()), time); - } - - private void reportHistogram(String name, Histogram histogram) throws IOException { - final Snapshot snapshot = histogram.getSnapshot(); - final long time = clock.getTime() / 1000; - sendIfEnabled(MetricAttribute.COUNT, name, histogram.getCount(), time); - sendIfEnabled(MetricAttribute.MAX, name, snapshot.getMax(), time); - sendIfEnabled(MetricAttribute.MEAN, name, snapshot.getMean(), time); - sendIfEnabled(MetricAttribute.MIN, name, snapshot.getMin(), time); - sendIfEnabled(MetricAttribute.STDDEV, name, snapshot.getStdDev(), time); - sendIfEnabled(MetricAttribute.P50, name, snapshot.getMedian(), time); - sendIfEnabled(MetricAttribute.P75, name, snapshot.get75thPercentile(), time); - sendIfEnabled(MetricAttribute.P95, name, snapshot.get95thPercentile(), time); - sendIfEnabled(MetricAttribute.P98, name, snapshot.get98thPercentile(), time); - sendIfEnabled(MetricAttribute.P99, name, snapshot.get99thPercentile(), time); - sendIfEnabled(MetricAttribute.P999, name, snapshot.get999thPercentile(), time); - } - - private void reportCounter(String name, Counter counter) throws IOException { - if (counter instanceof DeltaCounter) { - long count = counter.getCount(); - name = MetricConstants.DELTA_PREFIX + prefixAndSanitize(name.substring(1), "count"); - wavefront.send(name, count,clock.getTime() / 1000, source, pointTags); - counter.dec(count); - } else { - wavefront.send(prefixAndSanitize(name, "count"), counter.getCount(), clock.getTime() / 1000, source, pointTags); - } - } - - private void reportGauge(String name, Gauge gauge) throws IOException { - wavefront.send(prefixAndSanitize(name), gauge.getValue().doubleValue(), clock.getTime() / 1000, source, pointTags); - } - - private void sendIfEnabled(MetricAttribute type, String name, double value, long timestamp) throws IOException { - if (!getDisabledMetricAttributes().contains(type)) { - wavefront.send(prefixAndSanitize(name, type.getCode()), value, timestamp, source, pointTags); - } - } - - private String prefixAndSanitize(String... components) { - return sanitize(MetricRegistry.name(prefix, components)); - } - - private static String sanitize(String name) { - return SIMPLE_NAMES.matcher(name).replaceAll("_"); - } - - private static final Pattern SIMPLE_NAMES = Pattern.compile("[^a-zA-Z0-9_.\\-~]"); -} diff --git a/dropwizard-metrics/dropwizard-metrics5/README.md b/dropwizard-metrics/dropwizard-metrics5/README.md deleted file mode 100644 index 3b718e8c3..000000000 --- a/dropwizard-metrics/dropwizard-metrics5/README.md +++ /dev/null @@ -1,104 +0,0 @@ -# Wavefront Reporter - -This is a Wavefront Reporter for the Stable (5.0.0-rc2) version of [Dropwizard Metrics](https://dropwizard.github.io) (formerly Coda Hale & Yammer Metrics). - -It sends data to the Wavefront service via proxy or direct ingestion and supports point tags being assigned at the Reporter level as well as at the individual metric level. - -## Usage - -This Reporter sends data to Wavefront via proxy or direct ingestion. Version 3.5 or later is required. You can easily install the proxy by following [these instructions](https://docs.wavefront.com/proxies_installing.html). - -To use the Reporter you'll need to know the hostname and port (which by default is 2878) where the Wavefront proxy is running. - -It is designed to be used with the [stable version 5.0.0-rc2 of Dropwizard Metrics](https://github.com/dropwizard/metrics/tree/v5.0.0-rc2). - -### Setting up Maven - -You will need both the DropWizard `metrics-core` and the Wavefront `metrics-wavefront` libraries as dependencies. Logging depends on `org.slf4j`: - -```Maven - - io.dropwizard.metrics5 - metrics-core - 5.0.0-rc2 - - - com.wavefront - dropwizard-metrics5 - [LATEST VERSION] - - - org.slf4j - slf4j-simple - 1.7.16 - -``` - -### Example Usage - -The Wavefront Reporter lets you use DropWizard metrics exactly as you normally would. See its [getting started guide](https://dropwizard.github.io/metrics/3.1.0/getting-started/) if you haven't used it before. - -It simply gives you a new Reporter that will seamlessly work with Wavefront. First `import com.wavefront.integrations.metrics5.WavefrontReporter;` - -Then for example to create a Reporter which will emit data every 10 seconds for: - -- A `MetricsRegistry` named `metrics` -- A Wavefront proxy on `localhost` at port `2878` -- Data that should appear in Wavefront as `source=app-1.company.com` -- Two point tags named `dc` and `service` -- Two metric level point tags named `pointTag1` and `pointTag2` - -you would do something like this: - -```java -MetricRegistry registry = new MetricRegistry(); -HashMap tags = new HashMap<>(); -tags.put("pointTag1", "ptag1"); -tags.put("pointTag2", "ptag2"); -MetricName counterMetric = new MetricName("proxy.foo.bar", tags); -// Register the counter with the metric registry -Counter counter = registry.counter(counterMetric); -WavefrontReporter reporter = WavefrontReporter.forRegistry(metrics) - .withSource("app-1.company.com") - .withPointTag("dc", "dallas") - .withPointTag("service", "query") - .build("localhost", 2878); -``` - -You must provide the source using the `.withSource(String source)` method and pass the Hostname and Port of the Wavefront proxy using the `.build(String hostname, long port)` method. - -The Reporter provides all the same options that the [GraphiteReporter](http://metrics.dropwizard.io/3.1.0/manual/graphite/) does. By default: - -- There is no prefix on the Metrics -- Rates will be converted to Seconds -- Durations will be converted to Milliseconds -- `MetricFilter.ALL` will be used for the Filter -- `Clock.defaultClock()` will be used for the Clock - -In addition you can also: - -- Supply point tags for the Reporter to use. There are two ways to specify point tags at the Reporter level, individually using `.withPointTag(String tagK, String tagV)` or create a `Map` and call `.withPointTags(my-map)` to do many at once. -- Call `.withJvmMetrics()` when building the Reporter if you want it to add some default JVM metrics to the given MetricsRegistry - -If `.withJvmMetrics()` is used the following metrics will be added to the registry: - -```java -registry.register("jvm.uptime", new Gauge() { - @Override - public Long getValue() { - return ManagementFactory.getRuntimeMXBean().getUptime(); - } -}); -registry.register("jvm.current_time", new Gauge() { - @Override - public Long getValue() { - return clock.getTime(); - } -}); - -registry.register("jvm.classes", new ClassLoadingGaugeSet()); -registry.register("jvm.fd_usage", new FileDescriptorRatioGauge()); -registry.register("jvm.buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer())); -registry.register("jvm.gc", new GarbageCollectorMetricSet()); -registry.register("jvm.memory", new MemoryUsageGaugeSet()); -``` diff --git a/dropwizard-metrics/dropwizard-metrics5/pom.xml b/dropwizard-metrics/dropwizard-metrics5/pom.xml deleted file mode 100644 index 622a77ed7..000000000 --- a/dropwizard-metrics/dropwizard-metrics5/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - 4.0.0 - - - com.wavefront - wavefront - 4.35-SNAPSHOT - ../../pom.xml - - - dropwizard-metrics5 - 4.35-SNAPSHOT - Wavefront Dropwizard5 Metrics reporter - Report metrics via the Wavefront Proxy - - - - com.wavefront - java-client - 4.35-SNAPSHOT - - - io.dropwizard.metrics5 - metrics-core - 5.0.0-rc2 - - - io.dropwizard.metrics5 - metrics-jvm - 5.0.0-rc2 - - - org.json - json - 20160212 - - - - diff --git a/dropwizard-metrics/dropwizard-metrics5/src/main/java/com/wavefront/integrations/dropwizard_metrics5/WavefrontReporter.java b/dropwizard-metrics/dropwizard-metrics5/src/main/java/com/wavefront/integrations/dropwizard_metrics5/WavefrontReporter.java deleted file mode 100644 index fbc767e9b..000000000 --- a/dropwizard-metrics/dropwizard-metrics5/src/main/java/com/wavefront/integrations/dropwizard_metrics5/WavefrontReporter.java +++ /dev/null @@ -1,573 +0,0 @@ -package com.wavefront.integrations.dropwizard_metrics5; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import com.wavefront.common.MetricConstants; -import com.wavefront.integrations.Wavefront; -import com.wavefront.integrations.WavefrontDirectSender; -import com.wavefront.integrations.WavefrontSender; -import io.dropwizard.metrics5.ScheduledReporter; -import io.dropwizard.metrics5.MetricRegistry; -import io.dropwizard.metrics5.Clock; -import io.dropwizard.metrics5.MetricFilter; -import io.dropwizard.metrics5.MetricAttribute; -import io.dropwizard.metrics5.Timer; -import io.dropwizard.metrics5.Gauge; -import io.dropwizard.metrics5.MetricName; -import io.dropwizard.metrics5.Counter; -import io.dropwizard.metrics5.Histogram; -import io.dropwizard.metrics5.Snapshot; -import io.dropwizard.metrics5.Meter; -import io.dropwizard.metrics5.Metered; -import io.dropwizard.metrics5.DeltaCounter; -import io.dropwizard.metrics5.jvm.ClassLoadingGaugeSet; -import io.dropwizard.metrics5.jvm.SafeFileDescriptorRatioGauge; -import io.dropwizard.metrics5.jvm.BufferPoolMetricSet; -import io.dropwizard.metrics5.jvm.GarbageCollectorMetricSet; -import io.dropwizard.metrics5.jvm.MemoryUsageGaugeSet; -import io.dropwizard.metrics5.jvm.ThreadStatesGaugeSet; -import org.json.JSONArray; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.validation.constraints.NotNull; -import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.util.*; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; - -/** - * A reporter which publishes metric values to a Wavefront Proxy from a Dropwizard {@link MetricRegistry}. - * This reporter is based on Dropwizard version 5.0.0-rc2 that has native support for tags. This reporter - * leverages the tags maintained as part of the MetricName object that is registered with the metric registry - * for all metrics types(Counter, Gauge, Histogram, Meter, Timer) - * - * @author Subramaniam Narayanan - */ -public class WavefrontReporter extends ScheduledReporter { - private static final Logger LOGGER = LoggerFactory.getLogger(WavefrontReporter.class); - - /** - * Returns a new {@link Builder} for {@link WavefrontReporter}. - * - * @param registry the registry to report - * @return a {@link Builder} instance for a {@link WavefrontReporter} - */ - public static Builder forRegistry(MetricRegistry registry) { - return new Builder(registry); - } - - /** - * A builder for {@link WavefrontReporter} instances. Defaults to not using a prefix, using the - * default clock, converting rates to events/second, converting durations to milliseconds, a host - * named "unknown", no point Tags, and not filtering any metrics. - */ - public static class Builder { - private final MetricRegistry registry; - private Clock clock; - private String prefix; - private TimeUnit rateUnit; - private TimeUnit durationUnit; - private MetricFilter filter; - private String source; - private Map pointTags; - private boolean includeJvmMetrics; - private Set disabledMetricAttributes; - - private Builder(MetricRegistry registry) { - this.registry = registry; - this.clock = Clock.defaultClock(); - this.prefix = null; - this.rateUnit = TimeUnit.SECONDS; - this.durationUnit = TimeUnit.MILLISECONDS; - this.filter = MetricFilter.ALL; - this.source = "dropwizard-metrics"; - this.pointTags = new HashMap<>(); - this.includeJvmMetrics = false; - this.disabledMetricAttributes = Collections.emptySet(); - } - - /** - * Use the given {@link Clock} instance for the time. Defaults to Clock.defaultClock() - * - * @param clock a {@link Clock} instance - * @return {@code this} - */ - public Builder withClock(Clock clock) { - this.clock = clock; - return this; - } - - /** - * Prefix all metric names with the given string. Defaults to null. - * - * @param prefix the prefix for all metric names - * @return {@code this} - */ - public Builder prefixedWith(String prefix) { - this.prefix = prefix; - return this; - } - - /** - * Set the host for this reporter. This is equivalent to withSource. - * - * @param host the host for all metrics - * @return {@code this} - */ - public Builder withHost(String host) { - this.source = host; - return this; - } - - /** - * Set the source for this reporter. This is equivalent to withHost. - * - * @param source the host for all metrics - * @return {@code this} - */ - public Builder withSource(String source) { - this.source = source; - return this; - } - - /** - * Set the Point Tags for this reporter. - * - * @param pointTags the pointTags Map for all metrics - * @return {@code this} - */ - public Builder withPointTags(Map pointTags) { - this.pointTags.putAll(pointTags); - return this; - } - - /** - * Set a point tag for this reporter. - * - * @param ptagK the key of the Point Tag - * @param ptagV the value of the Point Tag - * @return {@code this} - */ - public Builder withPointTag(String ptagK, String ptagV) { - this.pointTags.put(ptagK, ptagV); - return this; - } - - /** - * Convert rates to the given time unit. Defaults to Seconds. - * - * @param rateUnit a unit of time - * @return {@code this} - */ - public Builder convertRatesTo(TimeUnit rateUnit) { - this.rateUnit = rateUnit; - return this; - } - - /** - * Convert durations to the given time unit. Defaults to Milliseconds. - * - * @param durationUnit a unit of time - * @return {@code this} - */ - public Builder convertDurationsTo(TimeUnit durationUnit) { - this.durationUnit = durationUnit; - return this; - } - - /** - * Only report metrics which match the given filter. Defaults to MetricFilter.ALL - * - * @param filter a {@link MetricFilter} - * @return {@code this} - */ - public Builder filter(MetricFilter filter) { - this.filter = filter; - return this; - } - - /** - * Don't report the passed metric attributes for all metrics (e.g. "p999", "stddev" or "m15"). - * See {@link MetricAttribute}. - * - * @param disabledMetricAttributes a set of {@link MetricAttribute} - * @return {@code this} - */ - public Builder disabledMetricAttributes(Set disabledMetricAttributes) { - this.disabledMetricAttributes = disabledMetricAttributes; - return this; - } - - /** - * Include JVM Metrics from this Reporter. - * - * @return {@code this} - */ - public Builder withJvmMetrics() { - this.includeJvmMetrics = true; - return this; - } - - /** - * Builds a {@link WavefrontReporter} from the VCAP_SERVICES env variable, sending metrics - * using the given {@link WavefrontSender}. This should be used in PCF environment only. It - * uses 'wavefront-proxy' as the name to fetch the proxy details from VCAP_SERVICES. - * - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter bindToCloudFoundryService() { - return bindToCloudFoundryService("wavefront-proxy", false); - } - - /** - * Builds a {@link WavefrontReporter} from the VCAP_SERVICES env variable, sending metrics - * using the given {@link WavefrontSender}. This should be used in PCF environment only. It - * assumes failOnError to be false. - * - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter bindToCloudFoundryService(@NotNull String proxyServiceName) { - return bindToCloudFoundryService(proxyServiceName, false); - } - - /** - * Builds a {@link WavefrontReporter} from the VCAP_SERVICES env variable, sending metrics - * using the given {@link WavefrontSender}. This should be used in PCF environment only. - * - * @param proxyServiceName The name of the wavefront proxy service. If wavefront-tile is used to - * deploy the proxy, then the service name will be 'wavefront-proxy'. - * @param failOnError A flag to determine what to do if the service parameters are not - * available. If 'true' then the method will throw RuntimeException else - * it will log an error message and continue. - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter bindToCloudFoundryService(@NotNull String proxyServiceName, - boolean failOnError) { - - Preconditions.checkNotNull(proxyServiceName, "proxyServiceName arg should not be null"); - - String proxyHostname; - int proxyPort; - // read the env variable VCAP_SERVICES - String services = System.getenv("VCAP_SERVICES"); - if (services == null || services.length() == 0) { - if (failOnError) { - throw new RuntimeException("VCAP_SERVICES environment variable is unavailable."); - } else { - LOGGER.error("Environment variable VCAP_SERVICES is empty. No metrics will be reported " + - "to wavefront proxy."); - // since the wavefront-proxy is not tied to the app, use dummy hostname and port. - proxyHostname = ""; - proxyPort = 2878; - } - } else { - // parse the json to read the hostname and port - JSONObject json = new JSONObject(services); - // When wavefront tile is installed on PCF, it will be automatically named wavefront-proxy - JSONArray jsonArray = json.getJSONArray(proxyServiceName); - if (jsonArray == null || jsonArray.isNull(0)) { - if (failOnError) { - throw new RuntimeException(proxyServiceName + " is not present in the VCAP_SERVICES " + - "env variable. Please verify and provide the wavefront proxy service name."); - } else { - LOGGER.error(proxyServiceName + " is not present in VCAP_SERVICES env variable. No " + - "metrics will be reported to wavefront proxy."); - // since the wavefront-proxy is not tied to the app, use dummy hostname and port. - proxyHostname = ""; - proxyPort = 2878; - } - } else { - JSONObject details = jsonArray.getJSONObject(0); - JSONObject credentials = details.getJSONObject("credentials"); - proxyHostname = credentials.getString("hostname"); - proxyPort = credentials.getInt("port"); - } - } - return new WavefrontReporter(registry, - proxyHostname, - proxyPort, - clock, - prefix, - source, - pointTags, - rateUnit, - durationUnit, - filter, - includeJvmMetrics, - disabledMetricAttributes); - } - - /** - * Builds a {@link WavefrontReporter} with the given properties, sending metrics directly - * to a given Wavefront server using direct ingestion APIs. - * - * @param server Wavefront server hostname of the form "https://serverName.wavefront.com" - * @param token Wavefront API token with direct ingestion permission - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter buildDirect(String server, String token) { - WavefrontSender wavefrontSender = new WavefrontDirectSender(server, token); - return new WavefrontReporter(registry, wavefrontSender, clock, prefix, source, pointTags, rateUnit, - durationUnit, filter, includeJvmMetrics, disabledMetricAttributes); - } - - /** - * Builds a {@link WavefrontReporter} with the given properties, sending metrics using the given - * {@link WavefrontSender}. - * - * @param proxyHostname Wavefront Proxy hostname. - * @param proxyPort Wavefront Proxy port. - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter build(String proxyHostname, int proxyPort) { - return new WavefrontReporter(registry, - proxyHostname, - proxyPort, - clock, - prefix, - source, - pointTags, - rateUnit, - durationUnit, - filter, - includeJvmMetrics, - disabledMetricAttributes); - } - - /** - * Builds a {@link WavefrontReporter} with the given properties, sending metrics using the given - * {@link WavefrontSender}. - * - * @param wavefrontSender a {@link WavefrontSender}. - * @return a {@link WavefrontReporter} - */ - public WavefrontReporter build(WavefrontSender wavefrontSender) { - return new WavefrontReporter(registry, - wavefrontSender, - clock, - prefix, - source, - pointTags, - rateUnit, - durationUnit, - filter, - includeJvmMetrics, - disabledMetricAttributes); - } -} - - private final WavefrontSender wavefront; - private final Clock clock; - private final String prefix; - private final String source; - private final Map pointTags; - - private WavefrontReporter(MetricRegistry registry, - WavefrontSender wavefrontSender, - final Clock clock, - String prefix, - String source, - Map pointTags, - TimeUnit rateUnit, - TimeUnit durationUnit, - MetricFilter filter, - boolean includeJvmMetrics, - Set disabledMetricAttributes) { - super(registry, "wavefront-reporter", filter, rateUnit, durationUnit, Executors.newSingleThreadScheduledExecutor(), - true, disabledMetricAttributes == null ? Collections.emptySet() : disabledMetricAttributes); - this.wavefront = wavefrontSender; - this.clock = clock; - this.prefix = prefix; - this.source = source; - this.pointTags = pointTags; - - if (includeJvmMetrics) { - registry.register("jvm.uptime", (Gauge) () -> ManagementFactory.getRuntimeMXBean().getUptime()); - registry.register("jvm.current_time", (Gauge) clock::getTime); - registry.register("jvm.classes", new ClassLoadingGaugeSet()); - registry.register("jvm.fd_usage", new SafeFileDescriptorRatioGauge()); - registry.register("jvm.buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer())); - registry.register("jvm.gc", new GarbageCollectorMetricSet()); - registry.register("jvm.memory", new MemoryUsageGaugeSet()); - registry.register("jvm.thread-states", new ThreadStatesGaugeSet()); - } - } - - private WavefrontReporter(MetricRegistry registry, - String proxyHostname, - int proxyPort, - final Clock clock, - String prefix, - String source, - Map pointTags, - TimeUnit rateUnit, - TimeUnit durationUnit, - MetricFilter filter, - boolean includeJvmMetrics, - Set disabledMetricAttributes) { - this(registry, new Wavefront(proxyHostname, proxyPort), clock, prefix, source, pointTags, rateUnit, - durationUnit, filter, includeJvmMetrics, disabledMetricAttributes); - } - - /** - * Called periodically by the polling thread. Subclasses should report all the given metrics. - * - * @param gauges all of the gauges in the registry - * @param counters all of the counters in the registry - * @param histograms all of the histograms in the registry - * @param meters all of the meters in the registry - * @param timers all of the timers in the registry - */ - @Override - @SuppressWarnings("rawtypes") - public void report(SortedMap gauges, - SortedMap counters, - SortedMap histograms, - SortedMap meters, - SortedMap timers) { - try { - if (!wavefront.isConnected()) { - wavefront.connect(); - } - - for (Map.Entry entry : gauges.entrySet()) { - if (entry.getValue().getValue() instanceof Number) { - reportGauge(entry.getKey(), entry.getValue()); - } - } - - for (Map.Entry entry : counters.entrySet()) { - reportCounter(entry.getKey(), entry.getValue()); - } - - for (Map.Entry entry : histograms.entrySet()) { - reportHistogram(entry.getKey(), entry.getValue()); - } - - for (Map.Entry entry : meters.entrySet()) { - reportMetered(entry.getKey(), entry.getValue()); - } - - for (Map.Entry entry : timers.entrySet()) { - reportTimer(entry.getKey(), entry.getValue()); - } - - wavefront.flush(); - } catch (IOException e) { - LOGGER.warn("Unable to report to Wavefront", wavefront, e); - try { - wavefront.close(); - } catch (IOException e1) { - LOGGER.warn("Error closing Wavefront", wavefront, e); - } - } - } - - @Override - public void stop() { - try { - super.stop(); - } finally { - try { - wavefront.close(); - } catch (IOException e) { - LOGGER.debug("Error disconnecting from Wavefront", wavefront, e); - } - } - } - - private void reportTimer(MetricName metricName, Timer timer) throws IOException { - final Snapshot snapshot = timer.getSnapshot(); - final long time = clock.getTime() / 1000; - sendIfEnabled(MetricAttribute.MAX, metricName, convertDuration(snapshot.getMax()), time); - sendIfEnabled(MetricAttribute.MEAN, metricName, convertDuration(snapshot.getMean()), time); - sendIfEnabled(MetricAttribute.MIN, metricName, convertDuration(snapshot.getMin()), time); - sendIfEnabled(MetricAttribute.STDDEV, metricName, convertDuration(snapshot.getStdDev()), time); - sendIfEnabled(MetricAttribute.P50, metricName, convertDuration(snapshot.getMedian()), time); - sendIfEnabled(MetricAttribute.P75, metricName, convertDuration(snapshot.get75thPercentile()), time); - sendIfEnabled(MetricAttribute.P95, metricName, convertDuration(snapshot.get95thPercentile()), time); - sendIfEnabled(MetricAttribute.P98, metricName, convertDuration(snapshot.get98thPercentile()), time); - sendIfEnabled(MetricAttribute.P99, metricName, convertDuration(snapshot.get99thPercentile()), time); - sendIfEnabled(MetricAttribute.P999, metricName, convertDuration(snapshot.get999thPercentile()), time); - - reportMetered(metricName, timer); - } - - private void reportMetered(MetricName metricName, Metered meter) throws IOException { - final long time = clock.getTime() / 1000; - sendIfEnabled(MetricAttribute.COUNT, metricName, meter.getCount(), time); - sendIfEnabled(MetricAttribute.M1_RATE, metricName, convertRate(meter.getOneMinuteRate()), time); - sendIfEnabled(MetricAttribute.M5_RATE, metricName, convertRate(meter.getFiveMinuteRate()), time); - sendIfEnabled(MetricAttribute.M15_RATE, metricName, convertRate(meter.getFifteenMinuteRate()), time); - sendIfEnabled(MetricAttribute.MEAN_RATE, metricName, convertRate(meter.getMeanRate()), time); - } - - private void reportHistogram(MetricName metricName, Histogram histogram) throws IOException { - final Snapshot snapshot = histogram.getSnapshot(); - final long time = clock.getTime() / 1000; - sendIfEnabled(MetricAttribute.COUNT, metricName, histogram.getCount(), time); - sendIfEnabled(MetricAttribute.MAX, metricName, snapshot.getMax(), time); - sendIfEnabled(MetricAttribute.MEAN, metricName, snapshot.getMean(), time); - sendIfEnabled(MetricAttribute.MIN, metricName, snapshot.getMin(), time); - sendIfEnabled(MetricAttribute.STDDEV, metricName, snapshot.getStdDev(), time); - sendIfEnabled(MetricAttribute.P50, metricName, snapshot.getMedian(), time); - sendIfEnabled(MetricAttribute.P75, metricName, snapshot.get75thPercentile(), time); - sendIfEnabled(MetricAttribute.P95, metricName, snapshot.get95thPercentile(), time); - sendIfEnabled(MetricAttribute.P98, metricName, snapshot.get98thPercentile(), time); - sendIfEnabled(MetricAttribute.P99, metricName, snapshot.get99thPercentile(), time); - sendIfEnabled(MetricAttribute.P999, metricName, snapshot.get999thPercentile(), time); - } - - private void reportCounter(MetricName metricName, Counter counter) throws IOException { - if (counter instanceof DeltaCounter) { - long count = counter.getCount(); - String name = MetricConstants.DELTA_PREFIX + prefixAndSanitize(metricName.getKey().substring(1), "count"); - wavefront.send(name, count,clock.getTime() / 1000, source, getMetricTags(metricName)); - counter.dec(count); - } else { - wavefront.send(prefixAndSanitize(metricName.getKey(), "count"), counter.getCount(), clock.getTime() / 1000, source, getMetricTags(metricName)); - } - } - - private void reportGauge(MetricName metricName, Gauge gauge) throws IOException { - wavefront.send(prefixAndSanitize(metricName.getKey()), gauge.getValue().doubleValue(), clock.getTime() / 1000, source, getMetricTags(metricName)); - } - - private void sendIfEnabled(MetricAttribute type, MetricName metricName, double value, long timestamp) throws IOException { - if (!getDisabledMetricAttributes().contains(type)) { - wavefront.send(prefixAndSanitize(metricName.getKey(), type.getCode()), value, timestamp, source, getMetricTags(metricName)); - } - } - - private Map getMetricTags(MetricName metricName) { - int tagCount = pointTags.size() + metricName.getTags().size(); - // If there are no tags(point tag(s) or global return an empty map - if (tagCount == 0) { - return Collections.emptyMap(); - } - - // NOTE: If the individual metric share the same key as the global point tag key, the - // metric level value will override global level value for that point tag. - // Example: Global point tag is <"Key1", "Value-Global"> - // and metric level point tag is: <"Key1", "Value-Metric1"> - // the point tag sent to Wavefront will be <"Key1", "Value-Metric1"> - HashMap metricTags = Maps.newHashMapWithExpectedSize(tagCount); - metricTags.putAll(pointTags); - metricName.getTags().forEach((k, v) -> metricTags.putIfAbsent(k, v)); - return metricTags; - } - - private String prefixAndSanitize(String... components) { - return sanitize(MetricRegistry.name(prefix, components).getKey()); - } - - private static String sanitize(String name) { - return SIMPLE_NAMES.matcher(name).replaceAll("_"); - } - - private static final Pattern SIMPLE_NAMES = Pattern.compile("[^a-zA-Z0-9_.\\-~]"); -} diff --git a/examples/dropwizard-metrics/pom.xml b/examples/dropwizard-metrics/pom.xml deleted file mode 100644 index b63094168..000000000 --- a/examples/dropwizard-metrics/pom.xml +++ /dev/null @@ -1,70 +0,0 @@ - - 4.0.0 - - dropwizard-metrics-examples - 0.0.1-SNAPSHOT - Wavefront Dropwizard Metrics Examples - Wavefront Dropwizard Metrics Examples - - - com.wavefront - wavefront - 4.30-SNAPSHOT - - - - - io.dropwizard.metrics - metrics-core - 4.0.2 - - - io.dropwizard.metrics - metrics-jvm - 4.0.2 - - - com.wavefront - dropwizard-metrics - 4.29 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5.1 - - 1.8 - 1.8 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.0.0 - - - package - - shade - - - - - - META-INF/license/** - license/** - - - - - - - - - - diff --git a/examples/dropwizard-metrics/src/main/java/com/wavefront/examples/DirectReporting.java b/examples/dropwizard-metrics/src/main/java/com/wavefront/examples/DirectReporting.java deleted file mode 100644 index 2869a97f9..000000000 --- a/examples/dropwizard-metrics/src/main/java/com/wavefront/examples/DirectReporting.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.wavefront.examples; - -import java.util.concurrent.TimeUnit; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.MetricRegistry; -import com.wavefront.integrations.metrics.WavefrontReporter; - -/** - * Example for reporting dropwizard metrics into Wavefront via Direct Ingestion. - * - * @author Vikram Raman - */ -public class DirectReporting { - - public static void main(String args[]) throws InterruptedException { - - String server = args[0]; - String token = args[1]; - - MetricRegistry registry = new MetricRegistry(); - Counter counter = registry.counter("direct.metric.foo.bar"); - - WavefrontReporter reporter = WavefrontReporter.forRegistry(registry). - withSource("app-1.company.com"). - withPointTag("dc", "dallas"). - withPointTag("service", "query"). - buildDirect(server, token); - reporter.start(5, TimeUnit.SECONDS); - - int i = 0; - while (i++ < 30) { - counter.inc(10); - Thread.sleep(1000); - } - reporter.stop(); - } -} diff --git a/examples/dropwizard-metrics/src/main/java/com/wavefront/examples/ProxyReporting.java b/examples/dropwizard-metrics/src/main/java/com/wavefront/examples/ProxyReporting.java deleted file mode 100644 index 55614ae91..000000000 --- a/examples/dropwizard-metrics/src/main/java/com/wavefront/examples/ProxyReporting.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.wavefront.examples; - -import java.util.concurrent.TimeUnit; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.MetricRegistry; -import com.wavefront.integrations.metrics.WavefrontReporter; - -/** - * Example for reporting dropwizard metrics into Wavefront via proxy. - * - * @author Vikram Raman - */ -public class ProxyReporting { - public static void main(String args[]) throws InterruptedException { - - String host = args[0]; - int port = Integer.parseInt(args[1]); - - MetricRegistry registry = new MetricRegistry(); - Counter counter = registry.counter("proxy.metric.foo.bar"); - - WavefrontReporter reporter = WavefrontReporter.forRegistry(registry). - withSource("app-1.company.com"). - withPointTag("dc", "dallas"). - withPointTag("service", "query"). - build(host, port); - reporter.start(5, TimeUnit.SECONDS); - - int i = 0; - while (i++ < 30) { - counter.inc(10); - Thread.sleep(1000); - } - reporter.stop(); - } -} diff --git a/examples/dropwizard-metrics5/pom.xml b/examples/dropwizard-metrics5/pom.xml deleted file mode 100644 index 70577f4f3..000000000 --- a/examples/dropwizard-metrics5/pom.xml +++ /dev/null @@ -1,71 +0,0 @@ - - 4.0.0 - - dropwizard-metrics5-examples - 0.0.1-SNAPSHOT - Wavefront Dropwizard Metrics Examples - Wavefront Dropwizard5 Metrics Examples - - - com.wavefront - wavefront - 4.30-SNAPSHOT - ../../pom.xml - - - - - io.dropwizard.metrics5 - metrics-core - 5.0.0-rc2 - - - io.dropwizard.metrics5 - metrics-jvm - 5.0.0-rc2 - - - com.wavefront - dropwizard-metrics5 - 4.30-SNAPSHOT - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5.1 - - 1.8 - 1.8 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.0.0 - - - package - - shade - - - - - - META-INF/license/** - license/** - - - - - - - - - - diff --git a/examples/dropwizard-metrics5/src/main/java/com/wavefront/examples/DirectReporting.java b/examples/dropwizard-metrics5/src/main/java/com/wavefront/examples/DirectReporting.java deleted file mode 100644 index 1582af29e..000000000 --- a/examples/dropwizard-metrics5/src/main/java/com/wavefront/examples/DirectReporting.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.wavefront.examples; - -import com.wavefront.integrations.dropwizard_metrics5.WavefrontReporter; - -import io.dropwizard.metrics5.Counter; -import io.dropwizard.metrics5.MetricName; -import io.dropwizard.metrics5.MetricRegistry; - -import java.util.HashMap; -import java.util.concurrent.TimeUnit; - -/** - * Example for reporting dropwizard metrics into Wavefront via Direct Ingestion. - * - * @author Subramaniam Narayanan - */ -public class DirectReporting { - public static void main(String args[]) throws InterruptedException { - String server = args[0]; - String token = args[1]; - - MetricRegistry registry = new MetricRegistry(); - HashMap tags = new HashMap<>(); - tags.put("pointkey1", "ptag1"); - tags.put("pointkey2", "ptag2"); - // Create metric name object to associated with the metric type. The key is the - // metric name and the value are the optional point tags. - MetricName counterMetric = new MetricName("direct.dw5metric.foo.bar", tags); - // Register the counter with the metric registry - Counter counter = registry.counter(counterMetric); - - // Create a Wavefront Reporter as a direct reporter - requires knowledge of - // Wavefront server to connect to along with a valid token. - // NOTE: If the individual metric share the same key as the global point tag key, the - // metric level value will override global level value for that point tag. - // Example: Global point tag is <"Key1", "Value-Global"> - // and metric level point tag is: <"Key1", "Value-Metric1"> - // the point tag sent to Wavefront will be <"Key1", "Value-Metric1"> - WavefrontReporter reporter = WavefrontReporter.forRegistry(registry). - withSource("app-1.company.com"). - withPointTag("gkey1", "gvalue1"). - buildDirect(server, token); - reporter.start(5, TimeUnit.SECONDS); - - int i = 0; - while (i++ < 30) { - // Periodically update counter - counter.inc(10); - Thread.sleep(1000); - } - reporter.stop(); - } -} diff --git a/examples/dropwizard-metrics5/src/main/java/com/wavefront/examples/ProxyReporting.java b/examples/dropwizard-metrics5/src/main/java/com/wavefront/examples/ProxyReporting.java deleted file mode 100644 index d3709fe0a..000000000 --- a/examples/dropwizard-metrics5/src/main/java/com/wavefront/examples/ProxyReporting.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.wavefront.examples; - -import com.wavefront.integrations.dropwizard_metrics5.WavefrontReporter; - -import io.dropwizard.metrics5.Counter; -import io.dropwizard.metrics5.MetricName; -import io.dropwizard.metrics5.MetricRegistry; - -import java.util.HashMap; -import java.util.concurrent.TimeUnit; - -/** - * Example for reporting dropwizard metrics into Wavefront via proxy. - * - * @author Subramaniam Narayanan - */ -public class ProxyReporting { - public static void main(String args[]) throws InterruptedException { - String host = args[0]; - int port = Integer.parseInt(args[1]); - - MetricRegistry registry = new MetricRegistry(); - HashMap tags = new HashMap<>(); - tags.put("pointkey1", "ptag1"); - tags.put("pointkey2", "ptag2"); - // Create metric name object to associated with the metric type. The key is the - // metric name and the value are the optional point tags. - // NOTE: If the individual metric share the same key as the global point tag key, the - // metric level value will override global level value for that point tag. - // Example: Global point tag is <"Key1", "Value-Global"> - // and metric level point tag is: <"Key1", "Value-Metric1"> - // the point tag sent to Wavefront will be <"Key1", "Value-Metric1"> - MetricName counterMetric = new MetricName("proxy.dw5metric.foo.bar", tags); - // Register the counter with the metric registry - Counter counter = registry.counter(counterMetric); - // Create a Wavefront Reporter as a direct reporter - requires knowledge of - // Wavefront server to connect to along with a valid token. - WavefrontReporter reporter = WavefrontReporter.forRegistry(registry). - withSource("app-1.company.com"). - withPointTag("gkey1", "gvalue1"). - build(host, port); - reporter.start(5, TimeUnit.SECONDS); - - int i = 0; - while (i++ < 30) { - // Periodically update counter - counter.inc(10); - Thread.sleep(5000); - } - reporter.stop(); - } -} diff --git a/java-client/pom.xml b/java-client/pom.xml deleted file mode 100644 index 39adb4666..000000000 --- a/java-client/pom.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - 4.0.0 - - java-client - Wavefront Java Integrations Core Library - Java client for sending data to Wavefront - - - com.wavefront - wavefront - 4.35-SNAPSHOT - - - - - Clement Pang - clement@wavefront.com - Wavefront - http://www.wavefront.com - - - Conor Beverland - conor@wavefront.com - Wavefront - http://www.wavefront.com - - - - - - com.google.code.findbugs - jsr305 - 3.0.0 - - - junit - junit - 4.11 - test - - - com.wavefront - java-lib - - - diff --git a/java-client/src/main/java/com/wavefront/integrations/AbstractDirectConnectionHandler.java b/java-client/src/main/java/com/wavefront/integrations/AbstractDirectConnectionHandler.java deleted file mode 100644 index 332b61acd..000000000 --- a/java-client/src/main/java/com/wavefront/integrations/AbstractDirectConnectionHandler.java +++ /dev/null @@ -1,148 +0,0 @@ -package com.wavefront.integrations; - -import com.wavefront.api.DataIngesterAPI; -import com.wavefront.common.NamedThreadFactory; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.zip.GZIPOutputStream; - -import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -/** - * Abstract base class for sending data directly to a Wavefront service. - * - * @author Vikram Raman (vikram@wavefront.com) - */ -public abstract class AbstractDirectConnectionHandler implements WavefrontConnectionHandler, Runnable { - - private static final String DEFAULT_SOURCE = "wavefrontDirectSender"; - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDirectConnectionHandler.class); - - private ScheduledExecutorService scheduler; - private final String server; - private final String token; - private DataIngesterAPI directService; - - protected AbstractDirectConnectionHandler(String server, String token) { - this.server = server; - this.token = token; - } - - @Override - public synchronized void connect() throws IllegalStateException, IOException { - if (directService == null) { - directService = new DataIngesterService(server, token); - scheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory(DEFAULT_SOURCE)); - scheduler.scheduleAtFixedRate(this, 1, 1, TimeUnit.SECONDS); - } - } - - @Override - public void flush() throws IOException { - internalFlush(); - } - - protected abstract void internalFlush() throws IOException; - - @Override - public synchronized boolean isConnected() { - return directService != null; - } - - @Override - public synchronized void close() throws IOException { - if (directService != null) { - try { - scheduler.shutdownNow(); - } catch (SecurityException ex) { - LOGGER.debug("shutdown error", ex); - } - scheduler = null; - directService = null; - } - } - - protected Response report(String format, InputStream is) throws IOException { - return directService.report(format, is); - } - - private static final class DataIngesterService implements DataIngesterAPI { - - private final String token; - private final URI uri; - private static final String BAD_REQUEST = "Bad client request"; - private static final int CONNECT_TIMEOUT = 30000; - private static final int READ_TIMEOUT = 10000; - - public DataIngesterService(String server, String token) { - this.token = token; - uri = URI.create(server); - } - - @Override - public Response report(String format, InputStream stream) throws IOException { - - /** - * Refer https://docs.oracle.com/javase/8/docs/technotes/guides/net/http-keepalive.html - * for details around why this code is written as it is. - */ - - int statusCode = 400; - String respMsg = BAD_REQUEST; - HttpURLConnection urlConn = null; - try { - URL url = new URL(uri.getScheme(), uri.getHost(), uri.getPort(), String.format("/report?f=" + format)); - urlConn = (HttpURLConnection) url.openConnection(); - urlConn.setDoOutput(true); - urlConn.addRequestProperty(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM); - urlConn.addRequestProperty(HttpHeaders.CONTENT_ENCODING, "gzip"); - urlConn.addRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + token); - - urlConn.setConnectTimeout(CONNECT_TIMEOUT); - urlConn.setReadTimeout(READ_TIMEOUT); - - try (GZIPOutputStream gzipOS = new GZIPOutputStream(urlConn.getOutputStream())) { - byte[] buffer = new byte[4096]; - int len = 0; - while ((len = stream.read(buffer)) > 0) { - gzipOS.write(buffer); - } - gzipOS.flush(); - } - statusCode = urlConn.getResponseCode(); - respMsg = urlConn.getResponseMessage(); - readAndClose(urlConn.getInputStream()); - } catch (IOException ex) { - if (urlConn != null) { - statusCode = urlConn.getResponseCode(); - respMsg = urlConn.getResponseMessage(); - readAndClose(urlConn.getErrorStream()); - } - } - return Response.status(statusCode).entity(respMsg).build(); - } - - private void readAndClose(InputStream stream) throws IOException { - if (stream != null) { - try (InputStream is = stream) { - byte[] buffer = new byte[4096]; - int ret = 0; - // read entire stream before closing - while ((ret = is.read(buffer)) > 0) {} - } - } - } - } -} diff --git a/java-client/src/main/java/com/wavefront/integrations/AbstractProxyConnectionHandler.java b/java-client/src/main/java/com/wavefront/integrations/AbstractProxyConnectionHandler.java deleted file mode 100644 index 3909ca1d4..000000000 --- a/java-client/src/main/java/com/wavefront/integrations/AbstractProxyConnectionHandler.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.wavefront.integrations; - -import com.wavefront.metrics.ReconnectingSocket; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.function.Supplier; - -import javax.annotation.Nullable; -import javax.net.SocketFactory; - -/** - * Abstract base class for sending data to a Wavefront proxy. - * - * @author Clement Pang (clement@wavefront.com). - * @author Vikram Raman (vikram@wavefront.com). - */ -public abstract class AbstractProxyConnectionHandler implements WavefrontConnectionHandler { - - private final InetSocketAddress address; - private final SocketFactory socketFactory; - private volatile ReconnectingSocket reconnectingSocket; - @Nullable - private final Long connectionTimeToLiveMillis; - @Nullable - private final Supplier timeSupplier; - - protected AbstractProxyConnectionHandler(InetSocketAddress address, SocketFactory socketFactory) { - this(address, socketFactory, null, null); - } - - protected AbstractProxyConnectionHandler(InetSocketAddress address, SocketFactory socketFactory, - @Nullable Long connectionTimeToLiveMillis, - @Nullable Supplier timeSupplier) { - this.address = address; - this.socketFactory = socketFactory; - this.connectionTimeToLiveMillis = connectionTimeToLiveMillis; - this.timeSupplier = timeSupplier; - this.reconnectingSocket = null; - } - - @Override - public synchronized void connect() throws IllegalStateException, IOException { - if (reconnectingSocket != null) { - throw new IllegalStateException("Already connected"); - } - try { - reconnectingSocket = new ReconnectingSocket(address.getHostName(), address.getPort(), socketFactory, - connectionTimeToLiveMillis, timeSupplier); - } catch (Exception e) { - throw new IOException(e); - } - } - - @Override - public boolean isConnected() { - return reconnectingSocket != null; - } - - @Override - public void flush() throws IOException { - if (reconnectingSocket != null) { - reconnectingSocket.flush(); - } - } - - @Override - public synchronized void close() throws IOException { - if (reconnectingSocket != null) { - reconnectingSocket.close(); - reconnectingSocket = null; - } - } - - /** - * Sends the given data to the Wavefront proxy. - * - * @param lineData line data in a Wavefront supported format - * @throws Exception If there was failure sending the data - */ - protected void sendData(String lineData) throws Exception { - reconnectingSocket.write(lineData); - } -} diff --git a/java-client/src/main/java/com/wavefront/integrations/Main.java b/java-client/src/main/java/com/wavefront/integrations/Main.java deleted file mode 100644 index f2f0aad2f..000000000 --- a/java-client/src/main/java/com/wavefront/integrations/Main.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.wavefront.integrations; - -import java.io.IOException; - -/** - * Driver class for ad-hoc experiments with {@link WavefrontSender} - * - * @author Mori Bellamy (mori@wavefront.com) - */ -public class Main { - - public static void main(String[] args) throws InterruptedException, IOException { - String host = args[0]; - String port = args[1]; - System.out.println(host + ":" + port); - Wavefront wavefront = new Wavefront(host, Integer.parseInt(port)); - while (true) { - wavefront.send("mymetric.foo", 42); - wavefront.flush(); - Thread.sleep(2000); - } - } -} diff --git a/java-client/src/main/java/com/wavefront/integrations/Wavefront.java b/java-client/src/main/java/com/wavefront/integrations/Wavefront.java deleted file mode 100644 index ce0f1d688..000000000 --- a/java-client/src/main/java/com/wavefront/integrations/Wavefront.java +++ /dev/null @@ -1,205 +0,0 @@ -package com.wavefront.integrations; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.UnknownHostException; -import java.nio.charset.Charset; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; -import java.util.regex.Pattern; - -import javax.annotation.Nullable; -import javax.net.SocketFactory; - -/** - * Wavefront Client that sends data directly via TCP to the Wavefront Proxy Agent. User should probably - * attempt to reconnect when exceptions are thrown from any methods. - * - * @author Clement Pang (clement@wavefront.com). - * @author Conor Beverland (conor@wavefront.com). - */ -public class Wavefront extends AbstractProxyConnectionHandler implements WavefrontSender { - - private static final Pattern WHITESPACE = Pattern.compile("[\\s]+"); - // this may be optimistic about Carbon/Wavefront - private static final Charset UTF_8 = Charset.forName("UTF-8"); - - private AtomicInteger failures = new AtomicInteger(); - /** - * Source to use if there's none. - */ - private String source; - - /** - * Creates a new client which connects to the given address using the default - * {@link SocketFactory}. - * - * @param agentHostName The hostname of the Wavefront Proxy Agent - * @param port The port of the Wavefront Proxy Agent - */ - public Wavefront(String agentHostName, int port) { - this(agentHostName, port, SocketFactory.getDefault()); - } - - /** - * Creates a new client which connects to the given address and socket factory. - * - * @param agentHostName The hostname of the Wavefront Proxy Agent - * @param port The port of the Wavefront Proxy Agent - * @param socketFactory the socket factory - */ - public Wavefront(String agentHostName, int port, SocketFactory socketFactory) { - this(new InetSocketAddress(agentHostName, port), socketFactory); - } - - /** - * Creates a new client which connects to the given address using the default - * {@link SocketFactory}. - * - * @param agentAddress the address of the Wavefront Proxy Agent - */ - public Wavefront(InetSocketAddress agentAddress) { - this(agentAddress, SocketFactory.getDefault()); - } - - /** - * Creates a new client which connects to the given address and socket factory using the given - * character set. - * - * @param agentAddress the address of the Wavefront Proxy Agent - * @param socketFactory the socket factory - */ - public Wavefront(InetSocketAddress agentAddress, SocketFactory socketFactory) { - this(agentAddress, socketFactory, null, null); - } - - /** - * Creates a new client which connects to the given address and socket factory and enforces connection TTL limit - * - * @param agentAddress the address of the Wavefront Proxy Agent - * @param socketFactory the socket factory - * @param connectionTimeToLiveMillis Connection TTL, with expiration checked after each flush. When null, - * TTL is not enforced. - * @param timeSupplier Get current timestamp in millis - */ - public Wavefront(InetSocketAddress agentAddress, SocketFactory socketFactory, - @Nullable Long connectionTimeToLiveMillis, @Nullable Supplier timeSupplier) { - super(agentAddress, socketFactory, connectionTimeToLiveMillis, timeSupplier); - } - - private void initializeSource() throws UnknownHostException { - if (source == null) { - source = InetAddress.getLocalHost().getHostName(); - } - } - - @Override - public void send(String name, double value) throws IOException { - initializeSource(); - internalSend(name, value, null, source, null); - } - - @Override - public void send(String name, double value, @Nullable Long timestamp) throws IOException { - initializeSource(); - internalSend(name, value, timestamp, source, null); - } - - @Override - public void send(String name, double value, @Nullable Long timestamp, String source) throws IOException { - internalSend(name, value, timestamp, source, null); - } - - @Override - public void send(String name, double value, String source, @Nullable Map pointTags) - throws IOException { - internalSend(name, value, null, source, pointTags); - } - - @Override - public void send(String name, double value, @Nullable Long timestamp, String source, - @Nullable Map pointTags) throws IOException { - internalSend(name, value, timestamp, source, pointTags); - } - - private void internalSend(String name, double value, @Nullable Long timestamp, String source, - @Nullable Map pointTags) throws IOException { - if (!isConnected()) { - try { - connect(); - } catch (IllegalStateException ex) { - // already connected. - } - } - if (isBlank(name)) { - throw new IllegalArgumentException("metric name cannot be blank"); - } - if (isBlank(source)) { - throw new IllegalArgumentException("source cannot be blank"); - } - final StringBuilder sb = new StringBuilder(); - try { - sb.append(sanitize(name)); - sb.append(' '); - sb.append(Double.toString(value)); - if (timestamp != null) { - sb.append(' '); - sb.append(Long.toString(timestamp)); - } - sb.append(" host="); - sb.append(sanitize(source)); - if (pointTags != null) { - for (final Map.Entry tag : pointTags.entrySet()) { - if (isBlank(tag.getKey())) { - throw new IllegalArgumentException("point tag key cannot be blank"); - } - if (isBlank(tag.getValue())) { - throw new IllegalArgumentException("point tag value cannot be blank"); - } - sb.append(' '); - sb.append(sanitize(tag.getKey())); - sb.append('='); - sb.append(sanitize(tag.getValue())); - } - } - sb.append('\n'); - try { - sendData(sb.toString()); - } catch (Exception e) { - throw new IOException(e); - } - } catch (IOException e) { - failures.incrementAndGet(); - throw e; - } - } - - @Override - public int getFailureCount() { - return failures.get(); - } - - static String sanitize(String s) { - final String whitespaceSanitized = WHITESPACE.matcher(s).replaceAll("-"); - if (s.contains("\"") || s.contains("'")) { - // for single quotes, once we are double-quoted, single quotes can exist happily inside it. - return "\"" + whitespaceSanitized.replaceAll("\"", "\\\\\"") + "\""; - } else { - return "\"" + whitespaceSanitized + "\""; - } - } - - private static boolean isBlank(String s) { - if (s == null || s.isEmpty()) { - return true; - } - for (int i = 0; i < s.length(); i++) { - if (!Character.isWhitespace(s.charAt(i))) { - return false; - } - } - return true; - } -} diff --git a/java-client/src/main/java/com/wavefront/integrations/WavefrontConnectionHandler.java b/java-client/src/main/java/com/wavefront/integrations/WavefrontConnectionHandler.java deleted file mode 100644 index 59da051a8..000000000 --- a/java-client/src/main/java/com/wavefront/integrations/WavefrontConnectionHandler.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.wavefront.integrations; - -import java.io.Closeable; -import java.io.IOException; - -/** - * Wavefront Client that sends data to a Wavefront proxy or Wavefront service. - * - * @author Vikram Raman (vikram@wavefront.com) - */ -public interface WavefrontConnectionHandler extends Closeable { - - /** - * Connects to the server. - * - * @throws IllegalStateException if the client is already connected - * @throws IOException if there is an error connecting - */ - void connect() throws IllegalStateException, IOException; - - /** - * Flushes buffer, if applicable - * - * @throws IOException - */ - void flush() throws IOException; - - /** - * Returns true if ready to send data - */ - boolean isConnected(); - - /** - * Returns the number of failed writes to the server. - * - * @return the number of failed writes to the server - */ - int getFailureCount(); -} diff --git a/java-client/src/main/java/com/wavefront/integrations/WavefrontDirectSender.java b/java-client/src/main/java/com/wavefront/integrations/WavefrontDirectSender.java deleted file mode 100644 index 7908940c6..000000000 --- a/java-client/src/main/java/com/wavefront/integrations/WavefrontDirectSender.java +++ /dev/null @@ -1,180 +0,0 @@ -package com.wavefront.integrations; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.atomic.AtomicInteger; - -import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; -import javax.ws.rs.core.Response; - -/** - * Wavefront Client that sends data directly to Wavefront via the direct ingestion APIs. - * - * @author Vikram Raman (vikram@wavefront.com) - */ -public class WavefrontDirectSender extends AbstractDirectConnectionHandler implements WavefrontSender { - - private static final String DEFAULT_SOURCE = "wavefrontDirectSender"; - private static final Logger LOGGER = LoggerFactory.getLogger(WavefrontDirectSender.class); - private static final String quote = "\""; - private static final String escapedQuote = "\\\""; - private static final int MAX_QUEUE_SIZE = 50000; - private static final int BATCH_SIZE = 10000; - - private final LinkedBlockingQueue buffer = new LinkedBlockingQueue<>(MAX_QUEUE_SIZE); - private final AtomicInteger failures = new AtomicInteger(); - - /** - * Creates a new client that connects directly to a given Wavefront service. - * - * @param server A Wavefront server URL of the form "https://clusterName.wavefront.com" - * @param token A valid API token with direct ingestion permissions - */ - public WavefrontDirectSender(String server, String token) { - super(server, token); - } - - @Override - public void send(String name, double value) throws IOException { - addPoint(name, value, null, DEFAULT_SOURCE, null); - } - - @Override - public void send(String name, double value, @Nullable Long timestamp) throws IOException { - addPoint(name, value, timestamp, DEFAULT_SOURCE, null); - } - - @Override - public void send(String name, double value, @Nullable Long timestamp, String source) throws IOException { - addPoint(name, value, timestamp, source, null); - } - - @Override - public void send(String name, double value, String source, @Nullable Map pointTags) throws IOException { - addPoint(name, value, null, source, pointTags); - } - - @Override - public void send(String name, double value, @Nullable Long timestamp, String source, - @Nullable Map pointTags) throws IOException { - addPoint(name, value, timestamp, source, pointTags); - } - - private void addPoint(@NotNull String name, double value, @Nullable Long timestamp, @NotNull String source, - @Nullable Map pointTags) throws IOException { - String point = pointToString(name, value, timestamp, source, pointTags); - if (point != null && !buffer.offer(point)) { - LOGGER.debug("Buffer full, dropping point " + name); - } - } - - private static String escapeQuotes(String raw) { - return StringUtils.replace(raw, quote, escapedQuote); - } - - @Nullable - static String pointToString(String name, double value, @Nullable Long timestamp, String source, - @Nullable Map pointTags) { - - if (StringUtils.isBlank(name) || StringUtils.isBlank(source)) { - LOGGER.debug("Invalid point: Empty name/source"); - return null; - } - - StringBuilder sb = new StringBuilder(quote). - append(escapeQuotes(name)).append(quote).append(" "). - append(Double.toString(value)).append(" "); - if (timestamp != null) { - sb.append(Long.toString(timestamp)).append(" "); - } - sb.append("source=").append(quote).append(escapeQuotes(source)).append(quote); - - if (pointTags != null) { - for (Map.Entry entry : pointTags.entrySet()) { - sb.append(' ').append(quote).append(escapeQuotes(entry.getKey())).append(quote). - append("="). - append(quote).append(escapeQuotes(entry.getValue())).append(quote); - } - } - return sb.toString(); - } - - @Override - protected void internalFlush() throws IOException { - - if (!isConnected()) { - return; - } - - List points = getPointsBatch(); - if (points.isEmpty()) { - return; - } - - Response response = null; - try (InputStream is = pointsToStream(points)) { - response = report("graphite_v2", is); - if (response.getStatusInfo().getFamily() == Response.Status.Family.SERVER_ERROR || - response.getStatusInfo().getFamily() == Response.Status.Family.CLIENT_ERROR) { - LOGGER.debug("Error reporting points, respStatus=" + response.getStatus()); - try { - buffer.addAll(points); - } catch (Exception ex) { - // unlike offer(), addAll adds partially and throws an exception if buffer full - LOGGER.debug("Buffer full, dropping attempted points"); - } - } - } catch (IOException ex) { - failures.incrementAndGet(); - throw ex; - } finally { - if (response != null) { - response.close(); - } - } - } - - private InputStream pointsToStream(List points) { - StringBuilder sb = new StringBuilder(); - boolean newLine = false; - for (String point : points) { - if (newLine) { - sb.append("\n"); - } - sb.append(point); - newLine = true; - } - return new ByteArrayInputStream(sb.toString().getBytes()); - } - - private List getPointsBatch() { - int blockSize = Math.min(buffer.size(), BATCH_SIZE); - List points = new ArrayList<>(blockSize); - buffer.drainTo(points, blockSize); - return points; - } - - @Override - public int getFailureCount() { - return failures.get(); - } - - @Override - public void run() { - try { - this.internalFlush(); - } catch (Throwable ex) { - LOGGER.debug("Unable to report to Wavefront", ex); - } - } -} diff --git a/java-client/src/main/java/com/wavefront/integrations/WavefrontSender.java b/java-client/src/main/java/com/wavefront/integrations/WavefrontSender.java deleted file mode 100644 index 8b51e1440..000000000 --- a/java-client/src/main/java/com/wavefront/integrations/WavefrontSender.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.wavefront.integrations; - -import java.io.Closeable; -import java.io.IOException; -import java.net.UnknownHostException; -import java.util.Map; - -import javax.annotation.Nullable; - -/** - * Wavefront Client that sends data directly via TCP to the Wavefront Proxy Agent. - * - * @author Clement Pang (clement@wavefront.com). - * @author Conor Beverland (conor@wavefront.com). - */ -public interface WavefrontSender extends WavefrontConnectionHandler, Closeable { - - /** - * Send a measurement to Wavefront. The current machine's hostname would be used as the source. The point will be - * timestamped at the agent. - * - * @param name The name of the metric. Spaces are replaced with '-' (dashes) and quotes will be automatically - * escaped. - * @param value The value to be sent. - * @throws IOException Throws if there was an error sending the metric. - * @throws UnknownHostException Throws if there's an error determining the current host. - */ - void send(String name, double value) throws IOException; - - /** - * Send a measurement to Wavefront. The current machine's hostname would be used as the source. - * - * @param name The name of the metric. Spaces are replaced with '-' (dashes) and quotes will be automatically - * escaped. - * @param value The value to be sent. - * @param timestamp The timestamp in seconds since the epoch to be sent. - * @throws IOException Throws if there was an error sending the metric. - * @throws UnknownHostException Throws if there's an error determining the current host. - */ - void send(String name, double value, @Nullable Long timestamp) throws IOException; - - /** - * Send a measurement to Wavefront. - * - * @param name The name of the metric. Spaces are replaced with '-' (dashes) and quotes will be automatically - * escaped. - * @param value The value to be sent. - * @param timestamp The timestamp in seconds since the epoch to be sent. - * @param source The source (or host) that's sending the metric. - * @throws IOException if there was an error sending the metric. - */ - void send(String name, double value, @Nullable Long timestamp, String source) throws IOException; - - /** - * Send the given measurement to the server. - * - * @param name The name of the metric. Spaces are replaced with '-' (dashes) and quotes will be automatically - * escaped. - * @param value The value to be sent. - * @param source The source (or host) that's sending the metric. Null to use machine hostname. - * @param pointTags The point tags associated with this measurement. - * @throws IOException if there was an error sending the metric. - */ - void send(String name, double value, String source, @Nullable Map pointTags) throws IOException; - - /** - * Send the given measurement to the server. - * - * @param name The name of the metric. Spaces are replaced with '-' (dashes) and quotes will be automatically - * escaped. - * @param value The value to be sent. - * @param timestamp The timestamp in seconds since the epoch to be sent. Null to use agent assigned timestamp. - * @param source The source (or host) that's sending the metric. Null to use machine hostname. - * @param pointTags The point tags associated with this measurement. - * @throws IOException if there was an error sending the metric. - */ - void send(String name, double value, @Nullable Long timestamp, String source, - @Nullable Map pointTags) throws IOException; -} diff --git a/java-client/src/test/java/com/wavefront/integrations/WavefrontDirectSenderTest.java b/java-client/src/test/java/com/wavefront/integrations/WavefrontDirectSenderTest.java deleted file mode 100644 index 9885ebe64..000000000 --- a/java-client/src/test/java/com/wavefront/integrations/WavefrontDirectSenderTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.wavefront.integrations; - -import com.google.common.collect.ImmutableMap; - -import org.junit.Assert; -import org.junit.Test; - -import static org.junit.Assert.assertNull; - -/** - * Tests for {@link WavefrontDirectSender} - * - * @author Vikram Raman (vikram@wavefront.com). - */ -public class WavefrontDirectSenderTest { - - @Test - public void testPointToString() { - assertNull(WavefrontDirectSender.pointToString(null, 0.0, null, "source", null)); - assertNull(WavefrontDirectSender.pointToString("name", 0.0, null, null, null)); - - Assert.assertEquals("\"name\" 10.0 1469751813 source=\"source\" \"foo\"=\"bar\" \"bar\"=\"foo\"", - WavefrontDirectSender.pointToString("name",10L, 1469751813L, "source", - ImmutableMap.of("foo", "bar", "bar", "foo"))); - } -} diff --git a/java-client/src/test/java/com/wavefront/integrations/WavefrontTest.java b/java-client/src/test/java/com/wavefront/integrations/WavefrontTest.java deleted file mode 100644 index c38d705bc..000000000 --- a/java-client/src/test/java/com/wavefront/integrations/WavefrontTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.wavefront.integrations; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * Tests for {@link Wavefront} - * - * @author Clement Pang (clement@wavefront.com). - */ -public class WavefrontTest { - - @Test - public void testSanitize() { - assertEquals("\"hello\"", Wavefront.sanitize("hello")); - assertEquals("\"hello-world\"", Wavefront.sanitize("hello world")); - assertEquals("\"hello.world\"", Wavefront.sanitize("hello.world")); - assertEquals("\"hello\\\"world\\\"\"", Wavefront.sanitize("hello\"world\"")); - assertEquals("\"hello'world\"", Wavefront.sanitize("hello'world")); - } -} diff --git a/java-lib/pom.xml b/java-lib/pom.xml deleted file mode 100644 index 0d8b1165c..000000000 --- a/java-lib/pom.xml +++ /dev/null @@ -1,159 +0,0 @@ - - - 4.0.0 - - java-lib - - - com.wavefront - wavefront - 4.35-SNAPSHOT - - - Wavefront Java Libraries - - - - com.google.guava - guava - - - javax.validation - validation-api - 1.1.0.Final - - - joda-time - joda-time - - - com.fasterxml.jackson.core - jackson-databind - - - org.jboss.resteasy - resteasy-jaxrs - - - com.fasterxml.jackson.core - jackson-core - - - com.yammer.metrics - metrics-core - - - commons-lang - commons-lang - - - commons-collections - commons-collections - - - junit - junit - test - - - com.google.truth - truth - test - - - javax.ws.rs - javax.ws.rs-api - - - com.squareup.okhttp3 - okhttp - 3.8.1 - - - org.apache.avro - avro - - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - ${netty.version} - linux-x86_64 - - - org.antlr - antlr4-runtime - - - com.google.code.findbugs - jsr305 - - - - net.razorvine - pyrolite - 4.10 - - - com.tdunning - t-digest - 3.2 - - - io.dropwizard.metrics - metrics-core - 3.1.2 - - - io.dropwizard.metrics5 - metrics-core - 5.0.0-rc2 - - - - - - - org.apache.avro - avro-maven-plugin - 1.8.2 - - String - ${project.basedir}/src/main/resources/wavefront/templates/ - ${project.basedir}/src/main/avro - - - - schemas - generate-sources - - idl-protocol - - - - - - org.antlr - antlr4-maven-plugin - 4.7.1 - - target/generated-sources/antlr4/queryserver/parser - ${project.basedir}/src/main/antlr4 - - -visitor - - - - - - antlr4 - - - - - - - diff --git a/java-lib/src/main/antlr4/DSLexer.g4 b/java-lib/src/main/antlr4/DSLexer.g4 deleted file mode 100644 index 4e0f859a1..000000000 --- a/java-lib/src/main/antlr4/DSLexer.g4 +++ /dev/null @@ -1,125 +0,0 @@ -lexer grammar DSLexer; - -EQ - : '=' - ; - -NEQ - : '!=' - ; - -IpV4Address - : Octet '.' Octet '.' Octet '.' Octet - ; - -MinusSign - : '-' - ; - -PlusSign - : '+' - ; - - -IpV6Address - : ('::')? ((Segment ':') | (Segment '::'))+ (Segment | (Segment '::')) - | '::' - | '::' Segment ('::')? - | ('::')? Segment '::' - | ('::')? ((Segment '::') - | Segment ':')+ IpV4Address - | '::' IpV4Address - ; - -// negative numbers are not accounted for here since we need to -// handle for instance 5 - 6 (and not consume the minus sign into the number making it just two numbers). -Number - : Digit+ ('.' Digit+)? (('e' | 'E') (MinusSign | PlusSign)? Digit+)? - | '.' Digit+ (('e' | 'E') (MinusSign | PlusSign)? Digit+)? - ; - -Letters - : Letter+ Digit* - ; - -Quoted - : '"' ( '\\"' | . )*? '"' - | '\'' ( '\\\'' | . )*? '\'' - ; - -Literal - : '~'? Letter (Letter - | Digit - | '.' - | '-' - | '_' - | '|' - | '~' - | '{' - | '}' - | SLASH - | STAR - | DELTA - | AT)* - ; - -// Special token that we do allow for tag values. -RelaxedLiteral - : (Letter | Digit) (Letter - | Digit - | '.' - | '-' - | '_' - | '|' - | '~' - | '{' - | '}')* - ; - -BinType - : '!M' - | '!H' - | '!D' - ; - -Weight - : '#' Number - ; - -fragment -Letter - : 'a'..'z' - | 'A'..'Z' - ; - -fragment -Digit - : '0'..'9' - ; - -fragment -Hex - : 'a'..'f' - | 'A'..'F' - | Digit - ; - -fragment -Segment - : Hex Hex Hex Hex - | Hex Hex Hex - | Hex Hex - | Hex - ; - -fragment -Octet - : ('1'..'9') (('0'..'9') ('0'..'9')?)? - | '0' - ; - -STAR : '*' ; -SLASH : '/' ; -AT : '@'; -DELTA : '\u2206' | '\u0394'; -WS : [ \t\r\n]+ -> channel(HIDDEN) ; \ No newline at end of file diff --git a/java-lib/src/main/antlr4/DSWrapper.g4 b/java-lib/src/main/antlr4/DSWrapper.g4 deleted file mode 100644 index 1b1162326..000000000 --- a/java-lib/src/main/antlr4/DSWrapper.g4 +++ /dev/null @@ -1,11 +0,0 @@ -grammar DSWrapper; - -import DSLexer; - -@header { - package queryserver.parser; -} - -program - : EOF - ; \ No newline at end of file diff --git a/java-lib/src/main/avro/Reporting.avdl b/java-lib/src/main/avro/Reporting.avdl deleted file mode 100644 index 171728515..000000000 --- a/java-lib/src/main/avro/Reporting.avdl +++ /dev/null @@ -1,66 +0,0 @@ -// NOTE: talk to panghy before changing this file. -@namespace("wavefront.report") -protocol Reporting { - enum HistogramType { - TDIGEST, DOUBLE_TRUNCATE - } - - record Histogram { - // Number of milliseconds that samples cover - int duration; - // Histogram is a list of sample bins and counts - HistogramType type; - array bins; - array counts; - } - - record ReportPoint { - string metric; - // Milliseconds since 1970 - long timestamp; - union { double, long, string, Histogram } value; - string host = "unknown"; - string table = "tsdb"; - map annotations = {}; - } - - record Annotation { - string key; - string value; - } - - record Span { - // name of the span (expecting low cardinality, e.g. "checkout", "getAlerts") - string name; - // uuid of the span - string spanId; - // uuid of the trace - string traceId; - // start millis of the span - long startMillis; - // duration of the span - long duration; - // source (host) of the span - string source = "unknown"; - // the customer of the span - string customer; - // annotations (indexed and unindexed). - array annotations = {}; - } - - // The parts of a ReportPoint that uniquely identify a timeseries to wavefront. - record TimeSeries { - string metric; - string host = "unknown"; - string table = "tsdb"; - @java-class("java.util.TreeMap") map annotations = {}; - } - - record ReportSourceTag { - string sourceTagLiteral; // constant '@SourceTag' or '@SourceDescription' - string action; // can be either 'save' or 'delete' - string source; - union {null, string} description; - array annotations = {}; // might be empty - } -} \ No newline at end of file diff --git a/java-lib/src/main/java/com/codahale/metrics/DeltaCounter.java b/java-lib/src/main/java/com/codahale/metrics/DeltaCounter.java deleted file mode 100644 index b61497d1d..000000000 --- a/java-lib/src/main/java/com/codahale/metrics/DeltaCounter.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.codahale.metrics; - -import com.google.common.annotations.VisibleForTesting; - -import com.wavefront.common.MetricConstants; - -/** - * A counter for Wavefront delta metrics. - * - * Differs from a counter in that it is reset in the WavefrontReporter every time the value is reported. - * - * @author Vikram Raman (vikram@wavefront.com) - */ -public class DeltaCounter extends Counter { - - @VisibleForTesting - public static synchronized DeltaCounter get(MetricRegistry registry, String metricName) { - - if (registry == null || metricName == null || metricName.isEmpty()) { - throw new IllegalArgumentException("Invalid arguments"); - } - - if (!(metricName.startsWith(MetricConstants.DELTA_PREFIX) || metricName.startsWith(MetricConstants.DELTA_PREFIX_2))) { - metricName = MetricConstants.DELTA_PREFIX + metricName; - } - DeltaCounter counter = new DeltaCounter(); - registry.register(metricName, counter); - return counter; - } -} \ No newline at end of file diff --git a/java-lib/src/main/java/com/codahale/metrics/jvm/SafeFileDescriptorRatioGauge.java b/java-lib/src/main/java/com/codahale/metrics/jvm/SafeFileDescriptorRatioGauge.java deleted file mode 100644 index 85268f8db..000000000 --- a/java-lib/src/main/java/com/codahale/metrics/jvm/SafeFileDescriptorRatioGauge.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.codahale.metrics.jvm; - -import com.codahale.metrics.RatioGauge; -import com.sun.management.UnixOperatingSystemMXBean; - -import java.lang.management.ManagementFactory; -import java.lang.management.OperatingSystemMXBean; - -/** - * Java 9 compatible implementation of FileDescriptorRatioGauge that doesn't use reflection - * and is not susceptible to an InaccessibleObjectException - * - * The gauge represents a ratio of used to total file descriptors. - * - * @author Vasily Vorontsov (vasily@wavefront.com) - */ -public class SafeFileDescriptorRatioGauge extends RatioGauge { - private final OperatingSystemMXBean os; - - /** - * Creates a new gauge using the platform OS bean. - */ - public SafeFileDescriptorRatioGauge() { - this(ManagementFactory.getOperatingSystemMXBean()); - } - - /** - * Creates a new gauge using the given OS bean. - * - * @param os an {@link OperatingSystemMXBean} - */ - public SafeFileDescriptorRatioGauge(OperatingSystemMXBean os) { - this.os = os; - } - - /** - * @return {@link com.codahale.metrics.RatioGauge.Ratio} of used to total file descriptors. - */ - protected Ratio getRatio() { - if (!(this.os instanceof UnixOperatingSystemMXBean)) { - return Ratio.of(Double.NaN, Double.NaN); - } - Long openFds = ((UnixOperatingSystemMXBean)os).getOpenFileDescriptorCount(); - Long maxFds = ((UnixOperatingSystemMXBean)os).getMaxFileDescriptorCount(); - return Ratio.of(openFds.doubleValue(), maxFds.doubleValue()); - } -} diff --git a/java-lib/src/main/java/com/wavefront/api/DataIngesterAPI.java b/java-lib/src/main/java/com/wavefront/api/DataIngesterAPI.java deleted file mode 100644 index bffa43d66..000000000 --- a/java-lib/src/main/java/com/wavefront/api/DataIngesterAPI.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.wavefront.api; - -import java.io.IOException; -import java.io.InputStream; - -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -/** - * The API for reporting points directly to a Wavefront server. - * - * @author Vikram Raman - */ -@Path("/") -public interface DataIngesterAPI { - - @POST - @Path("report") - @Consumes({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_FORM_URLENCODED, - MediaType.TEXT_PLAIN}) - Response report(@QueryParam("f") String format, InputStream stream) throws IOException; -} diff --git a/java-lib/src/main/java/com/wavefront/api/WavefrontAPI.java b/java-lib/src/main/java/com/wavefront/api/WavefrontAPI.java deleted file mode 100644 index 3b12f3d80..000000000 --- a/java-lib/src/main/java/com/wavefront/api/WavefrontAPI.java +++ /dev/null @@ -1,169 +0,0 @@ -package com.wavefront.api; - -import com.fasterxml.jackson.databind.JsonNode; -import com.wavefront.api.agent.AgentConfiguration; -import com.wavefront.api.agent.ShellOutputDTO; -import org.jboss.resteasy.annotations.GZIP; - -import javax.validation.Valid; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import java.util.List; -import java.util.UUID; - -/** - * API for the Agent. - * - * @author Clement Pang (clement@wavefront.com) - */ -@Path("/") -public interface WavefrontAPI { - - /** - * Polls for the configuration for the agent. - * - * @param agentId Agent id to poll for configuration. - * @param hostname Hostname of the agent. - * @param currentMillis Current millis on the agent (to adjust for timing). - * @param token Token to auto-register the agent. - * @param version Version of the agent. - * @return Configuration for the agent. - */ - @GET - @Path("daemon/{agentId}/config") - @Produces(MediaType.APPLICATION_JSON) - AgentConfiguration getConfig(@PathParam("agentId") UUID agentId, - @QueryParam("hostname") String hostname, - @QueryParam("currentMillis") final Long currentMillis, - @QueryParam("bytesLeftForBuffer") Long bytesLeftForbuffer, - @QueryParam("bytesPerMinuteForBuffer") Long bytesPerMinuteForBuffer, - @QueryParam("currentQueueSize") Long currentQueueSize, - @QueryParam("token") String token, - @QueryParam("version") String version); - - @POST - @Path("daemon/{sshDaemonId}/checkin") - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) - AgentConfiguration checkin(@PathParam("sshDaemonId") final UUID agentId, - @QueryParam("hostname") String hostname, - @QueryParam("token") String token, - @QueryParam("version") String version, - @QueryParam("currentMillis") final Long currentMillis, - @QueryParam("local") Boolean localAgent, - @GZIP JsonNode agentMetrics, - @QueryParam("push") Boolean pushAgent, - @QueryParam("ephemeral") Boolean ephemeral); - - - /** - * Post batched data from pushed data (graphitehead, statsd) that was proxied through the collector. - * - * @param agentId Agent Id of the agent reporting the result. - * @param workUnitId Work unit that the agent is reporting. - * @param currentMillis Current millis on the agent (to adjust for timing). - * @param format The format of the data - * @param pushData The batched push data - */ - @POST - @Consumes(MediaType.TEXT_PLAIN) - @Path("daemon/{agentId}/pushdata/{workUnitId}") - Response postPushData(@PathParam("agentId") UUID agentId, - @PathParam("workUnitId") UUID workUnitId, - @Deprecated @QueryParam("currentMillis") Long currentMillis, - @QueryParam("format") String format, - @GZIP String pushData); - - /** - * Reports an error that occured in the agent. - * - * @param agentId Agent reporting the error. - * @param details Details of the error. - */ - @POST - @Path("daemon/{agentId}/error") - void agentError(@PathParam("agentId") UUID agentId, - @FormParam("details") String details); - - @POST - @Path("daemon/{agentId}/config/processed") - void agentConfigProcessed(@PathParam("agentId") UUID agentId); - - /** - * Post work unit results from an agent executing a particular work unit on a host machine. - * - * @param agentId Agent Id of the agent reporting the result. - * @param workUnitId Work unit that the agent is reporting. - * @param targetId The target that's reporting the result. - * @param shellOutputDTO The output of running the work unit. - */ - @POST - @Consumes(MediaType.APPLICATION_JSON) - @Path("daemon/{agentId}/workunit/{workUnitId}/{hostId}") - Response postWorkUnitResult(@PathParam("agentId") UUID agentId, - @PathParam("workUnitId") UUID workUnitId, - @PathParam("hostId") UUID targetId, - @GZIP @Valid ShellOutputDTO shellOutputDTO); - - /** - * Reports that a host has failed to connect. - * - * @param agentId Agent reporting the error. - * @param hostId Host that is experiencing connection issues. - * @param details Details of the error. - */ - @POST - @Path("daemon/{agentId}/host/{hostId}/fail") - void hostConnectionFailed(@PathParam("agentId") UUID agentId, - @PathParam("hostId") UUID hostId, - @FormParam("details") String details); - - /** - * Reports that a connection to a host has been established. - * - * @param agentId Agent reporting the event. - * @param hostId Host. - */ - @POST - @Path("daemon/{agentId}/host/{hostId}/connect") - void hostConnectionEstablished(@PathParam("agentId") UUID agentId, - @PathParam("hostId") UUID hostId); - - /** - * Reports that an auth handshake to a host has been completed. - * - * @param agentId Agent reporting the event. - * @param hostId Host. - */ - @POST - @Path("daemon/{agentId}/host/{hostId}/auth") - void hostAuthenticated(@PathParam("agentId") UUID agentId, - @PathParam("hostId") UUID hostId); - - @DELETE - @Path("v2/source/{id}/tag/{tagValue}") - @Produces(MediaType.APPLICATION_JSON) - Response removeTag(@PathParam("id") String id, @QueryParam("t") String token, - @PathParam("tagValue") String tagValue); - - @DELETE - @Path("v2/source/{id}/description") - @Produces(MediaType.APPLICATION_JSON) - Response removeDescription(@PathParam("id") String id, @QueryParam("t") String token); - - @POST - @Path("v2/source/{id}/tag") - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) - Response setTags(@PathParam ("id") String id, @QueryParam("t") String token, - List tagValuesToSet); - - @POST - @Path("v2/source/{id}/description") - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) - Response setDescription(@PathParam("id") String id, @QueryParam("t") String token, - String description); -} diff --git a/java-lib/src/main/java/com/wavefront/api/agent/AgentConfiguration.java b/java-lib/src/main/java/com/wavefront/api/agent/AgentConfiguration.java deleted file mode 100644 index 3dabd952d..000000000 --- a/java-lib/src/main/java/com/wavefront/api/agent/AgentConfiguration.java +++ /dev/null @@ -1,206 +0,0 @@ -package com.wavefront.api.agent; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; - -import java.io.File; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.UUID; - -/** - * Configuration for the SSH Daemon. - * - * @author Clement Pang (clement@sunnylabs.com) - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class AgentConfiguration { - - public String name; - public String defaultUsername; - public String defaultPublicKey; - public boolean allowAnyHostKeys; - public Long currentTime; - private List targets; - private List workUnits; - private Boolean collectorSetsPointsPerBatch; - private Long pointsPerBatch; - private Boolean collectorSetsRetryBackoff; - private Double retryBackoffBaseSeconds; - private Boolean collectorSetsRateLimit; - private Long collectorRateLimit; - private Boolean shutOffAgents = false; - private Boolean showTrialExpired = false; - // If the value is true, then histogram feature is disabled. - // If the value is null or false, then histograms are not disabled i.e. enabled (default behavior) - private Boolean histogramDisabled; - // If the value is true, then trace feature is disabled; feature enabled if the value is null or false - private Boolean traceDisabled; - - public Boolean getCollectorSetsRetryBackoff() { - return collectorSetsRetryBackoff; - } - - public void setCollectorSetsRetryBackoff(Boolean collectorSetsRetryBackoff) { - this.collectorSetsRetryBackoff = collectorSetsRetryBackoff; - } - - public Double getRetryBackoffBaseSeconds() { - return retryBackoffBaseSeconds; - } - - public void setRetryBackoffBaseSeconds(Double retryBackoffBaseSeconds) { - this.retryBackoffBaseSeconds = retryBackoffBaseSeconds; - } - - public Boolean getCollectorSetsRateLimit() { - return this.collectorSetsRateLimit; - } - - public void setCollectorSetsRateLimit(Boolean collectorSetsRateLimit) { - this.collectorSetsRateLimit = collectorSetsRateLimit; - } - - public Long getCollectorRateLimit() { - return this.collectorRateLimit; - } - - public void setCollectorRateLimit(Long collectorRateLimit) { - this.collectorRateLimit = collectorRateLimit; - } - - public List getWorkUnits() { - if (workUnits == null) return Collections.emptyList(); - return workUnits; - } - - public List getTargets() { - if (targets == null) return Collections.emptyList(); - return targets; - } - - public void setCollectorSetsPointsPerBatch(Boolean collectorSetsPointsPerBatch) { - this.collectorSetsPointsPerBatch = collectorSetsPointsPerBatch; - } - - public Boolean getCollectorSetsPointsPerBatch() { - return collectorSetsPointsPerBatch; - } - - public void setTargets(List targets) { - this.targets = targets; - } - - public void setWorkUnits(List workUnits) { - this.workUnits = workUnits; - } - - public Long getPointsPerBatch() { - return pointsPerBatch; - } - - public void setPointsPerBatch(Long pointsPerBatch) { - this.pointsPerBatch = pointsPerBatch; - } - - public Boolean getShutOffAgents() { return shutOffAgents; } - - public void setShutOffAgents(Boolean shutOffAgents) { - this.shutOffAgents = shutOffAgents; - } - - public Boolean getShowTrialExpired() { return showTrialExpired; } - - public void setShowTrialExpired(Boolean trialExpired) { - this.showTrialExpired = trialExpired; - } - - public Boolean getHistogramDisabled() { - return histogramDisabled; - } - - public void setHistogramDisabled(Boolean histogramDisabled) { - this.histogramDisabled = histogramDisabled; - } - - public Boolean getTraceDisabled() { - return this.traceDisabled; - } - - public void setTraceDisabled(Boolean traceDisabled) { - this.traceDisabled = traceDisabled; - } - - public void validate(boolean local) { - Set knownHostUUIDs = Collections.emptySet(); - if (targets != null) { - if (defaultPublicKey != null) { - Preconditions.checkArgument(new File(defaultPublicKey).exists(), "defaultPublicKey does not exist"); - } - knownHostUUIDs = Sets.newHashSetWithExpectedSize(targets.size()); - for (SshTargetDTO target : targets) { - Preconditions.checkNotNull(target, "target cannot be null"); - target.validate(this); - Preconditions.checkState(knownHostUUIDs.add(target.id), "duplicate target id: " + target.id); - if (target.user == null) { - Preconditions.checkNotNull(defaultUsername, - "must have default username if user is not specified, host entry: " + target.host); - } - if (target.publicKey == null) { - Preconditions.checkNotNull(defaultPublicKey, - "must have default publickey if publicKey is not specified, host entry: " + target.host); - } - if (!allowAnyHostKeys) { - Preconditions.checkNotNull(target.hostKey, "must specify hostKey if " + - "'allowAnyHostKeys' is set to false, host entry: " + target.host); - } - } - } - if (workUnits != null) { - for (WorkUnit unit : workUnits) { - Preconditions.checkNotNull(unit, "workUnit cannot be null"); - unit.validate(); - if (!local) { - Preconditions.checkState(knownHostUUIDs.containsAll(unit.targets), "workUnit: " + - unit.name + " refers to a target host that does not exist"); - } - } - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - AgentConfiguration that = (AgentConfiguration) o; - - if (allowAnyHostKeys != that.allowAnyHostKeys) return false; - if (defaultPublicKey != null ? !defaultPublicKey.equals(that.defaultPublicKey) : that.defaultPublicKey != null) - return false; - if (defaultUsername != null ? !defaultUsername.equals(that.defaultUsername) : that.defaultUsername != null) - return false; - if (name != null ? !name.equals(that.name) : that.name != null) return false; - if (targets != null ? !targets.equals(that.targets) : that.targets != null) return false; - if (workUnits != null ? !workUnits.equals(that.workUnits) : that.workUnits != null) - return false; - - return true; - } - - @Override - public int hashCode() { - int result = name != null ? name.hashCode() : 0; - result = 31 * result + (defaultUsername != null ? defaultUsername.hashCode() : 0); - result = 31 * result + (defaultPublicKey != null ? defaultPublicKey.hashCode() : 0); - result = 31 * result + (allowAnyHostKeys ? 1 : 0); - result = 31 * result + (targets != null ? targets.hashCode() : 0); - result = 31 * result + (workUnits != null ? workUnits.hashCode() : 0); - return result; - } -} diff --git a/java-lib/src/main/java/com/wavefront/api/agent/Constants.java b/java-lib/src/main/java/com/wavefront/api/agent/Constants.java deleted file mode 100644 index 53fd60e0d..000000000 --- a/java-lib/src/main/java/com/wavefront/api/agent/Constants.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.wavefront.api.agent; - -import java.util.UUID; - -/** - * Agent MetricConstants. - * - * @author Clement Pang (clement@wavefront.com) - */ -public abstract class Constants { - - /** - * Formatted for graphite head - */ - public static final String PUSH_FORMAT_GRAPHITE = "graphite"; - /** - * Formatted for graphite head (without customer id in the metric name). - */ - public static final String PUSH_FORMAT_GRAPHITE_V2 = "graphite_v2"; - public static final String PUSH_FORMAT_WAVEFRONT = "wavefront"; // alias for graphite_v2 - - /** - * Wavefront histogram format - */ - public static final String PUSH_FORMAT_HISTOGRAM = "histogram"; - - /** - * Wavefront tracing format - */ - public static final String PUSH_FORMAT_TRACING = "trace"; - - /** - * Work unit id for blocks of graphite-formatted data. - */ - public static final UUID GRAPHITE_BLOCK_WORK_UNIT = - UUID.fromString("12b37289-90b2-4b98-963f-75a27110b8da"); -} diff --git a/java-lib/src/main/java/com/wavefront/api/agent/MetricStage.java b/java-lib/src/main/java/com/wavefront/api/agent/MetricStage.java deleted file mode 100644 index bce99c32c..000000000 --- a/java-lib/src/main/java/com/wavefront/api/agent/MetricStage.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.wavefront.api.agent; - -/** - * What stage of development is this metric in? This is intended for public consumption. - * - * @author Clement Pang (clement@wavefront.com) - */ -public enum MetricStage { - TRIAL, // Should only be run once on a target, unless it's changed - PER_FETCH, // Should be run once each time the daemon phones home - ACTIVE // Should be run in a continuous loop, based on delay -} diff --git a/java-lib/src/main/java/com/wavefront/api/agent/ShellOutputDTO.java b/java-lib/src/main/java/com/wavefront/api/agent/ShellOutputDTO.java deleted file mode 100644 index a7a8f1136..000000000 --- a/java-lib/src/main/java/com/wavefront/api/agent/ShellOutputDTO.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.wavefront.api.agent; - -import com.wavefront.api.json.InstantMarshaller; -import org.codehaus.jackson.map.annotate.JsonDeserialize; -import org.codehaus.jackson.map.annotate.JsonSerialize; -import org.joda.time.Instant; - -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Null; -import javax.validation.groups.Default; -import java.io.Serializable; -import java.util.UUID; - -/** - * A POJO representing the shell output from running commands in a work unit. The {@link Default} - * validation group is intended for submission from the daemon to the server. - * - * @author Clement Pang (clement@wavefront.com). - */ -public class ShellOutputDTO implements Serializable { - @NotNull - public UUID id; - @NotNull - public UUID targetId; - /** - * Computed by the server. - */ - @Null(groups = Default.class) - public UUID machineId; - @NotNull - public UUID workUnitId; - @NotNull - public UUID sshDaemonId; - @NotNull - public String output; - @NotNull - public Integer exitCode; - @NotNull - @JsonSerialize(using = InstantMarshaller.Serializer.class) - @JsonDeserialize(using = InstantMarshaller.Deserializer.class) - public Instant commandStartTime; - @NotNull - @JsonSerialize(using = InstantMarshaller.Serializer.class) - @JsonDeserialize(using = InstantMarshaller.Deserializer.class) - public Instant commandEndTime; - /** - * Filled-in by the server. - */ - @Null(groups = Default.class) - @JsonSerialize(using = InstantMarshaller.Serializer.class) - @JsonDeserialize(using = InstantMarshaller.Deserializer.class) - public Instant serverTime; - /** - * Filled-in by the server. - */ - @Null(groups = Default.class) - public String customerId; -} diff --git a/java-lib/src/main/java/com/wavefront/api/agent/SshTargetDTO.java b/java-lib/src/main/java/com/wavefront/api/agent/SshTargetDTO.java deleted file mode 100644 index ad3d43096..000000000 --- a/java-lib/src/main/java/com/wavefront/api/agent/SshTargetDTO.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.wavefront.api.agent; - -import com.google.common.base.Preconditions; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -import java.util.UUID; - -/** - * Represents an SSH target to connect to. - * - * @author Clement Pang (clement@sunnylabs.com) - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class SshTargetDTO { - public UUID id; - public String host; - public int port = 22; - public String hostKey; - public String user; - public String publicKey; - - public void validate(AgentConfiguration config) { - Preconditions.checkNotNull(id, "id cannot be null for host"); - Preconditions.checkNotNull(host, "host cannot be null"); - Preconditions.checkState(port > 0, "port must be greater than 0"); - Preconditions.checkNotNull(publicKey, "publicKey cannot be null"); - if (user == null) user = config.defaultUsername; - if (publicKey == null) publicKey = config.defaultPublicKey; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - SshTargetDTO sshTargetDTO = (SshTargetDTO) o; - - if (port != sshTargetDTO.port) return false; - if (host != null ? !host.equals(sshTargetDTO.host) : sshTargetDTO.host != null) return false; - if (hostKey != null ? !hostKey.equals(sshTargetDTO.hostKey) : sshTargetDTO.hostKey != null) - return false; - if (!id.equals(sshTargetDTO.id)) return false; - if (publicKey != null ? !publicKey.equals(sshTargetDTO.publicKey) : sshTargetDTO.publicKey != null) - return false; - if (user != null ? !user.equals(sshTargetDTO.user) : sshTargetDTO.user != null) return false; - - return true; - } - - @Override - public int hashCode() { - int result = id.hashCode(); - result = 31 * result + (host != null ? host.hashCode() : 0); - result = 31 * result + port; - result = 31 * result + (hostKey != null ? hostKey.hashCode() : 0); - result = 31 * result + (user != null ? user.hashCode() : 0); - result = 31 * result + (publicKey != null ? publicKey.hashCode() : 0); - return result; - } -} diff --git a/java-lib/src/main/java/com/wavefront/api/agent/WorkUnit.java b/java-lib/src/main/java/com/wavefront/api/agent/WorkUnit.java deleted file mode 100644 index c36ca2140..000000000 --- a/java-lib/src/main/java/com/wavefront/api/agent/WorkUnit.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.wavefront.api.agent; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -import java.util.Set; -import java.util.UUID; - -/** - * A work unit to execute. - * - * @author Clement Pang (clement@sunnylabs.com) - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class WorkUnit { - /** - * Unique id for the work unit (used for reporting). - */ - public UUID id; - /** - * Friendly name for the work unit. - */ - public String name; - /** - * Seconds between work unit executions. - */ - public long delay; - /** - * Command to execute. - */ - public String command; - /** - * Stage of this work unit -- trial, per-fetch, or active. - */ - public MetricStage stage; - /** - * Targets that participate in this work unit. - */ - public Set targets = Sets.newHashSet(); - - public void validate() { - Preconditions.checkNotNull(id, "id cannot be null for a work unit"); - Preconditions.checkNotNull(name, "name cannot be null for a work unit"); - Preconditions.checkNotNull(targets, "targets cannot be null for work unit: " + name); - Preconditions.checkNotNull(command, "command must not be null for work unit: " + name); - Preconditions.checkNotNull(stage, "stage cannot be null for work unit: " + name); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - WorkUnit workUnit = (WorkUnit) o; - - if (delay != workUnit.delay) return false; - if (command != null ? !command.equals(workUnit.command) : workUnit.command != null) - return false; - if (!id.equals(workUnit.id)) return false; - if (name != null ? !name.equals(workUnit.name) : workUnit.name != null) return false; - if (targets != null ? !targets.equals(workUnit.targets) : workUnit.targets != null) - return false; - if (stage != null ? !stage.equals(workUnit.stage) : workUnit.stage != null) return false; - - return true; - } - - @Override - public int hashCode() { - int result = id.hashCode(); - result = 31 * result + (name != null ? name.hashCode() : 0); - result = 31 * result + (int) (delay ^ (delay >>> 32)); - result = 31 * result + (command != null ? command.hashCode() : 0); - result = 31 * result + (targets != null ? targets.hashCode() : 0); - result = 31 * result + (stage != null ? stage.hashCode() : 0); - return result; - } - - public WorkUnit clone() { - WorkUnit cloned = new WorkUnit(); - cloned.delay = this.delay; - cloned.name = this.name; - cloned.command = this.command; - cloned.targets = Sets.newHashSet(this.targets); - cloned.stage = this.stage; - cloned.id = this.id; - return cloned; - } -} diff --git a/java-lib/src/main/java/com/wavefront/api/json/InstantMarshaller.java b/java-lib/src/main/java/com/wavefront/api/json/InstantMarshaller.java deleted file mode 100644 index a8a8eaee5..000000000 --- a/java-lib/src/main/java/com/wavefront/api/json/InstantMarshaller.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.wavefront.api.json; - -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.map.DeserializationContext; -import org.codehaus.jackson.map.JsonDeserializer; -import org.codehaus.jackson.map.JsonSerializer; -import org.codehaus.jackson.map.SerializerProvider; -import org.joda.time.Instant; - -import java.io.IOException; - -/** - * Marshaller for Joda Instant to JSON and back. - * - * @author Clement Pang (clement@wavefront.com) - */ -public class InstantMarshaller { - - public static class Serializer extends JsonSerializer { - - @Override - public void serialize(Instant value, JsonGenerator jgen, SerializerProvider provider) throws IOException { - jgen.writeNumber(value.getMillis()); - } - } - - - public static class Deserializer extends JsonDeserializer { - @Override - public Instant deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { - return new Instant(jp.getLongValue()); - } - } -} diff --git a/java-lib/src/main/java/com/wavefront/common/Clock.java b/java-lib/src/main/java/com/wavefront/common/Clock.java deleted file mode 100644 index 576b931dc..000000000 --- a/java-lib/src/main/java/com/wavefront/common/Clock.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.wavefront.common; - -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Gauge; -import com.yammer.metrics.core.MetricName; - -/** - * Clock to manage agent time synced with the server. - * - * @author Clement Pang (clement@wavefront.com). - */ -public abstract class Clock { - private static Long serverTime; - private static Long localTime; - private static Long clockDrift; - - private static Gauge clockDriftGauge; - - public static void set(long serverTime) { - localTime = System.currentTimeMillis(); - Clock.serverTime = serverTime; - clockDrift = serverTime - localTime; - if (clockDriftGauge == null) { - // doesn't have to be synchronized, ok to initialize clockDriftGauge more than once - clockDriftGauge = Metrics.newGauge(new MetricName("clock", "", "drift"), new Gauge() { - @Override - public Long value() { - return clockDrift == null ? null : (long)Math.floor(clockDrift / 1000 + 0.5d); - } - }); - } - } - - public static long now() { - if (serverTime == null) return System.currentTimeMillis(); - else return (System.currentTimeMillis() - localTime) + serverTime; - } -} diff --git a/java-lib/src/main/java/com/wavefront/common/MetricConstants.java b/java-lib/src/main/java/com/wavefront/common/MetricConstants.java deleted file mode 100644 index e91a9b8e7..000000000 --- a/java-lib/src/main/java/com/wavefront/common/MetricConstants.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.wavefront.common; - -/** - * Metric constants. - * - * @author Vikram Raman (vikram@wavefront.com) - */ -public abstract class MetricConstants { - public static final String DELTA_PREFIX = "\u2206"; // ∆: INCREMENT - public static final String DELTA_PREFIX_2 = "\u0394"; // Δ: GREEK CAPITAL LETTER DELTA -} diff --git a/java-lib/src/main/java/com/wavefront/common/MetricMangler.java b/java-lib/src/main/java/com/wavefront/common/MetricMangler.java deleted file mode 100644 index 592367707..000000000 --- a/java-lib/src/main/java/com/wavefront/common/MetricMangler.java +++ /dev/null @@ -1,178 +0,0 @@ -package com.wavefront.common; - -import com.google.common.base.Splitter; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import javax.annotation.Nullable; - -/** - * Handles updating the metric and source names by extracting components from the metric name. - * There are several options considered: - *

    - *
  • source name:
    • - *
      - *
    • extracted from one or more components of the metric name - * (where each component is separated by a '.')
      • - *
    • allow characters to be optionally replaced in the components extracted - * as source name with '.'
      • - *
    - *
  • metric name:
    • - *
      - *
    • remove components (in addition to the source name) - * (this allows things like 'hosts.sjc1234.cpu.loadavg.1m' to get - * changed to cpu.loadavg.1m after extracting sjc1234)
      • - *
    - *
- * This code was originally mostly contained in GraphiteFormatter class and moved into a single - * re-usable class. - * @author Mike McLaughlin (mike@wavefront.com) - */ -public class MetricMangler { - - // Fields to extract and assemble, in order, as the host name - private final List hostIndices = new ArrayList<>(); - private int maxField = 0; - - // Lookup set for which indices are hostname related - private final Set hostIndexSet = new HashSet<>(); - - // Characters which should be interpreted as dots - @Nullable - private final String delimiters; - - // Fields to remove - private final Set removeIndexSet = new HashSet<>(); - - /** - * Constructor. - * - * @param sourceFields comma separated field index(es) (1-based) where the source name will be - * extracted - * @param delimiters characters to be interpreted as dots - * @param removeFields comma separated field index(es) (1-based) of fields to remove from the - * metric name - * @throws IllegalArgumentException when one of the field index is <= 0 - */ - public MetricMangler(@Nullable String sourceFields, - @Nullable String delimiters, - @Nullable String removeFields) { - if (sourceFields != null) { - // Store ordered field indices and lookup set - Iterable fields = Splitter.on(",").omitEmptyStrings().trimResults().split(sourceFields); - for (String field : fields) { - if (field.trim().length() > 0) { - int fieldIndex = Integer.parseInt(field); - if (fieldIndex <= 0) { - throw new IllegalArgumentException("Can't define a field of index 0 or less; indices must be 1-based"); - } - hostIndices.add(fieldIndex - 1); - hostIndexSet.add(fieldIndex - 1); - if (fieldIndex > maxField) { - maxField = fieldIndex; - } - } - } - } - - if (removeFields != null) { - Iterable fields = Splitter.on(",").omitEmptyStrings().trimResults().split(removeFields); - for (String field : fields) { - if (field.trim().length() > 0) { - int fieldIndex = Integer.parseInt(field); - if (fieldIndex <= 0) { - throw new IllegalArgumentException("Can't define a field to remove of index 0 or less; indices must be 1-based"); - } - removeIndexSet.add(fieldIndex - 1); - } - } - } - - // Store as-is; going to loop through chars anyway - this.delimiters = delimiters; - } - - /** - * Simple struct to store and return the source, annotations and the updated metric. - * - * @see {@link #extractComponents(String)} - */ - public static class MetricComponents { - @Nullable - public String source; - @Nullable - public String metric; - @Nullable - public String[] annotations; - } - - /** - * Extracts the source from the metric name and returns the new metric name and the source name. - * - * @param metric the metric name - * @return the updated metric name and the extracted source - * @throws IllegalArgumentException when the number of segments (split on '.') is less than the - * maximum source component index - */ - public MetricComponents extractComponents(final String metric) { - final String[] segments = metric.split("\\."); - final MetricComponents rtn = new MetricComponents(); - - // Is the metric name long enough? - if (segments.length < maxField) { - throw new IllegalArgumentException( - String.format("Metric data |%s| provided was incompatible with format.", metric)); - } - - // Assemble the newly shorn metric name, in original order - StringBuilder buf = new StringBuilder(); - for (int i = 0; i < segments.length; i++) { - final String segment = segments[i]; - if (!hostIndexSet.contains(i) && !removeIndexSet.contains(i)) { - if (buf.length() > 0) { - buf.append('.'); - } - buf.append(segment); - } - } - rtn.metric = buf.toString(); - - // Extract Graphite 1.1+ tags, if present - if (rtn.metric.indexOf(";") > 0) { - final String[] annotationSegments = rtn.metric.split(";"); - rtn.annotations = Arrays.copyOfRange(annotationSegments, 1, annotationSegments.length); - rtn.metric = annotationSegments[0]; - } - - // Loop over host components in configured order, and replace all delimiters with dots - if (hostIndices != null && !hostIndices.isEmpty()) { - buf = new StringBuilder(); - for (int f = 0; f < hostIndices.size(); f++) { - char[] segmentChars = segments[hostIndices.get(f)].toCharArray(); - if (delimiters != null && !delimiters.isEmpty()) { - for (int i = 0; i < segmentChars.length; i++) { - for (int c = 0; c < delimiters.length(); c++) { - if (segmentChars[i] == delimiters.charAt(c)) { - segmentChars[i] = '.'; // overwrite it - } - } - } - } - if (f > 0) { - // join host segments with dot, if you're after the first one - buf.append('.'); - } - buf.append(segmentChars); - } - rtn.source = buf.toString(); - } else { - rtn.source = null; - } - - return rtn; - } -} diff --git a/java-lib/src/main/java/com/wavefront/common/MetricWhiteBlackList.java b/java-lib/src/main/java/com/wavefront/common/MetricWhiteBlackList.java deleted file mode 100644 index 2c960b9cc..000000000 --- a/java-lib/src/main/java/com/wavefront/common/MetricWhiteBlackList.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.wavefront.common; - -import com.google.common.base.Predicate; - -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Counter; - -import org.apache.commons.lang.StringUtils; - -import java.util.regex.Pattern; - -import javax.annotation.Nullable; - -/** - * White/Black list checker for a metric. This code was originally contained within the ChannelStringHandler. This - * class was created for easy re-use by classes such as the ChannelByteArrayHandler. - * - * @author Mike McLaughlin (mike@wavefront.com) - */ -public class MetricWhiteBlackList implements Predicate { - @Nullable - private final Pattern pointLineWhiteList; - @Nullable - private final Pattern pointLineBlackList; - - /** - * Counter for number of rejected metrics. - */ - private final Counter regexRejects; - - /** - * Constructor. - * - * @param pointLineWhiteListRegex the white list regular expression. - * @param pointLineBlackListRegex the black list regular expression - * @param portName the port used as metric name (validationRegex.point-rejected [port=portName]) - */ - public MetricWhiteBlackList(@Nullable final String pointLineWhiteListRegex, - @Nullable final String pointLineBlackListRegex, - final String portName) { - - if (!StringUtils.isBlank(pointLineWhiteListRegex)) { - this.pointLineWhiteList = Pattern.compile(pointLineWhiteListRegex); - } else { - this.pointLineWhiteList = null; - } - if (!StringUtils.isBlank(pointLineBlackListRegex)) { - this.pointLineBlackList = Pattern.compile(pointLineBlackListRegex); - } else { - this.pointLineBlackList = null; - } - - this.regexRejects = Metrics.newCounter( - new TaggedMetricName("validationRegex", "points-rejected", "port", portName)); - } - - /** - * Check to see if the given point line or metric passes the white and black list. - * - * @param pointLine the line to check - * @return true if the line passes checks; false o/w - */ - @Override - public boolean apply(String pointLine) { - if ((pointLineWhiteList != null && !pointLineWhiteList.matcher(pointLine).matches()) || - (pointLineBlackList != null && pointLineBlackList.matcher(pointLine).matches())) { - regexRejects.inc(); - return false; - } - return true; - } -} diff --git a/java-lib/src/main/java/com/wavefront/common/MetricsToTimeseries.java b/java-lib/src/main/java/com/wavefront/common/MetricsToTimeseries.java deleted file mode 100644 index df30f0382..000000000 --- a/java-lib/src/main/java/com/wavefront/common/MetricsToTimeseries.java +++ /dev/null @@ -1,148 +0,0 @@ -package com.wavefront.common; - -import com.google.common.collect.ImmutableMap; - -import com.yammer.metrics.core.Metered; -import com.yammer.metrics.core.MetricName; -import com.yammer.metrics.core.Sampling; -import com.yammer.metrics.core.Summarizable; -import com.yammer.metrics.core.VirtualMachineMetrics; -import com.yammer.metrics.stats.Snapshot; - -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; - -/** - * @author Mori Bellamy (mori@wavefront.com) - */ -public abstract class MetricsToTimeseries { - - public static Map explodeSummarizable(Summarizable metric) { - return explodeSummarizable(metric, false); - } - - /** - * Retrieve values for pre-defined stats (min/max/mean/sum/stddev) from a {@link Summarizable} metric (e.g. histogram) - * - * @param metric metric to retrieve stats from - * @param convertNanToZero simulate {@link com.yammer.metrics.core.Histogram} histogram behavior when used with - * {@link com.yammer.metrics.core.WavefrontHistogram} as input: - * when "true", empty WavefrontHistogram reports zero values for all stats - * @return summarizable stats - */ - public static Map explodeSummarizable(Summarizable metric, boolean convertNanToZero) { - return ImmutableMap.builder() - .put("min", sanitizeValue(metric.min(), convertNanToZero)) - .put("max", sanitizeValue(metric.max(), convertNanToZero)) - .put("mean", sanitizeValue(metric.mean(), convertNanToZero)) - .put("sum", metric.sum()) - .put("stddev", metric.stdDev()) - .build(); - } - - public static Map explodeSampling(Sampling sampling) { - return explodeSampling(sampling, false); - } - - /** - * Retrieve values for pre-defined stats (median/p75/p95/p99/p999) from a {@link Sampling} metric (e.g. histogram) - * - * @param sampling metric to retrieve stats from - * @param convertNanToZero simulate {@link com.yammer.metrics.core.Histogram} histogram behavior when used with - * {@link com.yammer.metrics.core.WavefrontHistogram} as input: - * when "true", empty WavefrontHistogram reports zero values for all stats - * @return sampling stats - */ - public static Map explodeSampling(Sampling sampling, boolean convertNanToZero) { - final Snapshot snapshot = sampling.getSnapshot(); - return ImmutableMap.builder() - .put("median", sanitizeValue(snapshot.getMedian(), convertNanToZero)) - .put("p75", sanitizeValue(snapshot.get75thPercentile(), convertNanToZero)) - .put("p95", sanitizeValue(snapshot.get95thPercentile(), convertNanToZero)) - .put("p99", sanitizeValue(snapshot.get99thPercentile(), convertNanToZero)) - .put("p999", sanitizeValue(snapshot.get999thPercentile(), convertNanToZero)) - .build(); - } - - public static Map explodeMetered(Metered metered) { - return ImmutableMap.builder() - .put("count", new Long(metered.count()).doubleValue()) - .put("mean", metered.meanRate()) - .put("m1", metered.oneMinuteRate()) - .put("m5", metered.fiveMinuteRate()) - .put("m15", metered.fifteenMinuteRate()) - .build(); - } - - public static Map memoryMetrics(VirtualMachineMetrics vm) { - return ImmutableMap.builder() - .put("totalInit", vm.totalInit()) - .put("totalUsed", vm.totalUsed()) - .put("totalMax", vm.totalMax()) - .put("totalCommitted", vm.totalCommitted()) - .put("heapInit", vm.heapInit()) - .put("heapUsed", vm.heapUsed()) - .put("heapMax", vm.heapMax()) - .put("heapCommitted", vm.heapCommitted()) - .put("heap_usage", vm.heapUsage()) - .put("non_heap_usage", vm.nonHeapUsage()) - .build(); - } - - public static Map memoryPoolsMetrics(VirtualMachineMetrics vm) { - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (Map.Entry pool : vm.memoryPoolUsage().entrySet()) { - builder.put(pool.getKey(), pool.getValue()); - } - return builder.build(); - } - - public static Map buffersMetrics(VirtualMachineMetrics.BufferPoolStats bps) { - return ImmutableMap.builder() - .put("count", (double) bps.getCount()) - .put("memoryUsed", (double) bps.getMemoryUsed()) - .put("totalCapacity", (double) bps.getTotalCapacity()) - .build(); - } - - public static Map vmMetrics(VirtualMachineMetrics vm) { - return ImmutableMap.builder() - .put("daemon_thread_count", (double) vm.daemonThreadCount()) - .put("thread_count", (double) vm.threadCount()) - .put("uptime", (double) vm.uptime()) - .put("fd_usage", vm.fileDescriptorUsage()) - .build(); - } - - public static Map threadStateMetrics(VirtualMachineMetrics vm) { - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (Map.Entry entry : vm.threadStatePercentages().entrySet()) { - builder.put(entry.getKey().toString().toLowerCase(), entry.getValue()); - } - return builder.build(); - } - - public static Map gcMetrics(VirtualMachineMetrics.GarbageCollectorStats gcs) { - return ImmutableMap.builder() - .put("runs", (double) gcs.getRuns()) - .put("time", (double) gcs.getTime(TimeUnit.MILLISECONDS)) - .build(); - } - - - private static final Pattern SIMPLE_NAMES = Pattern.compile("[^a-zA-Z0-9_.\\-~]"); - - public static String sanitize(String name) { - return SIMPLE_NAMES.matcher(name).replaceAll("_"); - } - - public static String sanitize(MetricName metricName) { - return sanitize(metricName.getGroup() + "." + metricName.getName()); - } - - public static double sanitizeValue(double value, boolean convertNanToZero) { - return Double.isNaN(value) && convertNanToZero ? 0 : value; - } - -} diff --git a/java-lib/src/main/java/com/wavefront/common/NamedThreadFactory.java b/java-lib/src/main/java/com/wavefront/common/NamedThreadFactory.java deleted file mode 100644 index 954579a3b..000000000 --- a/java-lib/src/main/java/com/wavefront/common/NamedThreadFactory.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.wavefront.common; - -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; - -import javax.validation.constraints.NotNull; - -/** - * Simple thread factory to be used with Executors.newScheduledThreadPool that allows assigning name prefixes - * to all pooled threads to simplify thread identification during troubleshooting. - * - * Created by vasily@wavefront.com on 3/16/17. - */ -public class NamedThreadFactory implements ThreadFactory{ - private final String threadNamePrefix; - private final AtomicInteger counter = new AtomicInteger(); - - public NamedThreadFactory(@NotNull String threadNamePrefix) { - this.threadNamePrefix = threadNamePrefix; - } - - @Override - public Thread newThread(@NotNull Runnable r) { - Thread toReturn = new Thread(r); - toReturn.setName(threadNamePrefix + "-" + counter.getAndIncrement()); - return toReturn; - } -} diff --git a/java-lib/src/main/java/com/wavefront/common/Pair.java b/java-lib/src/main/java/com/wavefront/common/Pair.java deleted file mode 100644 index a58bb3a10..000000000 --- a/java-lib/src/main/java/com/wavefront/common/Pair.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.wavefront.common; - -public class Pair { - public final T _1; - public final V _2; - - public Pair(T t, V v) { - this._1 = t; - this._2 = v; - } - - @Override - public int hashCode() { - return _1.hashCode() + 43 * _2.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof Pair) { - Pair pair = (Pair) obj; - return _1.equals(pair._1) && _2.equals(pair._2); - } - return false; - } - - public static Pair of(T t, V v) { - return new Pair(t, v); - } -} diff --git a/java-lib/src/main/java/com/wavefront/common/TaggedMetricName.java b/java-lib/src/main/java/com/wavefront/common/TaggedMetricName.java deleted file mode 100644 index 24dd1a6d4..000000000 --- a/java-lib/src/main/java/com/wavefront/common/TaggedMetricName.java +++ /dev/null @@ -1,123 +0,0 @@ -package com.wavefront.common; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.yammer.metrics.core.MetricName; -import wavefront.report.ReportPoint; - -import javax.annotation.Nonnull; -import javax.management.ObjectName; -import java.util.Collections; -import java.util.Map; - -/** - * A taggable metric name. - * - * @author Clement Pang (clement@wavefront.com) - */ -public class TaggedMetricName extends MetricName { - @Nonnull - private final Map tags; - - /** - * A simple metric that would be concatenated when reported, e.g. "jvm", "name" would become jvm.name. - * - * @param group Prefix of the metric. - * @param name The name of the metric. - */ - public TaggedMetricName(String group, String name) { - super(group, "", name); - tags = Collections.emptyMap(); - } - - public TaggedMetricName(String group, String name, String... tagAndValues) { - this(group, name, makeTags(tagAndValues)); - } - - public TaggedMetricName(String group, String name, Map tags) { - this(group, name, makeTags(tags)); - } - - public TaggedMetricName(String group, String name, Pair... tags) { - super(group, "", name, null, createMBeanName(group, "", name, tags)); - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (Pair tag : tags) { - if (tag != null && tag._1 != null && tag._2 != null) { - builder.put(tag._1, tag._2); - } - } - this.tags = builder.build(); - } - - public Map getTags() { - return tags; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - TaggedMetricName that = (TaggedMetricName) o; - - return getTags().equals(that.getTags()); - - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + getTags().hashCode(); - return result; - } - - public void updatePointBuilder(ReportPoint.Builder builder) { - builder.getAnnotations().putAll(tags); - } - - private static Pair[] makeTags(Map tags) { - @SuppressWarnings("unchecked") - Pair[] toReturn = new Pair[tags.size()]; - int i = 0; - for (Map.Entry entry : tags.entrySet()) { - toReturn[i] = new Pair(entry.getKey(), entry.getValue()); - i++; - } - return toReturn; - } - - private static Pair[] makeTags(String... tagAndValues) { - Preconditions.checkArgument((tagAndValues.length & 1) == 0, "must have even number of tag values"); - @SuppressWarnings("unchecked") - Pair[] toReturn = new Pair[tagAndValues.length / 2]; - for (int i = 0; i < tagAndValues.length; i += 2) { - String tag = tagAndValues[i]; - String value = tagAndValues[i + 1]; - if (tag != null && value != null) { - toReturn[i / 2] = new Pair(tag, value); - } - } - return toReturn; - } - - private static String createMBeanName(String group, String type, String name, Pair... tags) { - final StringBuilder nameBuilder = new StringBuilder(); - nameBuilder.append(ObjectName.quote(group)); - nameBuilder.append(":type="); - nameBuilder.append(ObjectName.quote(type)); - if (name.length() > 0) { - nameBuilder.append(",name="); - nameBuilder.append(ObjectName.quote(name)); - } - for (Pair tag : tags) { - if (tag != null) { - nameBuilder.append(","); - nameBuilder.append(tag._1); - nameBuilder.append("="); - nameBuilder.append(ObjectName.quote(tag._2)); - } - } - return nameBuilder.toString(); - } -} diff --git a/java-lib/src/main/java/com/wavefront/common/TraceConstants.java b/java-lib/src/main/java/com/wavefront/common/TraceConstants.java deleted file mode 100644 index ddeaf1621..000000000 --- a/java-lib/src/main/java/com/wavefront/common/TraceConstants.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.wavefront.common; - -/** - * Trace constants. - * - * @author Anil Kodali (akodali@vmware.com) - */ -public abstract class TraceConstants { - //TODO: Put the below constants in https://github.com/wavefrontHQ/wavefront-sdk-java - // Span References - // See https://opentracing.io/specification/ for more information about span references. - public static final String FOLLOWS_FROM_KEY = "followsFrom"; - public static final String PARENT_KEY = "parent"; -} diff --git a/java-lib/src/main/java/com/wavefront/data/Idempotent.java b/java-lib/src/main/java/com/wavefront/data/Idempotent.java deleted file mode 100644 index db7026858..000000000 --- a/java-lib/src/main/java/com/wavefront/data/Idempotent.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.wavefront.data; - -import java.io.IOException; -import java.lang.annotation.Documented; -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.*; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -/** - * Idempotent calls can be retried if a call fails. - * - * @author Clement Pang (clement@sunnylabs.com) - */ -@Target(value = {METHOD, PARAMETER, TYPE}) -@Retention(RUNTIME) -@Documented -public @interface Idempotent { - /** - * @return Number of times to retry a call when it fails. - */ - int retries() default 3; - - /** - * @return List of exceptions that should be retried. - */ - Class[] retryableExceptions() default IOException.class; -} diff --git a/java-lib/src/main/java/com/wavefront/data/ReportableEntityType.java b/java-lib/src/main/java/com/wavefront/data/ReportableEntityType.java deleted file mode 100644 index 7599178c8..000000000 --- a/java-lib/src/main/java/com/wavefront/data/ReportableEntityType.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.wavefront.data; - -/** - * Type of objects that Wavefront proxy can send to the server endpoint(s). - * - * @author vasily@wavefront.com - */ -public enum ReportableEntityType { - POINT("points"), - HISTOGRAM("points"), - SOURCE_TAG("sourceTags"), - TRACE("spans"); - - private final String name; - - ReportableEntityType(String name) { - this.name = name; - } - - @Override - public String toString() { - return name; - } -} diff --git a/java-lib/src/main/java/com/wavefront/data/Validation.java b/java-lib/src/main/java/com/wavefront/data/Validation.java deleted file mode 100644 index d2a1fc0f0..000000000 --- a/java-lib/src/main/java/com/wavefront/data/Validation.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.wavefront.data; - -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Counter; -import com.yammer.metrics.core.MetricName; - -import org.apache.commons.lang.StringUtils; - -import java.util.Map; - -import javax.annotation.Nullable; - -import wavefront.report.Histogram; -import wavefront.report.ReportPoint; - -import static com.wavefront.data.Validation.Level.NO_VALIDATION; - -/** - * Consolidates point validation logic for point handlers - * - * @author Tim Schmidt (tim@wavefront.com). - */ -public class Validation { - - public enum Level { - NO_VALIDATION, - NUMERIC_ONLY - } - - private final static Counter illegalCharacterPoints = Metrics.newCounter(new MetricName("point", "", "badchars")); - - public static boolean charactersAreValid(String input) { - // Legal characters are 44-57 (,-./ and numbers), 65-90 (upper), 97-122 (lower), 95 (_) - int l = input.length(); - if (l == 0) { - return false; - } - - for (int i = 0; i < l; i++) { - char cur = input.charAt(i); - if (!(44 <= cur && cur <= 57) && !(65 <= cur && cur <= 90) && !(97 <= cur && cur <= 122) && - cur != 95) { - if (!((i == 0 && cur == 0x2206) || (i == 0 && cur == 0x0394) || (i == 0 && cur == 126))) { - // first character can also be \u2206 (∆ - INCREMENT) or \u0394 (Δ - GREEK CAPITAL LETTER DELTA) - // or ~ tilda character for internal metrics - return false; - } - } - } - return true; - } - - public static boolean annotationKeysAreValid(ReportPoint point) { - for (String key : point.getAnnotations().keySet()) { - if (!charactersAreValid(key)) { - return false; - } - } - return true; - } - - public static void validatePoint(ReportPoint point, String source, @Nullable Level validationLevel) { - Object pointValue = point.getValue(); - - if (StringUtils.isBlank(point.getHost())) { - throw new IllegalArgumentException("WF-301: Source/host name is required"); - - } - if (point.getHost().length() >= 1024) { - throw new IllegalArgumentException("WF-301: Source/host name is too long: " + point.getHost()); - } - - if (point.getMetric().length() >= 1024) { - throw new IllegalArgumentException("WF-301: Metric name is too long: " + point.getMetric()); - } - - if (!charactersAreValid(point.getMetric())) { - illegalCharacterPoints.inc(); - throw new IllegalArgumentException("WF-400 " + source + ": Point metric has illegal character"); - } - - if (point.getAnnotations() != null) { - if (!annotationKeysAreValid(point)) { - throw new IllegalArgumentException("WF-401 " + source + ": Point annotation key has illegal character"); - } - - // Each tag of the form "k=v" must be < 256 - for (Map.Entry tag : point.getAnnotations().entrySet()) { - if (tag.getKey().length() + tag.getValue().length() >= 255) { - throw new IllegalArgumentException("Tag too long: " + tag.getKey() + "=" + tag.getValue()); - } - } - } - - if ((validationLevel != null) && (!validationLevel.equals(NO_VALIDATION))) { - // Is it the right type of point? - switch (validationLevel) { - case NUMERIC_ONLY: - if (!(pointValue instanceof Long) && !(pointValue instanceof Double) && !(pointValue instanceof Histogram)) { - throw new IllegalArgumentException("WF-403 " + source + ": Was not long/double/histogram object"); - } - if (pointValue instanceof Histogram) { - Histogram histogram = (Histogram) pointValue; - if (histogram.getCounts().size() == 0 || histogram.getBins().size() == 0 || - histogram.getCounts().stream().allMatch(i -> i == 0)) { - throw new IllegalArgumentException("WF-405 " + source + ": Empty histogram"); - } - } - break; - } - } - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/AbstractIngesterFormatter.java b/java-lib/src/main/java/com/wavefront/ingester/AbstractIngesterFormatter.java deleted file mode 100644 index 36ee118f1..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/AbstractIngesterFormatter.java +++ /dev/null @@ -1,992 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSortedSet; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.BaseErrorListener; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.Token; -import org.apache.commons.lang.time.DateUtils; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; - -import queryserver.parser.DSWrapperLexer; -import wavefront.report.Annotation; -import wavefront.report.Histogram; -import wavefront.report.HistogramType; -import wavefront.report.ReportPoint; -import wavefront.report.ReportSourceTag; -import wavefront.report.Span; - -/** - * This is the base class for formatting the content. - * - * @author Suranjan Pramanik (suranjan@wavefront.com) - */ -public abstract class AbstractIngesterFormatter { - - protected static final FormatterElement WHITESPACE_ELEMENT = new - ReportPointIngesterFormatter.Whitespace(); - protected static final Pattern SINGLE_QUOTE_PATTERN = Pattern.compile("\\'", Pattern.LITERAL); - protected static final Pattern DOUBLE_QUOTE_PATTERN = Pattern.compile("\\\"", Pattern.LITERAL); - protected static final String DOUBLE_QUOTE_REPLACEMENT = Matcher.quoteReplacement("\""); - protected static final String SINGLE_QUOTE_REPLACEMENT = Matcher.quoteReplacement("'"); - - private static final BaseErrorListener THROWING_ERROR_LISTENER = new BaseErrorListener() { - @Override - public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, - int charPositionInLine, String msg, RecognitionException e) { - throw new RuntimeException("Syntax error at line " + line + ", position " + charPositionInLine + ": " + msg, e); - } - }; - - protected final List elements; - - protected static final ThreadLocal dsWrapperLexerThreadLocal = - new ThreadLocal() { - @Override - protected DSWrapperLexer initialValue() { - final DSWrapperLexer lexer = new DSWrapperLexer(new ANTLRInputStream("")); - // note that other errors are not thrown by the lexer and hence we only need to handle the - // syntaxError case. - lexer.removeErrorListeners(); - lexer.addErrorListener(THROWING_ERROR_LISTENER); - return lexer; - } - }; - - AbstractIngesterFormatter(List elements) { - this.elements = elements; - } - - protected Queue getQueue(String input) { - DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get(); - lexer.setInputStream(new ANTLRInputStream(input)); - CommonTokenStream commonTokenStream = new CommonTokenStream(lexer); - commonTokenStream.fill(); - List tokens = commonTokenStream.getTokens(); - if (tokens.isEmpty()) { - throw new RuntimeException("Could not parse: " + input); - } - // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so - // means we need to be creating the AST and instead we could just use the lexer. in any case, - // we don't expect the graphite format to change anytime soon. - - // filter all EOF tokens first. - Queue queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect( - Collectors.toCollection(ArrayDeque::new)); - return queue; - } - - /** - * This class is a wrapper/proxy around ReportPoint and ReportSourceTag. It has a default - * implementation of all the methods of these two classes. The default implementation is to - * throw exception and the base classes will override them. - */ - protected abstract static class AbstractWrapper { - - Object getValue() { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void setValue(Histogram value) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void setValue(double value) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void setValue(String value) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void setValue(Long value) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - Long getTimestamp() { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void setTimestamp(Long value) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void setDuration(Long value) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void addAnnotation(String key, String value) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void addAnnotation(String value) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void setMetric(String value) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - String getLiteral() { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void setLiteral(String literal) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void setCustomer(String name) { - throw new UnsupportedOperationException("Should not be invoked."); - } - - void setName(String value) { - throw new UnsupportedOperationException("Should not be invoked."); - } - } - - /** - * This class provides a wrapper around ReportPoint. - */ - protected static class ReportPointWrapper extends AbstractWrapper { - ReportPoint reportPoint; - - ReportPointWrapper(ReportPoint reportPoint) { - this.reportPoint = reportPoint; - } - - @Override - Object getValue() { - return reportPoint.getValue(); - } - - @Override - void setValue(Histogram value) { - reportPoint.setValue(value); - } - - @Override - void setValue(double value) { - reportPoint.setValue(value); - } - - @Override - void setValue(String value) { - reportPoint.setValue(value); - } - - @Override - void setValue(Long value) { - reportPoint.setValue(value); - } - - @Override - Long getTimestamp() { - return reportPoint.getTimestamp(); - } - - @Override - void setTimestamp(Long value) { - reportPoint.setTimestamp(value); - } - - @Override - void addAnnotation(String key, String value) { - if (reportPoint.getAnnotations() == null) { - reportPoint.setAnnotations(new HashMap<>()); - } - reportPoint.getAnnotations().put(key, value); - } - - @Override - void setMetric(String value) { - reportPoint.setMetric(value); - } - } - - /** - * This class provides a wrapper around ReportSourceTag - */ - protected static class ReportSourceTagWrapper extends AbstractWrapper { - final ReportSourceTag reportSourceTag; - final Map annotations; - - ReportSourceTagWrapper(ReportSourceTag reportSourceTag) { - this.reportSourceTag = reportSourceTag; - this.annotations = Maps.newHashMap(); - } - - @Override - String getLiteral() { - return reportSourceTag.getSourceTagLiteral(); - } - - @Override - void setLiteral(String literal) { - reportSourceTag.setSourceTagLiteral(literal); - } - - @Override - void addAnnotation(String key, String value) { - annotations.put(key, value); - } - - @Override - void addAnnotation(String value) { - if (reportSourceTag.getAnnotations() == null) - reportSourceTag.setAnnotations(Lists.newArrayList()); - reportSourceTag.getAnnotations().add(value); - } - - @NotNull - Map getAnnotationMap() { - return annotations; - } - } - - /** - * This class provides a wrapper around Span. - */ - protected static class SpanWrapper extends AbstractWrapper { - Span span; - - SpanWrapper(Span span) { - this.span = span; - } - - @Override - void setDuration(Long value) { - span.setDuration(value); - } - - @Override - Long getTimestamp() { - return span.getStartMillis(); - } - - @Override - void setTimestamp(Long value) { - span.setStartMillis(value); - } - - @Override - void setName(String name) { // - span.setName(name); - } - - @Override - void addAnnotation(String key, String value) { - if (span.getAnnotations() == null) { - span.setAnnotations(Lists.newArrayList()); - } - span.getAnnotations().add(new Annotation(key, value)); - } - } - - protected interface FormatterElement { - /** - * Consume tokens from the queue. - */ - void consume(Queue tokenQueue, AbstractWrapper point); - } - - /** - * This class can be used to create a parser for a content that the proxy receives - e.g., - * ReportPoint and ReportSourceTag. - */ - public abstract static class IngesterFormatBuilder { - - final List elements = Lists.newArrayList(); - - public IngesterFormatBuilder appendCaseSensitiveLiteral(String literal) { - elements.add(new Literal(literal, true)); - return this; - } - - public IngesterFormatBuilder appendCaseSensitiveLiterals(String[] literals) { - elements.add(new Literals(literals, true)); - return this; - } - - public IngesterFormatBuilder appendCaseInsensitiveLiteral(String literal) { - elements.add(new Literal(literal, false)); - return this; - } - - public IngesterFormatBuilder appendMetricName() { - elements.add(new Metric()); - return this; - } - - public IngesterFormatBuilder appendValue() { - elements.add(new Value()); - return this; - } - - public IngesterFormatBuilder appendTimestamp() { - elements.add(new AdaptiveTimestamp(false)); - return this; - } - - public IngesterFormatBuilder appendOptionalTimestamp() { - elements.add(new AdaptiveTimestamp(true)); - return this; - } - - public IngesterFormatBuilder appendTimestamp(TimeUnit timeUnit) { - elements.add(new Timestamp(timeUnit, false)); - return this; - } - - public IngesterFormatBuilder appendOptionalTimestamp(TimeUnit timeUnit) { - elements.add(new Timestamp(timeUnit, true)); - return this; - } - - public IngesterFormatBuilder appendRawTimestamp() { - elements.add(new AdaptiveTimestamp(false, false)); - return this; - } - - public IngesterFormatBuilder appendDuration() { - elements.add(new Duration(false)); - return this; - } - - public IngesterFormatBuilder appendName() { - elements.add(new Name()); - return this; - } - - public IngesterFormatBuilder appendBoundedAnnotationsConsumer() { - elements.add(new GuardedLoop(new Tag(), ImmutableSortedSet.of(DSWrapperLexer.Literal, DSWrapperLexer.Letters, - DSWrapperLexer.Quoted), false)); - return this; - } - - public IngesterFormatBuilder appendAnnotationsConsumer() { - elements.add(new Loop(new Tag())); - return this; - } - - public IngesterFormatBuilder whiteSpace() { - elements.add(new Whitespace()); - return this; - } - - public IngesterFormatBuilder binType() { - elements.add(new BinType()); - return this; - } - - public IngesterFormatBuilder centroids() { - elements.add(new GuardedLoop(new Centroid(), Centroid.expectedToken(), false)); - return this; - } - - public IngesterFormatBuilder adjustTimestamp() { - elements.add(new TimestampAdjuster()); - return this; - } - - public IngesterFormatBuilder appendLoopOfKeywords() { - elements.add(new LoopOfKeywords()); - return this; - } - - public IngesterFormatBuilder appendLoopOfValues() { - elements.add(new Loop(new AlphaNumericValue())); - return this; - } - - /** - * Subclasses will provide concrete implementation for this method. - */ - public abstract AbstractIngesterFormatter build(); - } - - public static class Loop implements FormatterElement { - - private final FormatterElement element; - - public Loop(FormatterElement element) { - this.element = element; - } - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - while (!tokenQueue.isEmpty()) { - WHITESPACE_ELEMENT.consume(tokenQueue, point); - if (tokenQueue.isEmpty()) return; - element.consume(tokenQueue, point); - } - } - } - - public static class BinType implements FormatterElement { - - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - { - Token peek = tokenQueue.peek(); - - if (peek == null) { - throw new RuntimeException("Expected BinType, found EOF"); - } - if (peek.getType() != DSWrapperLexer.BinType) { - throw new RuntimeException("Expected BinType, found " + peek.getText()); - } - } - - int durationMillis = 0; - String binType = tokenQueue.poll().getText(); - - switch (binType) { - case "!M": - durationMillis = (int) DateUtils.MILLIS_PER_MINUTE; - break; - case "!H": - durationMillis = (int) DateUtils.MILLIS_PER_HOUR; - break; - case "!D": - durationMillis = (int) DateUtils.MILLIS_PER_DAY; - break; - default: - throw new RuntimeException("Unknown BinType " + binType); - } - - Histogram h = computeIfNull((Histogram) point.getValue(), Histogram::new); - h.setDuration(durationMillis); - h.setType(HistogramType.TDIGEST); - point.setValue(h); - } - } - - public static class Centroid implements FormatterElement { - - public static int expectedToken() { - return DSWrapperLexer.Weight; - } - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - { - Token peek = tokenQueue.peek(); - - Preconditions.checkNotNull(peek, "Expected Count, got EOF"); - Preconditions.checkArgument(peek.getType() == DSWrapperLexer.Weight, "Expected Count, got " + peek.getText()); - } - - String countStr = tokenQueue.poll().getText(); - int count; - try { - count = Integer.parseInt(countStr.substring(1)); - } catch (NumberFormatException e) { - throw new RuntimeException("Could not parse count " + countStr); - } - - WHITESPACE_ELEMENT.consume(tokenQueue, point); - - // Mean - double mean = parseValue(tokenQueue, "centroid mean"); - - Histogram h = computeIfNull((Histogram) point.getValue(), Histogram::new); - List bins = computeIfNull(h.getBins(), ArrayList::new); - bins.add(mean); - h.setBins(bins); - - List counts = computeIfNull(h.getCounts(), ArrayList::new); - counts.add(count); - h.setCounts(counts); - point.setValue(h); - } - } - - /** - * Similar to {@link Loop}, but expects a configurable non-whitespace {@link Token} - */ - public static class GuardedLoop implements FormatterElement { - private final FormatterElement element; - private final Set acceptedTokens; - private final boolean optional; - - public GuardedLoop(FormatterElement element, int acceptedToken, boolean optional) { - this.element = element; - this.acceptedTokens = ImmutableSortedSet.of(acceptedToken); - this.optional = optional; - } - - public GuardedLoop(FormatterElement element, Set acceptedTokens, boolean optional) { - this.element = element; - this.acceptedTokens = acceptedTokens; - this.optional = optional; - } - - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - boolean satisfied = optional; - while (!tokenQueue.isEmpty()) { - WHITESPACE_ELEMENT.consume(tokenQueue, point); - if (tokenQueue.peek() == null || !acceptedTokens.contains(tokenQueue.peek().getType())) { - break; - } - satisfied = true; - element.consume(tokenQueue, point); - } - - if (!satisfied) { - throw new RuntimeException("Expected at least one element, got none"); - } - } - } - - /** - * Pins the point's timestamp to the beginning of the respective interval - */ - public static class TimestampAdjuster implements FormatterElement { - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - Preconditions.checkArgument(point.getValue() != null - && point.getTimestamp() != null - && point.getValue() instanceof Histogram - && ((Histogram) point.getValue()).getDuration() != null, - "Expected a histogram point with timestamp and histogram duration"); - - long duration = ((Histogram) point.getValue()).getDuration(); - point.setTimestamp((point.getTimestamp() / duration) * duration); - } - } - - - public static class Tag implements FormatterElement { - - @Override - public void consume(Queue queue, AbstractWrapper point) { - // extract tags. - String tagk; - tagk = getLiteral(queue); - if (tagk.length() == 0) { - throw new RuntimeException("Invalid tag name"); - } - WHITESPACE_ELEMENT.consume(queue, point); - Token current = queue.poll(); - if (current == null || current.getType() != DSWrapperLexer.EQ) { - throw new RuntimeException("Tag keys and values must be separated by '='" + - (current != null ? ", " + "found: " + current.getText() : ", found EOF")); - } - WHITESPACE_ELEMENT.consume(queue, point); - String tagv = getLiteral(queue); - if (tagv.length() == 0) throw new RuntimeException("Invalid tag value for: " + tagk); - point.addAnnotation(tagk, tagv); - } - } - - private static double parseValue(Queue tokenQueue, String name) { - String value = ""; - Token current = tokenQueue.poll(); - if (current == null) throw new RuntimeException("Invalid " + name + ", found EOF"); - if (current.getType() == DSWrapperLexer.MinusSign) { - current = tokenQueue.poll(); - value = "-"; - } - if (current == null) throw new RuntimeException("Invalid " + name + ", found EOF"); - if (current.getType() == DSWrapperLexer.Quoted) { - if (!value.equals("")) { - throw new RuntimeException("Invalid " + name + ": " + value + current.getText()); - } - value += unquote(current.getText()); - } else if (current.getType() == DSWrapperLexer.Letters || - current.getType() == DSWrapperLexer.Literal || - current.getType() == DSWrapperLexer.Number) { - value += current.getText(); - } else { - throw new RuntimeException("Invalid " + name + ": " + current.getText()); - } - try { - return Double.parseDouble(value); - } catch (NumberFormatException nef) { - throw new RuntimeException("Invalid " + name + ": " + value); - } - } - - public static class AlphaNumericValue implements FormatterElement { - - @Override - public void consume(Queue tokenQueue, AbstractWrapper sourceTag) { - WHITESPACE_ELEMENT.consume(tokenQueue, sourceTag); - String value = ""; - Token current = tokenQueue.poll(); - if (current == null) throw new RuntimeException("Invalid value, found EOF"); - - if (current == null) throw new RuntimeException("Invalid value, found EOF"); - if (current.getType() == DSWrapperLexer.Quoted) { - if (!value.equals("")) { - throw new RuntimeException("invalid metric value: " + value + current.getText()); - } - value += ReportPointIngesterFormatter.unquote(current.getText()); - } else if (current.getType() == DSWrapperLexer.Letters || - current.getType() == DSWrapperLexer.Literal || - current.getType() == DSWrapperLexer.Number) { - value += current.getText(); - } else { - throw new RuntimeException("invalid value: " + current.getText()); - } - sourceTag.addAnnotation(value); - } - } - - public static class Value implements FormatterElement { - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - point.setValue(parseValue(tokenQueue, "metric value")); - } - } - - private static Long parseTimestamp(Queue tokenQueue, boolean optional, boolean convertToMillis) { - Token peek = tokenQueue.peek(); - if (peek == null || peek.getType() != DSWrapperLexer.Number) { - if (optional) return null; - else - throw new RuntimeException("Expected timestamp, found " + (peek == null ? "EOF" : peek.getText())); - } - try { - Double timestamp = Double.parseDouble(tokenQueue.poll().getText()); - Long timestampLong = timestamp.longValue(); - if (!convertToMillis) { - // as-is - return timestampLong; - } - int timestampDigits = timestampLong.toString().length(); - if (timestampDigits == 19) { - // nanoseconds. - return timestampLong / 1000000; - } else if (timestampDigits == 16) { - // microseconds - return timestampLong / 1000; - } else if (timestampDigits == 13) { - // milliseconds. - return timestampLong; - } else { - // treat it as seconds. - return (long) (1000.0 * timestamp); - } - } catch (NumberFormatException nfe) { - throw new RuntimeException("Invalid timestamp value: " + peek.getText()); - } - } - - public static class Duration implements FormatterElement { - - private final boolean optional; - - public Duration(boolean optional) { - this.optional = optional; - } - - @Override - public void consume(Queue tokenQueue, AbstractWrapper wrapper) { - Long timestamp = parseTimestamp(tokenQueue, optional, false); - - Long startTs = wrapper.getTimestamp(); - if (timestamp != null && startTs != null) { - long duration = (timestamp - startTs >= 0) ? timestamp - startTs : timestamp; - // convert both timestamps to millis - if (startTs > 999999999999999999L) { - // 19 digits == nanoseconds, - wrapper.setTimestamp(startTs / 1000_000); - wrapper.setDuration(duration / 1000_000); - } else if (startTs > 999999999999999L) { - // 16 digits == microseconds - wrapper.setTimestamp(startTs / 1000); - wrapper.setDuration(duration / 1000); - } else if (startTs > 999999999999L) { - // 13 digits == milliseconds - wrapper.setDuration(duration); - } else { - // seconds - wrapper.setTimestamp(startTs * 1000); - wrapper.setDuration(duration * 1000); - } - } else { - throw new RuntimeException("Both timestamp and duration expected"); - } - } - } - - public static class Name implements FormatterElement { - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - String name = getLiteral(tokenQueue); - if (name.length() == 0) throw new RuntimeException("Invalid name"); - point.setName(name); - } - } - - public static class AdaptiveTimestamp implements FormatterElement { - - private final boolean optional; - private final boolean convertToMillis; - - public AdaptiveTimestamp(boolean optional) { - this(optional, true); - } - - public AdaptiveTimestamp(boolean optional, boolean convertToMillis) { - this.optional = optional; - this.convertToMillis = convertToMillis; - } - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - Long timestamp = parseTimestamp(tokenQueue, optional, convertToMillis); - - // Do not override with null on satisfied - if (timestamp != null) { - point.setTimestamp(timestamp); - } - } - } - - public static class Timestamp implements FormatterElement { - - private final TimeUnit timeUnit; - private final boolean optional; - - public Timestamp(TimeUnit timeUnit, boolean optional) { - this.timeUnit = timeUnit; - this.optional = optional; - } - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - Token peek = tokenQueue.peek(); - if (peek == null) { - if (optional) return; - else throw new RuntimeException("Expecting timestamp, found EOF"); - } - if (peek.getType() == DSWrapperLexer.Number) { - try { - // we need to handle the conversion outselves. - long multiplier = timeUnit.toMillis(1); - if (multiplier < 1) { - point.setTimestamp(timeUnit.toMillis( - (long) Double.parseDouble(tokenQueue.poll().getText()))); - } else { - point.setTimestamp((long) - (multiplier * Double.parseDouble(tokenQueue.poll().getText()))); - } - } catch (NumberFormatException nfe) { - throw new RuntimeException("Invalid timestamp value: " + peek.getText()); - } - } else if (!optional) { - throw new RuntimeException("Expecting timestamp, found: " + peek.getText()); - } - } - } - - public static class Metric implements FormatterElement { - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - // extract the metric name. - String metric = getLiteral(tokenQueue); - if (metric.length() == 0) throw new RuntimeException("Invalid metric name"); - point.setMetric(metric); - } - } - - public static class Whitespace implements FormatterElement { - @Override - public void consume(Queue tokens, AbstractWrapper point) { - while (!tokens.isEmpty() && tokens.peek().getType() == DSWrapperLexer.WS) { - tokens.poll(); - } - } - } - - /** - * This class handles a sequence of key value pairs. Currently it works for source tag related - * inputs only. - */ - public static class LoopOfKeywords implements FormatterElement { - - private final FormatterElement tagElement = new Tag(); - - @Override - public void consume(Queue tokenQueue, AbstractWrapper sourceTag) { - if (sourceTag.getLiteral() == null) { - // throw an exception since we expected that field to be populated - throw new RuntimeException("Expected either @SourceTag or @SourceDescription in the " + - "message"); - } else if (sourceTag.getLiteral().equals("SourceTag")) { - // process it as a sourceTag -- 2 tag elements; action="add" source="aSource" - int count = 0, max = 2; - while (count < max) { - WHITESPACE_ELEMENT.consume(tokenQueue, sourceTag); - tagElement.consume(tokenQueue, sourceTag); - count++; - } - } else if (sourceTag.getLiteral().equals("SourceDescription")) { - // process it as a description -- all the remaining should be tags - while (!tokenQueue.isEmpty()) { - WHITESPACE_ELEMENT.consume(tokenQueue, sourceTag); - tagElement.consume(tokenQueue, sourceTag); - } - } else { - // throw exception, since it should be one of the above - throw new RuntimeException("Expected either @SourceTag or @SourceDescription in the " + - "message"); - } - } - } - - public static class Literals implements FormatterElement { - private final String[] literals; - private final boolean caseSensitive; - - public Literals(String[] literals, boolean caseSensitive) { - this.literals = literals; - this.caseSensitive = caseSensitive; - } - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - if (literals == null || literals.length != 2) - throw new RuntimeException("Sourcetag metadata parser is not properly initialized."); - - if (tokenQueue.isEmpty()) { - throw new RuntimeException("Expecting a literal string: " + literals[0] + " or " + - literals[1] + " but found EOF"); - } - String literal = getLiteral(tokenQueue); - if (caseSensitive) { - for (String specLiteral : literals) { - if (literal.equals(specLiteral)) { - point.setLiteral(literal.substring(1)); - return; - } - } - throw new RuntimeException("Expecting a literal string: " + literals[0] + " or " + - literals[1] + " but found: " + literal); - } else { - for (String specLiteral : literals) { - if (literal.equalsIgnoreCase(specLiteral)) { - point.setLiteral(literal.substring(1)); - return; - } - } - throw new RuntimeException("Expecting a literal string: " + literals[0] + " or " + - literals[1] + " but found: " + literal); - } - } - } - - public static class Literal implements FormatterElement { - - private final String literal; - private final boolean caseSensitive; - - public Literal(String literal, boolean caseSensitive) { - this.literal = literal; - this.caseSensitive = caseSensitive; - } - - @Override - public void consume(Queue tokenQueue, AbstractWrapper point) { - if (tokenQueue.isEmpty()) { - throw new RuntimeException("Expecting a literal string: " + literal + " but found EOF"); - } - String literal = getLiteral(tokenQueue); - if (caseSensitive) { - if (!literal.equals(this.literal)) { - throw new RuntimeException("Expecting a literal string: " + this.literal + " but found:" + - " " + literal); - } - } else { - if (!literal.equalsIgnoreCase(this.literal)) { - throw new RuntimeException("Expecting a literal string: " + this.literal + " but found:" + - " " + literal); - } - } - } - } - - - protected static String getLiteral(Queue tokens) { - StringBuilder toReturn = new StringBuilder(); - Token next = tokens.peek(); - if (next == null) return ""; - if (next.getType() == DSWrapperLexer.Quoted) { - return unquote(tokens.poll().getText()); - } - - while (next != null && - (next.getType() == DSWrapperLexer.Letters || - next.getType() == DSWrapperLexer.RelaxedLiteral || - next.getType() == DSWrapperLexer.Number || - next.getType() == DSWrapperLexer.SLASH || - next.getType() == DSWrapperLexer.AT || - next.getType() == DSWrapperLexer.Literal || - next.getType() == DSWrapperLexer.IpV4Address || - next.getType() == DSWrapperLexer.MinusSign || - next.getType() == DSWrapperLexer.IpV6Address || - next.getType() == DSWrapperLexer.DELTA)) { - toReturn.append(tokens.poll().getText()); - next = tokens.peek(); - } - return toReturn.toString(); - } - - /** - * @param text Text to unquote. - * @return Extracted value from inside a quoted string. - */ - @SuppressWarnings("WeakerAccess") // Has users. - public static String unquote(String text) { - if (text.startsWith("\"")) { - text = DOUBLE_QUOTE_PATTERN.matcher(text.substring(1, text.length() - 1)). - replaceAll(DOUBLE_QUOTE_REPLACEMENT); - } else if (text.startsWith("'")) { - text = SINGLE_QUOTE_PATTERN.matcher(text.substring(1, text.length() - 1)). - replaceAll(SINGLE_QUOTE_REPLACEMENT); - } - return text; - } - - public T drive(String input, String defaultHostName, String customerId) { - return drive(input, defaultHostName, customerId, null); - } - - public abstract T drive(String input, String defaultHostName, String customerId, - @Nullable List customerSourceTags); - - static T computeIfNull(@Nullable T input, Supplier supplier) { - if (input == null) return supplier.get(); - return input; - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/Decoder.java b/java-lib/src/main/java/com/wavefront/ingester/Decoder.java deleted file mode 100644 index 72a5a57de..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/Decoder.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.wavefront.ingester; - -import java.util.List; - -import wavefront.report.ReportPoint; - -/** - * A decoder of an input line. - * - * @author Clement Pang (clement@wavefront.com). - */ -public interface Decoder { - /** - * Decode graphite points and dump them into an output array. The supplied customer id will be set - * and no customer id extraction will be attempted. - * - * @param msg Message to parse. - * @param out List to output the parsed point. - * @param customerId The customer id to use as the table for the result ReportPoint. - */ - void decodeReportPoints(T msg, List out, String customerId); - - /** - * Certain decoders support decoding the customer id from the input line itself. - * - * @param msg Message to parse. - * @param out List to output the parsed point. - */ - void decodeReportPoints(T msg, List out); -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/GraphiteDecoder.java b/java-lib/src/main/java/com/wavefront/ingester/GraphiteDecoder.java deleted file mode 100644 index d9e246d3b..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/GraphiteDecoder.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; - -import java.util.List; -import java.util.regex.Pattern; - -import wavefront.report.ReportPoint; - -/** - * Graphite decoder that takes in a point of the type: - * - * [metric] [value] [timestamp] [annotations] - * - * @author Clement Pang (clement@wavefront.com). - */ -public class GraphiteDecoder implements Decoder { - - private static final Pattern CUSTOMERID = Pattern.compile("[a-z]+"); - private static final AbstractIngesterFormatter FORMAT = - ReportPointIngesterFormatter.newBuilder() - .whiteSpace() - .appendMetricName().whiteSpace() - .appendValue().whiteSpace() - .appendOptionalTimestamp().whiteSpace() - .appendAnnotationsConsumer().whiteSpace().build(); - private final String hostName; - private List customSourceTags; - - public GraphiteDecoder(List customSourceTags) { - this.hostName = "unknown"; - Preconditions.checkNotNull(customSourceTags); - this.customSourceTags = customSourceTags; - } - - public GraphiteDecoder(String hostName, List customSourceTags) { - Preconditions.checkNotNull(hostName); - this.hostName = hostName; - Preconditions.checkNotNull(customSourceTags); - this.customSourceTags = customSourceTags; - } - - @Override - public void decodeReportPoints(String msg, List out, String customerId) { - ReportPoint point = FORMAT.drive(msg, hostName, customerId, customSourceTags); - if (out != null) { - out.add(point); - } - } - - @Override - public void decodeReportPoints(String msg, List out) { - List output = Lists.newArrayList(); - decodeReportPoints(msg, output, "dummy"); - if (!output.isEmpty()) { - for (ReportPoint rp : output) { - String metricName = rp.getMetric(); - List metricParts = Lists.newArrayList(Splitter.on(".").split(metricName)); - if (metricParts.size() <= 1) { - throw new RuntimeException("Metric name does not contain a customer id: " + metricName); - } - String customerId = metricParts.get(0); - if (CUSTOMERID.matcher(customerId).matches()) { - metricName = Joiner.on(".").join(metricParts.subList(1, metricParts.size())); - } - out.add(ReportPoint.newBuilder(rp).setMetric(metricName).setTable(customerId).build()); - } - } - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/GraphiteHostAnnotator.java b/java-lib/src/main/java/com/wavefront/ingester/GraphiteHostAnnotator.java deleted file mode 100644 index 3107cdf92..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/GraphiteHostAnnotator.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.wavefront.ingester; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.MessageToMessageDecoder; - -/** - * Given a raw graphite line, look for any host tag, and add it if implicit. Does not perform full - * decoding, though. - */ -public class GraphiteHostAnnotator extends MessageToMessageDecoder { - - private final String hostName; - private final List sourceTags = new ArrayList<>(); - - public GraphiteHostAnnotator(String hostName, final List customSourceTags) { - this.hostName = hostName; - this.sourceTags.add("source="); - this.sourceTags.add("host="); - this.sourceTags.addAll(customSourceTags.stream().map(customTag -> customTag + "=").collect(Collectors.toList())); - } - - // Decode from a possibly host-annotated graphite string to a definitely host-annotated graphite string. - @Override - protected void decode(ChannelHandlerContext ctx, String msg, List out) throws Exception { - for (String tag : sourceTags) { - int strIndex = msg.indexOf(tag); - // if a source tags is found and is followed by a non-whitespace tag value, add without change - if (strIndex > -1 && msg.length() - strIndex - tag.length() > 0 && msg.charAt(strIndex + tag.length()) > ' ') { - out.add(msg); - return; - } - } - out.add(msg + " source=\"" + hostName + "\""); - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/HistogramDecoder.java b/java-lib/src/main/java/com/wavefront/ingester/HistogramDecoder.java deleted file mode 100644 index c4dd538f1..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/HistogramDecoder.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.wavefront.ingester; - -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -import wavefront.report.ReportPoint; - -/** - * Decoder that takes in histograms of the type: - * - * [BinType] [Timestamp] [Centroids] [Metric] [Annotations] - * - * @author Tim Schmidt (tim@wavefront.com). - */ -public class HistogramDecoder implements Decoder { - private static final Logger logger = Logger.getLogger(HistogramDecoder.class.getCanonicalName()); - private static final AbstractIngesterFormatter FORMAT = - ReportPointIngesterFormatter.newBuilder() - .whiteSpace() - .binType() - .whiteSpace() - .appendOptionalTimestamp() - .adjustTimestamp() - .whiteSpace() - .centroids() - .whiteSpace() - .appendMetricName() - .whiteSpace() - .appendAnnotationsConsumer() - .build(); - - private final String defaultHostName; - - public HistogramDecoder() { - this("unknown"); - } - - public HistogramDecoder(String defaultHostName) { - this.defaultHostName = defaultHostName; - } - - - @Override - public void decodeReportPoints(String msg, List out, String customerId) { - ReportPoint point = FORMAT.drive(msg, defaultHostName, customerId); - if (point != null) { - out.add(ReportPoint.newBuilder(point).build()); - } - } - - @Override - public void decodeReportPoints(String msg, List out) { - logger.log(Level.WARNING, "This decoder does not support customerId extraction, ignoring " + msg); - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/Ingester.java b/java-lib/src/main/java/com/wavefront/ingester/Ingester.java deleted file mode 100644 index 525015f45..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/Ingester.java +++ /dev/null @@ -1,154 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.base.Function; - -import com.wavefront.common.TaggedMetricName; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Counter; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -import javax.annotation.Nullable; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelDuplexHandler; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.timeout.IdleState; -import io.netty.handler.timeout.IdleStateEvent; -import io.netty.handler.timeout.IdleStateHandler; - -/** - * Ingester thread that sets up decoders and a command handler to listen for metrics on a port. - * - * @author Clement Pang (clement@wavefront.com). - */ -public abstract class Ingester implements Runnable { - private static final Logger logger = Logger.getLogger(Ingester.class.getCanonicalName()); - - /** - * Default number of seconds before the channel idle timeout handler closes the connection. - */ - private static final int CHANNEL_IDLE_TIMEOUT_IN_SECS_DEFAULT = (int) TimeUnit.DAYS.toSeconds(1); - - /** - * The port that this ingester should be listening on - */ - protected final int listeningPort; - - /** - * The channel initializer object for the netty channel - */ - protected ChannelInitializer initializer; - - /** - * Counter metrics for accepted and terminated connections - */ - private Counter connectionsAccepted; - private Counter connectionsIdleClosed; - - @Nullable - protected Map, ?> parentChannelOptions; - @Nullable - protected Map, ?> childChannelOptions; - - public Ingester(@Nullable List> decoders, - ChannelHandler commandHandler, int port) { - this.listeningPort = port; - this.createInitializer(decoders, commandHandler); - initMetrics(port); - } - - public Ingester(ChannelHandler commandHandler, int port) { - this.listeningPort = port; - this.createInitializer(null, commandHandler); - initMetrics(port); - } - - public Ingester(ChannelInitializer initializer, int port) { - this.listeningPort = port; - this.initializer = initializer; - initMetrics(port); - } - - public Ingester withParentChannelOptions(Map, ?> parentChannelOptions) { - this.parentChannelOptions = parentChannelOptions; - return this; - } - - public Ingester withChildChannelOptions(Map, ?> childChannelOptions) { - this.childChannelOptions = childChannelOptions; - return this; - } - - private void initMetrics(int port) { - this.connectionsAccepted = Metrics.newCounter(new TaggedMetricName("listeners", "connections.accepted", - "port", String.valueOf(port))); - this.connectionsIdleClosed = Metrics.newCounter(new TaggedMetricName("listeners", "connections.idle.closed", - "port", String.valueOf(port))); - } - - /** - * Creates the ChannelInitializer for this ingester - */ - private void createInitializer(@Nullable final List> decoders, final ChannelHandler commandHandler) { - this.initializer = new ChannelInitializer() { - @Override - public void initChannel(SocketChannel ch) throws Exception { - connectionsAccepted.inc(); - ChannelPipeline pipeline = ch.pipeline(); - addDecoders(ch, decoders); - addIdleTimeoutHandler(pipeline); - pipeline.addLast(commandHandler); - } - }; - } - - /** - * Adds an idle timeout handler to the given pipeline - * - * @param pipeline the pipeline to add the idle timeout handler - */ - protected void addIdleTimeoutHandler(final ChannelPipeline pipeline) { - // Shared across all reports for proper batching - pipeline.addLast("idleStateHandler", - new IdleStateHandler(CHANNEL_IDLE_TIMEOUT_IN_SECS_DEFAULT, - 0, 0)); - pipeline.addLast("idleChannelTerminator", new ChannelDuplexHandler() { - @Override - public void userEventTriggered(ChannelHandlerContext ctx, - Object evt) throws Exception { - if (evt instanceof IdleStateEvent) { - if (((IdleStateEvent) evt).state() == IdleState.READER_IDLE) { - connectionsIdleClosed.inc(); - logger.warning("Closing idle connection, client inactivity timeout expired: " + ctx.channel()); - ctx.close(); - } - } - } - }); - } - - /** - * Adds additional decoders passed in during construction of this object (if not null). - * - * @param ch the channel and pipeline to add these decoders to - * @param decoders the list of decoders to add to the channel - */ - protected void addDecoders(final Channel ch, @Nullable List> decoders) { - if (decoders != null) { - ChannelPipeline pipeline = ch.pipeline(); - for (Function handler : decoders) { - pipeline.addLast(handler.apply(ch)); - } - } - } - -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/OpenTSDBDecoder.java b/java-lib/src/main/java/com/wavefront/ingester/OpenTSDBDecoder.java deleted file mode 100644 index d8e1663ac..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/OpenTSDBDecoder.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.base.Preconditions; - -import java.util.List; - -import wavefront.report.ReportPoint; - -/** - * OpenTSDB decoder that takes in a point of the type: - * - * PUT [metric] [timestamp] [value] [annotations] - * - * @author Clement Pang (clement@wavefront.com). - */ -public class OpenTSDBDecoder implements Decoder { - - private final String hostName; - private static final AbstractIngesterFormatter FORMAT = - ReportPointIngesterFormatter.newBuilder() - .whiteSpace() - .appendCaseInsensitiveLiteral("put").whiteSpace() - .appendMetricName().whiteSpace() - .appendTimestamp().whiteSpace() - .appendValue().whiteSpace() - .appendAnnotationsConsumer().whiteSpace().build(); - private List customSourceTags; - - public OpenTSDBDecoder(List customSourceTags) { - this.hostName = "unknown"; - Preconditions.checkNotNull(customSourceTags); - this.customSourceTags = customSourceTags; - } - - public OpenTSDBDecoder(String hostName, List customSourceTags) { - Preconditions.checkNotNull(hostName); - this.hostName = hostName; - Preconditions.checkNotNull(customSourceTags); - this.customSourceTags = customSourceTags; - } - - @Override - public void decodeReportPoints(String msg, List out, String customerId) { - ReportPoint point = FORMAT.drive(msg, hostName, customerId, customSourceTags); - if (out != null) { - out.add(point); - } - } - - @Override - public void decodeReportPoints(String msg, List out) { - ReportPoint point = FORMAT.drive(msg, hostName, "dummy", customSourceTags); - if (out != null) { - out.add(point); - } - } - - public String getDefaultHostName() { - return this.hostName; - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/PickleProtocolDecoder.java b/java-lib/src/main/java/com/wavefront/ingester/PickleProtocolDecoder.java deleted file mode 100644 index df48706d2..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/PickleProtocolDecoder.java +++ /dev/null @@ -1,139 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.base.Preconditions; -import com.wavefront.common.MetricMangler; - -import net.razorvine.pickle.Unpickler; -import wavefront.report.ReportPoint; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; - -/** - * Pickle protocol format decoder. - * https://docs.python.org/2/library/pickle.html - * @author Mike McLaughlin (mike@wavefront.com) - */ -public class PickleProtocolDecoder implements Decoder { - - protected static final Logger logger = Logger.getLogger(PickleProtocolDecoder.class.getCanonicalName()); - - private final int port; - private final String defaultHostName; - private final List customSourceTags; - private final MetricMangler metricMangler; - private final ThreadLocal unpicklerThreadLocal = new ThreadLocal() { - @Override - protected Unpickler initialValue() { - return new Unpickler(); - } - }; - - /** - * Constructor. - * @param hostName the default host name. - * @param customSourceTags list of source tags for this host. - * @param mangler the metric mangler object. - * @param port the listening port (for debug logging) - */ - public PickleProtocolDecoder(String hostName, List customSourceTags, - MetricMangler mangler, int port) { - Preconditions.checkNotNull(hostName); - this.defaultHostName = hostName; - Preconditions.checkNotNull(customSourceTags); - this.customSourceTags = customSourceTags; - this.metricMangler = mangler; - this.port = port; - } - - @Override - public void decodeReportPoints(byte[] msg, List out, String customerId) { - InputStream is = new ByteArrayInputStream(msg); - Object dataRaw; - try { - dataRaw = unpicklerThreadLocal.get().load(is); - if (!(dataRaw instanceof List)) { - throw new IllegalArgumentException( - String.format("[%d] unable to unpickle data (unpickle did not return list)", port)); - } - } catch (final IOException ioe) { - throw new IllegalArgumentException(String.format("[%d] unable to unpickle data", port), ioe); - } - - // [(path, (timestamp, value)), ...] - List data = (List) dataRaw; - for (Object[] o : data) { - Object[] details = (Object[])o[1]; - if (details == null || details.length != 2) { - logger.warning(String.format("[%d] Unexpected pickle protocol input", port)); - continue; - } - long ts; - if (details[0] == null) { - logger.warning(String.format("[%d] Unexpected pickle protocol input (timestamp is null)", port)); - continue; - } else if (details[0] instanceof Double) { - ts = ((Double)details[0]).longValue() * 1000; - } else if (details[0] instanceof Long) { - ts = ((Long)details[0]).longValue() * 1000; - } else if (details[0] instanceof Integer) { - ts = ((Integer)details[0]).longValue() * 1000; - } else { - logger.warning(String.format("[%d] Unexpected pickle protocol input (details[0]: %s)", - port, details[0].getClass().getName())); - continue; - } - - if (details[1] == null) { - continue; - } - - double value; - if (details[1] instanceof Double) { - value = ((Double)details[1]).doubleValue(); - } else if (details[1] instanceof Long) { - value = ((Long)details[1]).longValue(); - } else if (details[1] instanceof Integer) { - value = ((Integer)details[1]).intValue(); - } else { - logger.warning(String.format("[%d] Unexpected pickle protocol input (value is null)", port)); - continue; - } - - ReportPoint point = new ReportPoint(); - MetricMangler.MetricComponents components = - this.metricMangler.extractComponents(o[0].toString()); - point.setMetric(components.metric); - String host = components.source; - final Map annotations = point.getAnnotations(); - if (host == null && annotations != null) { - // iterate over the set of custom tags, breaking when one is found - for (final String tag : customSourceTags) { - host = annotations.remove(tag); - if (host != null) { - break; - } - } - if (host == null) { - host = this.defaultHostName; - } - } - point.setHost(host); - point.setTable(customerId); - point.setTimestamp(ts); - point.setValue(value); - point.setAnnotations(Collections.emptyMap()); - out.add(point); - } - } - - @Override - public void decodeReportPoints(byte[] msg, List out) { - decodeReportPoints(msg, out, "dummy"); - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/ReportPointDecoderWrapper.java b/java-lib/src/main/java/com/wavefront/ingester/ReportPointDecoderWrapper.java deleted file mode 100644 index f1363b537..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/ReportPointDecoderWrapper.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.wavefront.ingester; - -import java.util.List; - -import javax.validation.constraints.NotNull; - -import wavefront.report.ReportPoint; - -/** - * Wraps {@link Decoder} as {@link ReportableEntityDecoder}. - * - * @author vasily@wavefront.com - */ -public class ReportPointDecoderWrapper implements ReportableEntityDecoder { - - private final Decoder delegate; - - public ReportPointDecoderWrapper(@NotNull Decoder delegate) { - this.delegate = delegate; - } - - @Override - public void decode(String msg, List out, String customerId) { - delegate.decodeReportPoints(msg, out, customerId); - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/ReportPointIngesterFormatter.java b/java-lib/src/main/java/com/wavefront/ingester/ReportPointIngesterFormatter.java deleted file mode 100644 index e2305cc7a..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/ReportPointIngesterFormatter.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.wavefront.ingester; - -import com.wavefront.common.Clock; -import com.wavefront.common.MetricConstants; - -import org.antlr.v4.runtime.Token; - -import java.util.List; -import java.util.Map; -import java.util.Queue; - -import javax.annotation.Nullable; - -import wavefront.report.ReportPoint; - -/** - * Builder pattern for creating new ingestion formats. Inspired by the date time formatters in - * Joda. - * - * @author Clement Pang (clement@wavefront.com). - */ -public class ReportPointIngesterFormatter extends AbstractIngesterFormatter { - - private ReportPointIngesterFormatter(List elements) { - super(elements); - } - - /** - * A builder pattern to create a format for the report point parse. - */ - public static class ReportPointIngesterFormatBuilder extends IngesterFormatBuilder { - - @Override - public ReportPointIngesterFormatter build() { - return new ReportPointIngesterFormatter(elements); - } - } - - public static IngesterFormatBuilder newBuilder() { - return new ReportPointIngesterFormatBuilder(); - } - - @Override - public ReportPoint drive(String input, String defaultHostName, String customerId, - @Nullable List customSourceTags) { - Queue queue = getQueue(input); - - ReportPoint point = new ReportPoint(); - point.setTable(customerId); - // if the point has a timestamp, this would be overriden - point.setTimestamp(Clock.now()); - AbstractWrapper wrapper = new ReportPointWrapper(point); - try { - for (FormatterElement element : elements) { - element.consume(queue, wrapper); - } - } catch (Exception ex) { - throw new RuntimeException("Could not parse: " + input, ex); - } - if (!queue.isEmpty()) { - throw new RuntimeException("Could not parse: " + input); - } - - // Delta metrics cannot have negative values - if ((point.getMetric().startsWith(MetricConstants.DELTA_PREFIX) || point.getMetric().startsWith(MetricConstants.DELTA_PREFIX_2)) && - point.getValue() instanceof Number) { - double v = ((Number) point.getValue()).doubleValue(); - if (v <= 0) { - throw new RuntimeException("Delta metrics cannot be non-positive: " + input); - } - } - - String host = null; - Map annotations = point.getAnnotations(); - if (annotations != null) { - host = annotations.remove("source"); - if (host == null) { - host = annotations.remove("host"); - } else if (annotations.containsKey("host")) { - // we have to move this elsewhere since during querying, - // host= would be interpreted as host and not a point tag - annotations.put("_host", annotations.remove("host")); - } - if (annotations.containsKey("tag")) { - annotations.put("_tag", annotations.remove("tag")); - } - if (host == null && customSourceTags != null) { - // iterate over the set of custom tags, breaking when one is found - for (String tag : customSourceTags) { - host = annotations.get(tag); - if (host != null) { - break; - } - } - } - } - if (host == null) { - host = defaultHostName; - } - point.setHost(host); - return ReportPoint.newBuilder(point).build(); - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/ReportPointSerializer.java b/java-lib/src/main/java/com/wavefront/ingester/ReportPointSerializer.java deleted file mode 100644 index e4c8fc2d8..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/ReportPointSerializer.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.annotations.VisibleForTesting; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.DateUtils; - -import java.util.Map; -import java.util.function.Function; - -import javax.annotation.Nullable; - -import wavefront.report.ReportPoint; - -/** - * Convert a {@link ReportPoint} to its string representation in a canonical format (quoted metric name, - * tag values and keys (except for "source"). Supports numeric and {@link wavefront.report.Histogram} values. - * - * @author vasily@wavefront.com - */ -public class ReportPointSerializer implements Function { - - @Override - public String apply(ReportPoint point) { - return pointToString(point); - } - - private static String quote = "\""; - - private static String escapeQuotes(String raw) { - return StringUtils.replace(raw, quote, "\\\""); - } - - private static void appendTagMap(StringBuilder sb, @Nullable Map tags) { - if (tags == null) { - return; - } - for (Map.Entry entry : tags.entrySet()) { - sb.append(' ').append(quote).append(escapeQuotes(entry.getKey())).append(quote) - .append("=") - .append(quote).append(escapeQuotes(entry.getValue())).append(quote); - } - } - - @VisibleForTesting - protected static String pointToString(ReportPoint point) { - if (point.getValue() instanceof Double || point.getValue() instanceof Long || point.getValue() instanceof String) { - StringBuilder sb = new StringBuilder(quote) - .append(escapeQuotes(point.getMetric())).append(quote).append(" ") - .append(point.getValue()).append(" ") - .append(point.getTimestamp() / 1000).append(" ") - .append("source=").append(quote).append(escapeQuotes(point.getHost())).append(quote); - appendTagMap(sb, point.getAnnotations()); - return sb.toString(); - } else if (point.getValue() instanceof wavefront.report.Histogram) { - wavefront.report.Histogram h = (wavefront.report.Histogram) point.getValue(); - - StringBuilder sb = new StringBuilder(); - - // BinType - switch (h.getDuration()) { - case (int) DateUtils.MILLIS_PER_MINUTE: - sb.append("!M "); - break; - case (int) DateUtils.MILLIS_PER_HOUR: - sb.append("!H "); - break; - case (int) DateUtils.MILLIS_PER_DAY: - sb.append("!D "); - break; - default: - throw new RuntimeException("Unexpected histogram duration " + h.getDuration()); - } - - // Timestamp - sb.append(point.getTimestamp() / 1000).append(' '); - - // Centroids - int numCentroids = Math.min(CollectionUtils.size(h.getBins()), CollectionUtils.size(h.getCounts())); - for (int i = 0; i < numCentroids; ++i) { - // Count - sb.append('#').append(h.getCounts().get(i)).append(' '); - // Mean - sb.append(h.getBins().get(i)).append(' '); - } - - // Metric - sb.append(quote).append(escapeQuotes(point.getMetric())).append(quote).append(" "); - - // Source - sb.append("source=").append(quote).append(escapeQuotes(point.getHost())).append(quote); - appendTagMap(sb, point.getAnnotations()); - return sb.toString(); - } - throw new RuntimeException("Unsupported value class: " + point.getValue().getClass().getCanonicalName()); - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/ReportSourceTagDecoder.java b/java-lib/src/main/java/com/wavefront/ingester/ReportSourceTagDecoder.java deleted file mode 100644 index 9967205e8..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/ReportSourceTagDecoder.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.wavefront.ingester; - -import java.util.List; - -import wavefront.report.ReportSourceTag; - -/** - * This class is used to decode the source tags sent by the clients. - * - * [@SourceTag action=save source=source sourceTag1 sourceTag2] - * [@SourceDescription action=save source=source description=Description] - * - * @author Suranjan Pramanik (suranjan@wavefront.com). - */ -public class ReportSourceTagDecoder implements ReportableEntityDecoder{ - - public static final String SOURCE_TAG = "@SourceTag"; - public static final String SOURCE_DESCRIPTION = "@SourceDescription"; - - private static final AbstractIngesterFormatter FORMAT = - ReportSourceTagIngesterFormatter.newBuilder() - .whiteSpace() - .appendCaseSensitiveLiterals(new String[]{SOURCE_TAG, SOURCE_DESCRIPTION}) - .whiteSpace() - .appendLoopOfKeywords() - .whiteSpace() - .appendLoopOfValues() - .build(); - - @Override - public void decode(String msg, List out, String customerId) { - ReportSourceTag reportSourceTag = FORMAT.drive(msg, "dummy", customerId, null); - if (out != null) out.add(reportSourceTag); - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/ReportSourceTagIngesterFormatter.java b/java-lib/src/main/java/com/wavefront/ingester/ReportSourceTagIngesterFormatter.java deleted file mode 100644 index 1f99d6e85..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/ReportSourceTagIngesterFormatter.java +++ /dev/null @@ -1,102 +0,0 @@ -package com.wavefront.ingester; - -import org.antlr.v4.runtime.Token; - -import java.util.List; -import java.util.Map; -import java.util.Queue; - -import wavefront.report.ReportSourceTag; - -/** - * This class can be used to parse sourceTags and description. - * - * @author Suranjan Pramanik (suranjan@wavefront.com). - */ -public class ReportSourceTagIngesterFormatter extends AbstractIngesterFormatter { - - public static final String SOURCE = "source"; - public static final String DESCRIPTION = "description"; - public static final String ACTION = "action"; - public static final String ACTION_SAVE = "save"; - public static final String ACTION_DELETE = "delete"; - - private ReportSourceTagIngesterFormatter(List elements) { - super(elements); - } - - /** - * Factory method to create an instance of the format builder. - * - * @return The builder, which can be used to create the parser. - */ - public static SourceTagIngesterFormatBuilder newBuilder() { - return new SourceTagIngesterFormatBuilder(); - } - - /** - * This method can be used to parse the input line into a ReportSourceTag object. - * - * @return The parsed ReportSourceTag object. - */ - @Override - public ReportSourceTag drive(String input, String defaultHostName, String customerId, - List customerSourceTags) { - Queue queue = getQueue(input); - - ReportSourceTag sourceTag = new ReportSourceTag(); - ReportSourceTagWrapper wrapper = new ReportSourceTagWrapper(sourceTag); - try { - for (FormatterElement element : elements) { - element.consume(queue, wrapper); - } - } catch (Exception ex) { - throw new RuntimeException("Could not parse: " + input, ex); - } - if (!queue.isEmpty()) { - throw new RuntimeException("Could not parse: " + input); - } - Map annotations = wrapper.getAnnotationMap(); - for (Map.Entry entry : annotations.entrySet()) { - switch (entry.getKey()) { - case ReportSourceTagIngesterFormatter.ACTION: - sourceTag.setAction(entry.getValue()); - break; - case ReportSourceTagIngesterFormatter.SOURCE: - sourceTag.setSource(entry.getValue()); - break; - case ReportSourceTagIngesterFormatter.DESCRIPTION: - sourceTag.setDescription(entry.getValue()); - break; - default: - throw new RuntimeException("Unknown tag key = " + entry.getKey() + " specified."); - } - } - - // verify the values - especially 'action' field - if (sourceTag.getSource() == null) - throw new RuntimeException("No source key was present in the input: " + input); - - if (sourceTag.getAction() != null) { - // verify that only 'add' or 'delete' is present - String actionStr = sourceTag.getAction(); - if (!actionStr.equals(ACTION_SAVE) && !actionStr.equals(ACTION_DELETE)) - throw new RuntimeException("Action string did not match save/delete: " + input); - } else { - // no value was specified hence throw an exception - throw new RuntimeException("No action key was present in the input: " + input); - } - return ReportSourceTag.newBuilder(sourceTag).build(); - } - - /** - * A builder pattern to create a format for the source tag parser. - */ - public static class SourceTagIngesterFormatBuilder extends IngesterFormatBuilder { - - @Override - public ReportSourceTagIngesterFormatter build() { - return new ReportSourceTagIngesterFormatter(elements); - } - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/ReportSourceTagSerializer.java b/java-lib/src/main/java/com/wavefront/ingester/ReportSourceTagSerializer.java deleted file mode 100644 index 82a95cbf4..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/ReportSourceTagSerializer.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.wavefront.ingester; - -import java.util.function.Function; - -import wavefront.report.ReportSourceTag; - -/** - * Convert a {@link ReportSourceTag} to its string representation. - * - * @author vasily@wavefront.com - */ -public class ReportSourceTagSerializer implements Function { - @Override - public String apply(ReportSourceTag input) { - return input.toString(); - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/ReportableEntityDecoder.java b/java-lib/src/main/java/com/wavefront/ingester/ReportableEntityDecoder.java deleted file mode 100644 index 7f66fa5ed..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/ReportableEntityDecoder.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.wavefront.ingester; - -import java.util.List; - -import wavefront.report.ReportPoint; - -/** - * A decoder for input data. A more generic version of {@link Decoder}, - * that supports other object types besides {@link ReportPoint}. - * - * @author vasily@wavefront.com - */ -public interface ReportableEntityDecoder { - /** - * Decode entities (points, spans, etc) and dump them into an output array. The supplied customer id will be set - * and no customer id extraction will be attempted. - * - * @param msg Message to parse. - * @param out List to output the parsed point. - * @param customerId The customer id to use as the table for the resulting entities. - */ - void decode(T msg, List out, String customerId); - - /** - * Certain decoders support decoding the customer id from the input line itself. - * - * @param msg Message to parse. - * @param out List to output the parsed point. - */ - default void decode(T msg, List out) { - decode(msg, out, "dummy"); - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/SpanDecoder.java b/java-lib/src/main/java/com/wavefront/ingester/SpanDecoder.java deleted file mode 100644 index 57e42f704..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/SpanDecoder.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.base.Preconditions; - -import java.util.List; - -import wavefront.report.Span; - -/** - * Span decoder that takes in data in the following format: - * - * [span name] [annotations] [timestamp] [duration|timestamp] - * - * @author vasily@wavefront.com - */ -public class SpanDecoder implements ReportableEntityDecoder { - - private final String hostName; - private static final AbstractIngesterFormatter FORMAT = - SpanIngesterFormatter.newBuilder() - .whiteSpace() - .appendName().whiteSpace() - .appendBoundedAnnotationsConsumer().whiteSpace() - .appendRawTimestamp().whiteSpace() - .appendDuration().whiteSpace() - .build(); - - public SpanDecoder(String hostName) { - Preconditions.checkNotNull(hostName); - this.hostName = hostName; - } - - @Override - public void decode(String msg, List out, String customerId) { - Span span = FORMAT.drive(msg, hostName, customerId); - if (out != null) { - out.add(span); - } - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/SpanIngesterFormatter.java b/java-lib/src/main/java/com/wavefront/ingester/SpanIngesterFormatter.java deleted file mode 100644 index f65679ccd..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/SpanIngesterFormatter.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.wavefront.ingester; - -import org.antlr.v4.runtime.Token; - -import java.util.Iterator; -import java.util.List; -import java.util.Queue; - -import javax.annotation.Nullable; - -import wavefront.report.Annotation; -import wavefront.report.Span; - -/** - * Builder for Span formatter. - * - * @author vasily@wavefront.com - */ -public class SpanIngesterFormatter extends AbstractIngesterFormatter { - - private SpanIngesterFormatter(List elements) { - super(elements); - } - - /** - * A builder pattern to create a format for span parsing. - */ - public static class SpanIngesterFormatBuilder extends IngesterFormatBuilder { - - @Override - public SpanIngesterFormatter build() { - return new SpanIngesterFormatter(elements); - } - } - - public static IngesterFormatBuilder newBuilder() { - return new SpanIngesterFormatBuilder(); - } - - @Override - public Span drive(String input, String defaultHostName, String customerId, - @Nullable List customSourceTags) { - Queue queue = getQueue(input); - - Span span = new Span(); - span.setCustomer(customerId); - if (defaultHostName != null) { - span.setSource(defaultHostName); - } - AbstractWrapper wrapper = new SpanWrapper(span); - try { - for (FormatterElement element : elements) { - element.consume(queue, wrapper); - } - } catch (Exception ex) { - throw new RuntimeException("Could not parse: " + input, ex); - } - if (!queue.isEmpty()) { - throw new RuntimeException("Could not parse: " + input); - } - - List annotations = span.getAnnotations(); - if (annotations != null) { - boolean hasTrueSource = false; - Iterator iterator = annotations.iterator(); - while (iterator.hasNext()) { - final Annotation annotation = iterator.next(); - if (customSourceTags != null && !hasTrueSource && customSourceTags.contains(annotation.getKey())) { - span.setSource(annotation.getValue()); - } - switch (annotation.getKey()) { - case "source": - case "host": - span.setSource(annotation.getValue()); - iterator.remove(); - hasTrueSource = true; - break; - case "spanId": - span.setSpanId(annotation.getValue()); - iterator.remove(); - break; - case "traceId": - span.setTraceId(annotation.getValue()); - iterator.remove(); - break; - default: - break; - } - } - } - - if (span.getSource() == null) { - throw new RuntimeException("source can't be null: " + input); - } - if (span.getSpanId() == null) { - throw new RuntimeException("spanId can't be null: " + input); - } - if (span.getTraceId() == null) { - throw new RuntimeException("traceId can't be null: " + input); - } - return Span.newBuilder(span).build(); - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/SpanSerializer.java b/java-lib/src/main/java/com/wavefront/ingester/SpanSerializer.java deleted file mode 100644 index d4b2ce31b..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/SpanSerializer.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.wavefront.ingester; - -import org.apache.commons.lang.StringUtils; - -import java.util.function.Function; - -import wavefront.report.Annotation; -import wavefront.report.Span; - -/** - * Convert a {@link Span} to its string representation in a canonical format (quoted name and annotations). - * - * @author vasily@wavefront.com - */ -public class SpanSerializer implements Function { - - @Override - public String apply(Span span) { - return spanToString(span); - } - - private static String quote = "\""; - private static String escapedQuote = "\\\""; - - private static String escapeQuotes(String raw) { - return StringUtils.replace(raw, quote, escapedQuote); - } - - static String spanToString(Span span) { - StringBuilder sb = new StringBuilder(quote) - .append(escapeQuotes(span.getName())).append(quote).append(' '); - if (span.getSource() != null) { - sb.append("source=").append(quote).append(escapeQuotes(span.getSource())).append(quote).append(' '); - } - if (span.getSpanId() != null) { - sb.append("spanId=").append(quote).append(escapeQuotes(span.getSpanId())).append(quote).append(' '); - } - if (span.getTraceId() != null) { - sb.append("traceId=").append(quote).append(escapeQuotes(span.getTraceId())).append(quote); - } - if (span.getAnnotations() != null) { - for (Annotation entry : span.getAnnotations()) { - sb.append(' ').append(quote).append(escapeQuotes(entry.getKey())).append(quote) - .append("=") - .append(quote).append(escapeQuotes(entry.getValue())).append(quote); - } - } - sb.append(' ') - .append(span.getStartMillis()) - .append(' ') - .append(span.getDuration()); - return sb.toString(); - } -} - diff --git a/java-lib/src/main/java/com/wavefront/ingester/StreamIngester.java b/java-lib/src/main/java/com/wavefront/ingester/StreamIngester.java deleted file mode 100644 index 66f92d971..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/StreamIngester.java +++ /dev/null @@ -1,141 +0,0 @@ -package com.wavefront.ingester; - -import com.wavefront.metrics.ExpectedAgentMetric; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Counter; -import com.yammer.metrics.core.MetricName; - -import java.net.BindException; -import java.util.Map; -import java.util.logging.Logger; -import java.util.logging.Level; - -import javax.annotation.Nullable; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.ServerChannel; -import io.netty.channel.epoll.Epoll; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.epoll.EpollServerSocketChannel; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.channel.ChannelInboundHandler; -import io.netty.handler.codec.bytes.ByteArrayDecoder; - -/** - * Ingester thread that sets up decoders and a command handler to listen for metrics on a port. - * @author Mike McLaughlin (mike@wavefront.com) - */ -public class StreamIngester implements Runnable { - - protected static final Logger logger = Logger.getLogger(StreamIngester.class.getName()); - private Counter activeListeners = Metrics.newCounter(ExpectedAgentMetric.ACTIVE_LISTENERS.metricName); - private Counter bindErrors = Metrics.newCounter(ExpectedAgentMetric.LISTENERS_BIND_ERRORS.metricName); - - public interface FrameDecoderFactory { - ChannelInboundHandler getDecoder(); - } - - private final ChannelHandler commandHandler; - private final int listeningPort; - private final FrameDecoderFactory frameDecoderFactory; - - @Nullable - protected Map, ?> parentChannelOptions; - @Nullable - protected Map, ?> childChannelOptions; - - public StreamIngester(FrameDecoderFactory frameDecoderFactory, - ChannelHandler commandHandler, int port) { - this.listeningPort = port; - this.commandHandler = commandHandler; - this.frameDecoderFactory = frameDecoderFactory; - } - - public StreamIngester withParentChannelOptions(Map, ?> parentChannelOptions) { - this.parentChannelOptions = parentChannelOptions; - return this; - } - - public StreamIngester withChildChannelOptions(Map, ?> childChannelOptions) { - this.childChannelOptions = childChannelOptions; - return this; - } - - - public void run() { - activeListeners.inc(); - // Configure the server. - ServerBootstrap b = new ServerBootstrap(); - EventLoopGroup parentGroup; - EventLoopGroup childGroup; - Class socketChannelClass; - if (Epoll.isAvailable()) { - logger.fine("Using native socket transport for port " + listeningPort); - parentGroup = new EpollEventLoopGroup(1); - childGroup = new EpollEventLoopGroup(); - socketChannelClass = EpollServerSocketChannel.class; - } else { - logger.fine("Using NIO socket transport for port " + listeningPort); - parentGroup = new NioEventLoopGroup(1); - childGroup = new NioEventLoopGroup(); - socketChannelClass = NioServerSocketChannel.class; - } - try { - b.group(parentGroup, childGroup) - .channel(socketChannelClass) - .option(ChannelOption.SO_BACKLOG, 1024) - .localAddress(listeningPort) - .childHandler(new ChannelInitializer() { - @Override - public void initChannel(SocketChannel ch) throws Exception { - ChannelPipeline pipeline = ch.pipeline(); - pipeline.addLast("frame decoder", frameDecoderFactory.getDecoder()); - pipeline.addLast("byte array decoder", new ByteArrayDecoder()); - pipeline.addLast(commandHandler); - } - }); - - if (parentChannelOptions != null) { - for (Map.Entry, ?> entry : parentChannelOptions.entrySet()) - { - b.option((ChannelOption) entry.getKey(), entry.getValue()); - } - } - if (childChannelOptions != null) { - for (Map.Entry, ?> entry : childChannelOptions.entrySet()) - { - b.childOption((ChannelOption) entry.getKey(), entry.getValue()); - } - } - - // Start the server. - ChannelFuture f = b.bind().sync(); - - // Wait until the server socket is closed. - f.channel().closeFuture().sync(); - } catch (final InterruptedException e) { - logger.log(Level.WARNING, "Interrupted"); - parentGroup.shutdownGracefully(); - childGroup.shutdownGracefully(); - logger.info("Listener on port " + String.valueOf(listeningPort) + " shut down"); - } catch (Exception e) { - // ChannelFuture throws undeclared checked exceptions, so we need to handle it - if (e instanceof BindException) { - bindErrors.inc(); - logger.severe("Unable to start listener - port " + String.valueOf(listeningPort) + " is already in use!"); - } else { - logger.log(Level.SEVERE, "StreamIngester exception: ", e); - } - } finally { - activeListeners.dec(); - } - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/StringLineIngester.java b/java-lib/src/main/java/com/wavefront/ingester/StringLineIngester.java deleted file mode 100644 index 8821bab0e..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/StringLineIngester.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.base.Charsets; -import com.google.common.base.Function; - -import org.apache.commons.lang.StringUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import javax.annotation.Nullable; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandler; -import io.netty.handler.codec.LineBasedFrameDecoder; -import io.netty.handler.codec.string.StringDecoder; - -/** - * Default Ingester thread that sets up decoders and a command handler to listen for metrics that - * are string formatted lines on a port. - */ -public class StringLineIngester extends TcpIngester { - - private static final String PUSH_DATA_DELIMETER = "\n"; - private static final int MAXIMUM_FRAME_LENGTH_DEFAULT = 4096; - - public StringLineIngester(List> decoders, - ChannelHandler commandHandler, int port, int maxLength) { - super(createDecoderList(decoders, maxLength), commandHandler, port); - } - - public StringLineIngester(List> decoders, - ChannelHandler commandHandler, int port) { - this(decoders, commandHandler, port, MAXIMUM_FRAME_LENGTH_DEFAULT); - } - - public StringLineIngester(ChannelHandler commandHandler, int port, int maxLength) { - super(createDecoderList(null, maxLength), commandHandler, port); - } - - public StringLineIngester(ChannelHandler commandHandler, int port) { - this(commandHandler, port, MAXIMUM_FRAME_LENGTH_DEFAULT); - } - - /** - * Returns a copy of the given list plus inserts the 2 decoders needed for this specific ingester - * (LineBasedFrameDecoder and StringDecoder) - * - * @param decoders the starting list - * @param maxLength maximum frame length for decoding the input stream - * @return copy of the provided list with additional decodiers prepended - */ - private static List> createDecoderList(@Nullable final List> decoders, int maxLength) { - final List> copy; - if (decoders == null) { - copy = new ArrayList<>(); - } else { - copy = new ArrayList<>(decoders); - } - copy.add(0, new Function() { - @Override - public ChannelHandler apply(Channel input) { - return new LineBasedFrameDecoder(maxLength, true, false); - } - }); - copy.add(1, new Function() { - @Override - public ChannelHandler apply(Channel input) { - return new StringDecoder(Charsets.UTF_8); - } - }); - - return copy; - } - - public static List unjoinPushData(String pushData) { - return Arrays.asList(StringUtils.split(pushData, PUSH_DATA_DELIMETER)); - } - - public static String joinPushData(List pushData) { - return StringUtils.join(pushData, PUSH_DATA_DELIMETER); - } - - public static List indexPushData(String pushData) { - List index = new ArrayList<>(); - index.add(0); - int lastIndex = pushData.indexOf(PUSH_DATA_DELIMETER); - final int delimiterLength = PUSH_DATA_DELIMETER.length(); - while (lastIndex != -1) { - index.add(lastIndex); - index.add(lastIndex + delimiterLength); - lastIndex = pushData.indexOf(PUSH_DATA_DELIMETER, lastIndex + delimiterLength); - } - index.add(pushData.length()); - return index; - } - - /** - * Calculates the number of points in the pushData payload - * @param pushData a delimited string with the points payload - * @return number of points - */ - public static int pushDataSize(String pushData) { - int length = StringUtils.countMatches(pushData, PUSH_DATA_DELIMETER); - return length > 0 - ? length + 1 - : (pushData.length() > 0 ? 1 : 0); - - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/TcpIngester.java b/java-lib/src/main/java/com/wavefront/ingester/TcpIngester.java deleted file mode 100644 index f5704af01..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/TcpIngester.java +++ /dev/null @@ -1,108 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.base.Function; - -import com.wavefront.metrics.ExpectedAgentMetric; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Counter; - -import java.net.BindException; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.ServerChannel; -import io.netty.channel.epoll.Epoll; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.epoll.EpollServerSocketChannel; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioServerSocketChannel; - -/** - * Ingester thread that sets up decoders and a command handler to listen for metrics on a port. - * @author Mike McLaughlin (mike@wavefront.com) - */ -public class TcpIngester extends Ingester { - - private static final Logger logger = - Logger.getLogger(TcpIngester.class.getCanonicalName()); - private Counter activeListeners = Metrics.newCounter(ExpectedAgentMetric.ACTIVE_LISTENERS.metricName); - private Counter bindErrors = Metrics.newCounter(ExpectedAgentMetric.LISTENERS_BIND_ERRORS.metricName); - - public TcpIngester(List> decoders, - ChannelHandler commandHandler, int port) { - super(decoders, commandHandler, port); - } - - public TcpIngester(ChannelInitializer initializer, int port) { - super(initializer, port); - } - - public void run() { - activeListeners.inc(); - ServerBootstrap b = new ServerBootstrap(); - EventLoopGroup parentGroup; - EventLoopGroup childGroup; - Class socketChannelClass; - if (Epoll.isAvailable()) { - logger.fine("Using native socket transport for port " + listeningPort); - parentGroup = new EpollEventLoopGroup(1); - childGroup = new EpollEventLoopGroup(); - socketChannelClass = EpollServerSocketChannel.class; - } else { - logger.fine("Using NIO socket transport for port " + listeningPort); - parentGroup = new NioEventLoopGroup(1); - childGroup = new NioEventLoopGroup(); - socketChannelClass = NioServerSocketChannel.class; - } - try { - b.group(parentGroup, childGroup) - .channel(socketChannelClass) - .option(ChannelOption.SO_BACKLOG, 1024) - .localAddress(listeningPort) - .childHandler(initializer); - - if (parentChannelOptions != null) { - for (Map.Entry, ?> entry : parentChannelOptions.entrySet()) - { - b.option((ChannelOption) entry.getKey(), entry.getValue()); - } - } - if (childChannelOptions != null) { - for (Map.Entry, ?> entry : childChannelOptions.entrySet()) - { - b.childOption((ChannelOption) entry.getKey(), entry.getValue()); - } - } - - // Start the server. - ChannelFuture f = b.bind().sync(); - - // Wait until the server socket is closed. - f.channel().closeFuture().sync(); - } catch (final InterruptedException e) { - logger.log(Level.WARNING, "Interrupted"); - parentGroup.shutdownGracefully(); - childGroup.shutdownGracefully(); - logger.info("Listener on port " + String.valueOf(listeningPort) + " shut down"); - } catch (Exception e) { - // ChannelFuture throws undeclared checked exceptions, so we need to handle it - if (e instanceof BindException) { - bindErrors.inc(); - logger.severe("Unable to start listener - port " + String.valueOf(listeningPort) + " is already in use!"); - } else { - logger.log(Level.SEVERE, "TcpIngester exception: ", e); - } - } finally { - activeListeners.dec(); - } - } -} diff --git a/java-lib/src/main/java/com/wavefront/ingester/UdpIngester.java b/java-lib/src/main/java/com/wavefront/ingester/UdpIngester.java deleted file mode 100644 index b489cef00..000000000 --- a/java-lib/src/main/java/com/wavefront/ingester/UdpIngester.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.base.Function; - -import com.wavefront.metrics.ExpectedAgentMetric; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Counter; - -import java.net.BindException; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -import io.netty.bootstrap.Bootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandler; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.epoll.Epoll; -import io.netty.channel.epoll.EpollDatagramChannel; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioDatagramChannel; - -/** - * Bootstrapping for datagram ingester channels on a socket. - * - * @author Tim Schmidt (tim@wavefront.com). - */ -public class UdpIngester extends Ingester { - private static final Logger logger = - Logger.getLogger(UdpIngester.class.getCanonicalName()); - private Counter activeListeners = Metrics.newCounter(ExpectedAgentMetric.ACTIVE_LISTENERS.metricName); - private Counter bindErrors = Metrics.newCounter(ExpectedAgentMetric.LISTENERS_BIND_ERRORS.metricName); - - public UdpIngester(List> decoders, - ChannelHandler commandHandler, int port) { - super(decoders, commandHandler, port); - } - - @Override - public void run() { - activeListeners.inc(); - Bootstrap bootstrap = new Bootstrap(); - EventLoopGroup group; - Class datagramChannelClass; - if (Epoll.isAvailable()) { - logger.fine("Using native socket transport for port " + listeningPort); - group = new EpollEventLoopGroup(); - datagramChannelClass = EpollDatagramChannel.class; - } else { - logger.fine("Using NIO socket transport for port " + listeningPort); - group = new NioEventLoopGroup(); - datagramChannelClass = NioDatagramChannel.class; - } - try { - bootstrap - .group(group) - .channel(datagramChannelClass) - .localAddress(listeningPort) - .handler(initializer); - - // Start the server. - bootstrap.bind().sync().channel().closeFuture().sync(); - } catch (final InterruptedException e) { - logger.log(Level.WARNING, "Interrupted", e); - } catch (Exception e) { - // ChannelFuture throws undeclared checked exceptions, so we need to handle it - if (e instanceof BindException) { - bindErrors.inc(); - logger.severe("Unable to start listener - port " + String.valueOf(listeningPort) + " is already in use!"); - } else { - logger.log(Level.SEVERE, "UdpIngester exception: ", e); - } - } finally { - activeListeners.dec(); - group.shutdownGracefully(); - } - } -} diff --git a/java-lib/src/main/java/com/wavefront/metrics/ExpectedAgentMetric.java b/java-lib/src/main/java/com/wavefront/metrics/ExpectedAgentMetric.java deleted file mode 100644 index 62330d63b..000000000 --- a/java-lib/src/main/java/com/wavefront/metrics/ExpectedAgentMetric.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.wavefront.metrics; - -import com.yammer.metrics.core.MetricName; - -/** - * There are some metrics that need to have well known names. - * - * @author Andrew Kao (andrew@wavefront.com) - */ -public enum ExpectedAgentMetric { - ACTIVE_LISTENERS(new MetricName("listeners", "", "active")), - LISTENERS_BIND_ERRORS(new MetricName("listeners", "", "bind-errors")), - BUFFER_BYTES_LEFT(new MetricName("buffer", "", "bytes-left")), - BUFFER_BYTES_PER_MINUTE(new MetricName("buffer", "", "fill-rate")), - CURRENT_QUEUE_SIZE(new MetricName("buffer", "", "task-count")), - RDNS_CACHE_SIZE(new MetricName("listeners", "", "rdns-cache-size")); - - public MetricName metricName; - - public String getCombinedName() { - return metricName.getGroup() + "." + metricName.getName(); - } - - ExpectedAgentMetric(MetricName metricName) { - this.metricName = metricName; - } -} diff --git a/java-lib/src/main/java/com/wavefront/metrics/JsonMetricsGenerator.java b/java-lib/src/main/java/com/wavefront/metrics/JsonMetricsGenerator.java deleted file mode 100644 index 3b2950b0f..000000000 --- a/java-lib/src/main/java/com/wavefront/metrics/JsonMetricsGenerator.java +++ /dev/null @@ -1,488 +0,0 @@ -package com.wavefront.metrics; - -import com.fasterxml.jackson.core.JsonEncoding; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.util.TokenBuffer; -import com.tdunning.math.stats.Centroid; -import com.wavefront.common.MetricsToTimeseries; -import com.wavefront.common.Pair; -import com.wavefront.common.TaggedMetricName; -import com.yammer.metrics.core.Clock; -import com.yammer.metrics.core.Counter; -import com.yammer.metrics.core.Gauge; -import com.yammer.metrics.core.Histogram; -import com.yammer.metrics.core.Metered; -import com.yammer.metrics.core.Metric; -import com.yammer.metrics.core.MetricName; -import com.yammer.metrics.core.MetricProcessor; -import com.yammer.metrics.core.MetricsRegistry; -import com.yammer.metrics.core.SafeVirtualMachineMetrics; -import com.yammer.metrics.core.Sampling; -import com.yammer.metrics.core.Summarizable; -import com.yammer.metrics.core.Timer; -import com.yammer.metrics.core.VirtualMachineMetrics; -import com.yammer.metrics.core.WavefrontHistogram; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.Validate; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.MissingResourceException; -import java.util.ResourceBundle; -import java.util.SortedMap; -import java.util.function.Supplier; - -import javax.annotation.Nullable; - -import static com.wavefront.common.MetricsToTimeseries.sanitize; - -/** - * Generator of metrics as a JSON node and outputting it to an output stream or returning a json node. - * - * @author Clement Pang (clement@wavefront.com) - */ -public abstract class JsonMetricsGenerator { - - private static final JsonFactory factory = new JsonFactory(); - - private static final Clock clock = Clock.defaultClock(); - private static final VirtualMachineMetrics vm = SafeVirtualMachineMetrics.getInstance(); - - public static void generateJsonMetrics(OutputStream outputStream, MetricsRegistry registry, boolean includeVMMetrics, - boolean includeBuildMetrics, boolean clearMetrics, MetricTranslator metricTranslator) throws IOException { - JsonGenerator json = factory.createGenerator(outputStream, JsonEncoding.UTF8); - writeJson(json, registry, includeVMMetrics, includeBuildMetrics, clearMetrics, null, metricTranslator); - } - - public static JsonNode generateJsonMetrics(MetricsRegistry registry, boolean includeVMMetrics, - boolean includeBuildMetrics, boolean clearMetrics) throws IOException { - return generateJsonMetrics(registry, includeVMMetrics, includeBuildMetrics, clearMetrics, null, null); - } - - public static JsonNode generateJsonMetrics(MetricsRegistry registry, boolean includeVMMetrics, - boolean includeBuildMetrics, boolean clearMetrics, - @Nullable Map pointTags, - @Nullable MetricTranslator metricTranslator) throws IOException { - TokenBuffer t = new TokenBuffer(new ObjectMapper(), false); - writeJson(t, registry, includeVMMetrics, includeBuildMetrics, clearMetrics, pointTags, metricTranslator); - JsonParser parser = t.asParser(); - return parser.readValueAsTree(); - } - - static final class Context { - final boolean showFullSamples; - final JsonGenerator json; - - Context(JsonGenerator json, boolean showFullSamples) { - this.json = json; - this.showFullSamples = showFullSamples; - } - } - - public static void writeJson(JsonGenerator json, MetricsRegistry registry, boolean includeVMMetrics, - boolean includeBuildMetrics, boolean clearMetrics) throws IOException { - writeJson(json, registry, includeVMMetrics, includeBuildMetrics, clearMetrics, null, null); - } - - public static void writeJson(JsonGenerator json, MetricsRegistry registry, boolean includeVMMetrics, - boolean includeBuildMetrics, boolean clearMetrics, - @Nullable Map pointTags, - @Nullable MetricTranslator metricTranslator) throws IOException { - json.writeStartObject(); - if (includeVMMetrics) { - writeVmMetrics(json, pointTags); - } - if (includeBuildMetrics) { - try { - writeBuildMetrics(ResourceBundle.getBundle("build"), json, pointTags); - } catch (MissingResourceException ignored) { - } - } - writeRegularMetrics(new Processor(clearMetrics), json, registry, false, pointTags, metricTranslator); - json.writeEndObject(); - json.close(); - } - - private static void writeBuildMetrics(ResourceBundle props, JsonGenerator json, - Map pointTags) throws IOException { - json.writeFieldName("build"); - if (pointTags != null) { - json.writeStartObject(); - writeTags(json, pointTags); - json.writeFieldName("value"); - } - json.writeStartObject(); - if (props.containsKey("build.version")) { - // attempt to make a version string - int version = extractVersion(props.getString("build.version")); - if (version != 0) { - json.writeNumberField("version", version); - } - json.writeStringField("version_raw", props.getString("build.version")); - } - if (props.containsKey("build.commit")) { - json.writeStringField("commit", props.getString("build.commit")); - } - if (props.containsKey("build.hostname")) { - json.writeStringField("build_host", props.getString("build.hostname")); - } - if (props.containsKey("maven.build.timestamp")) { - if (StringUtils.isNumeric(props.getString("maven.build.timestamp"))) { - json.writeNumberField("timestamp", Long.valueOf(props.getString("maven.build.timestamp"))); - } - json.writeStringField("timestamp_raw", props.getString("maven.build.timestamp")); - } - json.writeEndObject(); - if (pointTags != null) { - json.writeEndObject(); - } - } - - static int extractVersion(String versionStr) { - int version = 0; - String[] components = versionStr.split("\\."); - for (int i = 0; i < Math.min(3, components.length); i++) { - String component = components[i]; - if (StringUtils.isNotBlank(component) && StringUtils.isNumeric(component)) { - version *= 1000; // we'll assume this will fit. 3.123.0 will become 3123000. - version += Integer.valueOf(component); - } else { - version = 0; // not actually a convertable name (probably something with SNAPSHOT). - break; - } - } - if (components.length == 2) { - version *= 1000; - } else if (components.length == 1) { - version *= 1000000; // make sure 3 outputs 3000000 - } - return version; - } - - private static void mergeMapIntoJson(JsonGenerator jsonGenerator, Map metrics) throws IOException { - for (Map.Entry entry : metrics.entrySet()) { - jsonGenerator.writeNumberField(entry.getKey(), entry.getValue()); - } - } - - private static void writeVmMetrics(JsonGenerator json, @Nullable Map pointTags) throws IOException { - json.writeFieldName("jvm"); // jvm - if (pointTags != null) { - json.writeStartObject(); - writeTags(json, pointTags); - json.writeFieldName("value"); - } - json.writeStartObject(); - { - json.writeFieldName("vm"); // jvm.vm - json.writeStartObject(); - { - json.writeStringField("name", vm.name()); - json.writeStringField("version", vm.version()); - } - json.writeEndObject(); - - json.writeFieldName("memory"); // jvm.memory - json.writeStartObject(); - { - mergeMapIntoJson(json, MetricsToTimeseries.memoryMetrics(vm)); - json.writeFieldName("memory_pool_usages"); // jvm.memory.memory_pool_usages - json.writeStartObject(); - { - mergeMapIntoJson(json, MetricsToTimeseries.memoryPoolsMetrics(vm)); - } - json.writeEndObject(); - } - json.writeEndObject(); - - final Map bufferPoolStats = vm.getBufferPoolStats(); - if (!bufferPoolStats.isEmpty()) { - json.writeFieldName("buffers"); // jvm.buffers - json.writeStartObject(); - { - json.writeFieldName("direct"); // jvm.buffers.direct - json.writeStartObject(); - { - mergeMapIntoJson(json, MetricsToTimeseries.buffersMetrics(bufferPoolStats.get("direct"))); - } - json.writeEndObject(); - - json.writeFieldName("mapped"); // jvm.buffers.mapped - json.writeStartObject(); - { - mergeMapIntoJson(json, MetricsToTimeseries.buffersMetrics(bufferPoolStats.get("mapped"))); - } - json.writeEndObject(); - } - json.writeEndObject(); - } - - mergeMapIntoJson(json, MetricsToTimeseries.vmMetrics(vm)); // jvm. - json.writeNumberField("current_time", clock.time()); - - json.writeFieldName("thread-states"); // jvm.thread-states - json.writeStartObject(); - { - mergeMapIntoJson(json, MetricsToTimeseries.threadStateMetrics(vm)); - } - json.writeEndObject(); - - json.writeFieldName("garbage-collectors"); // jvm.garbage-collectors - json.writeStartObject(); - { - for (Map.Entry entry : vm.garbageCollectors() - .entrySet()) { - json.writeFieldName(entry.getKey()); // jvm.garbage-collectors. - json.writeStartObject(); - { - mergeMapIntoJson(json, MetricsToTimeseries.gcMetrics(entry.getValue())); - } - json.writeEndObject(); - } - } - json.writeEndObject(); - } - json.writeEndObject(); - if (pointTags != null) { - json.writeEndObject(); - } - } - - private static void writeTags(JsonGenerator json, Map pointTags) throws IOException { - Validate.notNull(pointTags, "pointTags argument can't be null!"); - json.writeFieldName("tags"); - json.writeStartObject(); - for (Map.Entry tagEntry : pointTags.entrySet()) { - json.writeStringField(tagEntry.getKey(), tagEntry.getValue()); - } - json.writeEndObject(); - } - - public static void writeRegularMetrics(Processor processor, JsonGenerator json, MetricsRegistry registry, - boolean showFullSamples) throws IOException { - writeRegularMetrics(processor, json, registry, showFullSamples, null); - } - - public static void writeRegularMetrics(Processor processor, JsonGenerator json, - MetricsRegistry registry, boolean showFullSamples, - @Nullable Map pointTags) throws IOException { - writeRegularMetrics(processor, json, registry, showFullSamples, pointTags, null); - } - - public static void writeRegularMetrics(Processor processor, JsonGenerator json, - MetricsRegistry registry, boolean showFullSamples, - @Nullable Map pointTags, - @Nullable MetricTranslator metricTranslator) throws IOException { - for (Map.Entry> entry : registry.groupedMetrics().entrySet()) { - for (Map.Entry subEntry : entry.getValue().entrySet()) { - MetricName key = subEntry.getKey(); - Metric value = subEntry.getValue(); - if (metricTranslator != null) { - Pair pair = metricTranslator.apply(Pair.of(key, value)); - if (pair == null) continue; - key = pair._1; - value = pair._2; - } - boolean closeObjectRequired = false; - if (key instanceof TaggedMetricName || pointTags != null) { - closeObjectRequired = true; - // write the hashcode since we need to support metrics with the same name but with different tags. - // the server will remove the suffix. - json.writeFieldName(sanitize(key) + "$" + subEntry.hashCode()); - // write out the tags separately - // instead of metricName: {...} - // we write - // metricName_hashCode: { - // tags: { - // tagK: tagV,... - // }, - // value: {...} - // } - // - json.writeStartObject(); - json.writeFieldName("tags"); - json.writeStartObject(); - Map tags = new HashMap<>(); - if (pointTags != null) { - tags.putAll(pointTags); - } - if (key instanceof TaggedMetricName) { - tags.putAll(((TaggedMetricName) key).getTags()); - } - for (Map.Entry tagEntry : tags.entrySet()) { - json.writeStringField(tagEntry.getKey(), tagEntry.getValue()); - } - json.writeEndObject(); - json.writeFieldName("value"); - } else { - json.writeFieldName(sanitize(key)); - } - try { - value.processWith(processor, key, new Context(json, showFullSamples)); - } catch (Exception e) { - e.printStackTrace(); - } - // need to close the object as well. - if (closeObjectRequired) { - json.writeEndObject(); - } - } - } - } - - static final class Processor implements MetricProcessor { - - private final boolean clear; - - public Processor(boolean clear) { - this.clear = clear; - } - - private void internalProcessYammerHistogram(Histogram histogram, Context context) throws Exception { - final JsonGenerator json = context.json; - json.writeStartObject(); - { - json.writeNumberField("count", histogram.count()); - writeSummarizable(histogram, json); - writeSampling(histogram, json); - if (context.showFullSamples) { - json.writeObjectField("values", histogram.getSnapshot().getValues()); - } - if (clear) histogram.clear(); - } - json.writeEndObject(); - } - - private void internalProcessWavefrontHistogram(WavefrontHistogram hist, Context context) throws Exception { - final JsonGenerator json = context.json; - json.writeStartObject(); - json.writeArrayFieldStart("bins"); - for (WavefrontHistogram.MinuteBin bin : hist.bins(clear)) { - - final Collection centroids = bin.getDist().centroids(); - - json.writeStartObject(); - // Count - json.writeNumberField("count", bin.getDist().size()); - // Start - json.writeNumberField("startMillis", bin.getMinMillis()); - // Duration - json.writeNumberField("durationMillis", 60 * 1000); - // Means - json.writeArrayFieldStart("means"); - for (Centroid c : centroids) { - json.writeNumber(c.mean()); - } - json.writeEndArray(); - // Counts - json.writeArrayFieldStart("counts"); - for (Centroid c : centroids) { - json.writeNumber(c.count()); - } - json.writeEndArray(); - - json.writeEndObject(); - } - json.writeEndArray(); - json.writeEndObject(); - } - - @Override - public void processHistogram(MetricName name, Histogram histogram, Context context) throws Exception { - if (histogram instanceof WavefrontHistogram) { - internalProcessWavefrontHistogram((WavefrontHistogram) histogram, context); - } else /*Treat as standard yammer histogram */ { - internalProcessYammerHistogram(histogram, context); - } - } - - @Override - public void processCounter(MetricName name, Counter counter, Context context) throws Exception { - final JsonGenerator json = context.json; - json.writeNumber(counter.count()); - } - - @Override - public void processGauge(MetricName name, Gauge gauge, Context context) throws Exception { - final JsonGenerator json = context.json; - Object gaugeValue = evaluateGauge(gauge); - if (gaugeValue != null) { - json.writeObject(gaugeValue); - } else { - json.writeNull(); - } - } - - @Override - public void processMeter(MetricName name, Metered meter, Context context) throws Exception { - final JsonGenerator json = context.json; - json.writeStartObject(); - { - writeMeteredFields(meter, json); - } - json.writeEndObject(); - } - - @Override - public void processTimer(MetricName name, Timer timer, Context context) throws Exception { - final JsonGenerator json = context.json; - json.writeStartObject(); - { - json.writeFieldName("duration"); - json.writeStartObject(); - { - json.writeStringField("unit", timer.durationUnit().toString().toLowerCase()); - writeSummarizable(timer, json); - writeSampling(timer, json); - if (context.showFullSamples) { - json.writeObjectField("values", timer.getSnapshot().getValues()); - } - } - json.writeEndObject(); - - json.writeFieldName("rate"); - json.writeStartObject(); - { - writeMeteredFields(timer, json); - } - json.writeEndObject(); - } - json.writeEndObject(); - if (clear) timer.clear(); - } - } - - private static Object evaluateGauge(Gauge gauge) { - try { - return gauge.value(); - } catch (RuntimeException e) { - return "error reading gauge: " + e.getMessage(); - } - } - - private static void writeSummarizable(Summarizable metric, JsonGenerator json) throws IOException { - for (Map.Entry entry : MetricsToTimeseries.explodeSummarizable(metric).entrySet()) { - json.writeNumberField(entry.getKey(), entry.getValue()); - } - } - - private static void writeSampling(Sampling metric, JsonGenerator json) throws IOException { - for (Map.Entry entry : MetricsToTimeseries.explodeSampling(metric).entrySet()) { - json.writeNumberField(entry.getKey(), entry.getValue()); - } - } - - private static void writeMeteredFields(Metered metered, JsonGenerator json) throws IOException { - for (Map.Entry entry : MetricsToTimeseries.explodeMetered(metered).entrySet()) { - json.writeNumberField(entry.getKey(), entry.getValue()); - } - } -} diff --git a/java-lib/src/main/java/com/wavefront/metrics/JsonMetricsParser.java b/java-lib/src/main/java/com/wavefront/metrics/JsonMetricsParser.java deleted file mode 100644 index ec64f1743..000000000 --- a/java-lib/src/main/java/com/wavefront/metrics/JsonMetricsParser.java +++ /dev/null @@ -1,192 +0,0 @@ -package com.wavefront.metrics; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import wavefront.report.Histogram; -import wavefront.report.HistogramType; -import wavefront.report.ReportPoint; - -import static com.google.common.collect.Lists.newArrayList; - -/** - * Helper methods to turn json nodes into actual Wavefront report points - * - * @author Andrew Kao (andrew@wavefront.com) - */ -public class JsonMetricsParser { - private static final Pattern SIMPLE_NAMES = Pattern.compile("[^a-zA-Z0-9_.-]"); - private static final Pattern TAGGED_METRIC = Pattern.compile("(.*)\\$[0-9-]+"); - - public static void report(String table, String path, JsonNode node, List points, String host, - long timestamp) { - report(table, path, node, points, host, timestamp, Collections.emptyMap()); - } - - public static void report(String table, String path, JsonNode node, List points, String host, - long timestamp, Map tags) { - List> fields = newArrayList(node.fields()); - // if the node only has the follow nodes: "value" and "tags", parse the node as a value with tags. - if (fields.size() == 2) { - JsonNode valueNode = null; - JsonNode tagsNode = null; - for (Map.Entry next : fields) { - if (next.getKey().equals("value")) { - valueNode = next.getValue(); - } else if (next.getKey().equals("tags")) { - tagsNode = next.getValue(); - } - } - if (valueNode != null && tagsNode != null) { - Map combinedTags = Maps.newHashMap(tags); - combinedTags.putAll(makeTags(tagsNode)); - processValueNode(valueNode, table, path, host, timestamp, points, combinedTags); - return; - } - } - for (Map.Entry next : fields) { - String key; - Matcher taggedMetricMatcher = TAGGED_METRIC.matcher(next.getKey()); - if (taggedMetricMatcher.matches()) { - key = SIMPLE_NAMES.matcher(taggedMetricMatcher.group(1)).replaceAll("_"); - } else { - key = SIMPLE_NAMES.matcher(next.getKey()).replaceAll("_"); - } - String metric = path == null ? key : path + "." + key; - JsonNode value = next.getValue(); - processValueNode(value, table, metric, host, timestamp, points, tags); - } - } - - public static void processValueNode(JsonNode value, String table, String metric, String host, long timestamp, - List points, Map tags) { - if (value.isNumber()) { - if (value.isLong()) { - points.add(makePoint(table, metric, host, value.longValue(), timestamp, tags)); - } else { - points.add(makePoint(table, metric, host, value.doubleValue(), timestamp, tags)); - } - } else if (value.isTextual()) { - points.add(makePoint(table, metric, host, value.textValue(), timestamp, tags)); - } else if (value.isObject()) { - if /*wavefront histogram*/ (value.has("bins")) { - Iterator binIt = ((ArrayNode) value.get("bins")).elements(); - while (binIt.hasNext()) { - JsonNode bin = binIt.next(); - List counts = newArrayList(); - bin.get("counts").elements().forEachRemaining(v -> counts.add(v.intValue())); - List means = newArrayList(); - bin.get("means").elements().forEachRemaining(v -> means.add(v.doubleValue())); - - points.add(makeHistogramPoint( - table, - metric, - host, - tags, - bin.get("startMillis").longValue(), - bin.get("durationMillis").intValue(), - means, - counts)); - } - - - } else { - report(table, metric, value, points, host, timestamp, tags); - } - } else if (value.isBoolean()) { - points.add(makePoint(table, metric, host, value.booleanValue() ? 1.0 : 0.0, timestamp, tags)); - } - } - - public static ReportPoint makeHistogramPoint( - String customer, - String metric, - String host, - Map annotations, - long startMillis, - int durationMillis, - List bins, - List counts) { - Histogram histogram = Histogram.newBuilder() - .setType(HistogramType.TDIGEST) - .setDuration(durationMillis) - .setCounts(counts) - .setBins(bins).build(); - - return makePoint(customer, metric, host, startMillis, annotations).setValue(histogram).build(); - } - - public static ReportPoint makePoint(String table, String metric, String host, String value, long timestamp) { - return makePoint(table, metric, host, value, timestamp, Collections.emptyMap()); - } - - public static ReportPoint makePoint(String table, String metric, String host, String value, long timestamp, - Map annotations) { - ReportPoint.Builder builder = makePoint(table, metric, host, timestamp, annotations); - return builder.setValue(value).build(); - } - - public static ReportPoint makePoint(String table, String metric, String host, long value, long timestamp) { - return makePoint(table, metric, host, value, timestamp, Collections.emptyMap()); - } - - public static ReportPoint makePoint(String table, String metric, String host, long value, long timestamp, - Map annotations) { - ReportPoint.Builder builder = makePoint(table, metric, host, timestamp, annotations); - return builder.setValue(value).build(); - } - - public static ReportPoint makePoint(String table, String metric, String host, double value, long timestamp) { - return makePoint(table, metric, host, value, timestamp, Collections.emptyMap()); - } - - public static ReportPoint makePoint(String table, String metric, String host, double value, long timestamp, - Map annotations) { - ReportPoint.Builder builder = makePoint(table, metric, host, timestamp, annotations); - return builder.setValue(value).build(); - } - - private static ReportPoint.Builder makePoint(String table, String metric, String host, long timestamp, - Map annotations) { - return ReportPoint.newBuilder() - .setAnnotations(annotations) - .setMetric(metric) - .setTable(table) - .setTimestamp(timestamp) - .setHost(host); - } - - public static Map makeTags(JsonNode tags) { - ImmutableMap.Builder builder = ImmutableMap.builder(); - if (tags.isObject()) { - Iterator> fields = tags.fields(); - while (fields.hasNext()) { - Map.Entry next = fields.next(); - String key = SIMPLE_NAMES.matcher(next.getKey()).replaceAll("_"); - JsonNode value = next.getValue(); - if (value.isBoolean()) { - builder.put(key, String.valueOf(value.booleanValue())); - } else if (value.isNumber()) { - if (value.isLong()) { - builder.put(key, String.valueOf(value.asLong())); - } else { - builder.put(key, String.valueOf(value.asDouble())); - } - } else if (value.isTextual()) { - builder.put(key, value.asText()); - } - } - } - return builder.build(); - } -} diff --git a/java-lib/src/main/java/com/wavefront/metrics/JsonMetricsReporter.java b/java-lib/src/main/java/com/wavefront/metrics/JsonMetricsReporter.java deleted file mode 100644 index a945aa9b2..000000000 --- a/java-lib/src/main/java/com/wavefront/metrics/JsonMetricsReporter.java +++ /dev/null @@ -1,185 +0,0 @@ -package com.wavefront.metrics; - - -import com.wavefront.common.TaggedMetricName; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Counter; -import com.yammer.metrics.core.Gauge; -import com.yammer.metrics.core.MetricName; -import com.yammer.metrics.core.MetricsRegistry; -import com.yammer.metrics.core.Timer; -import com.yammer.metrics.core.TimerContext; -import com.yammer.metrics.reporting.AbstractPollingReporter; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.URI; -import java.net.URL; -import java.net.UnknownHostException; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.annotation.Nullable; -import javax.ws.rs.core.UriBuilder; - -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import okio.BufferedSink; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; - -/** - * Adapted from MetricsServlet. - * - * @author Sam Pullara (sam@wavefront.com) - * @author Clement Pang (clement@wavefront.com) - * @author Andrew Kao (andrew@wavefront.com) - */ -public class JsonMetricsReporter extends AbstractPollingReporter { - - private static final Logger logger = Logger.getLogger(JsonMetricsReporter.class.getCanonicalName()); - private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); - - private final boolean includeVMMetrics; - private final String table; - private final String sunnylabsHost; - private final Integer sunnylabsPort; // Null means use default URI port, probably 80 or 443. - private final String host; - private final Map tags; - private final Counter errors; - private final boolean clearMetrics, https; - private final MetricTranslator metricTranslator; - - private final OkHttpClient client = new OkHttpClient.Builder() - .connectTimeout(10, TimeUnit.SECONDS) - .writeTimeout(10, TimeUnit.SECONDS) - .readTimeout(30, TimeUnit.SECONDS) - .build(); - - private Timer latency; - private Counter reports; - - /** - * Track and report uptime of services. - */ - private final long START_TIME = System.currentTimeMillis(); - private final Gauge serverUptime = Metrics.newGauge(new TaggedMetricName("service", "uptime"), - new Gauge() { - @Override - public Long value() { - return System.currentTimeMillis() - START_TIME; - } - }); - - public JsonMetricsReporter(MetricsRegistry registry, String table, - String sunnylabsHost, Map tags, boolean clearMetrics) - throws UnknownHostException { - this(registry, true, table, sunnylabsHost, tags, clearMetrics); - } - - public JsonMetricsReporter(MetricsRegistry registry, boolean includeVMMetrics, - String table, String sunnylabsHost, Map tags, boolean clearMetrics) - throws UnknownHostException { - this(registry, includeVMMetrics, table, sunnylabsHost, tags, clearMetrics, true, null); - } - - public JsonMetricsReporter(MetricsRegistry registry, boolean includeVMMetrics, - String table, String sunnylabsHost, Map tags, boolean clearMetrics, - boolean https, MetricTranslator metricTranslator) - throws UnknownHostException { - super(registry, "json-metrics-reporter"); - this.metricTranslator = metricTranslator; - this.includeVMMetrics = includeVMMetrics; - this.tags = tags; - this.table = table; - - if (sunnylabsHost.contains(":")) { - int idx = sunnylabsHost.indexOf(":"); - String host = sunnylabsHost.substring(0, idx); - String strPort = sunnylabsHost.substring(idx + 1); - Integer port = null; - this.sunnylabsHost = host; - try { - port = Integer.parseInt(strPort); - } catch (NumberFormatException e) { - logger.log(Level.SEVERE, "Cannot infer port for JSON reporting", e); - } - this.sunnylabsPort = port; - } else { - this.sunnylabsHost = sunnylabsHost; - this.sunnylabsPort = null; - } - - this.clearMetrics = clearMetrics; - this.host = InetAddress.getLocalHost().getHostName(); - this.https = https; - if (!this.https) { - logger.severe("==================================================================="); - logger.severe("HTTPS is off for reporting! This should never be set in production!"); - logger.severe("==================================================================="); - } - - latency = Metrics.newTimer(new MetricName("jsonreporter", "jsonreporter", "latency"), MILLISECONDS, SECONDS); - reports = Metrics.newCounter(new MetricName("jsonreporter", "jsonreporter", "reports")); - errors = Metrics.newCounter(new MetricName("jsonreporter", "jsonreporter", "errors")); - } - - @Override - public void run() { - try { - reportMetrics(); - } catch (Throwable t) { - logger.log(Level.SEVERE, "Uncaught exception in reportMetrics loop", t); - } - } - - public void reportMetrics() { - TimerContext time = latency.time(); - try { - UriBuilder builder = UriBuilder.fromUri(new URI( - https ? "https" : "http", sunnylabsHost, "/report/metrics", null)); - if (sunnylabsPort != null) { - builder.port(sunnylabsPort); - } - builder.queryParam("h", host); - builder.queryParam("t", table); - for (Map.Entry tag : tags.entrySet()) { - builder.queryParam(tag.getKey(), tag.getValue()); - } - URL http = builder.build().toURL(); - logger.info("Reporting metrics (JSON) to: " + http); - Request request = new Request.Builder(). - url(http). - post(new RequestBody() { - @Nullable - @Override - public okhttp3.MediaType contentType() { - return JSON; - } - - @Override - public void writeTo(BufferedSink bufferedSink) throws IOException { - JsonMetricsGenerator.generateJsonMetrics(bufferedSink.outputStream(), - getMetricsRegistry(), includeVMMetrics, true, - clearMetrics, metricTranslator); - } - }). - build(); - try (final Response response = client.newCall(request).execute()) { - logger.info("Metrics (JSON) reported: " + response.code()); - } - reports.inc(); - } catch (Throwable e) { - logger.log(Level.WARNING, "Failed to report metrics (JSON)", e); - errors.inc(); - } finally { - time.stop(); - } - } -} diff --git a/java-lib/src/main/java/com/wavefront/metrics/MetricTranslator.java b/java-lib/src/main/java/com/wavefront/metrics/MetricTranslator.java deleted file mode 100644 index c809b3551..000000000 --- a/java-lib/src/main/java/com/wavefront/metrics/MetricTranslator.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.wavefront.metrics; - -import com.wavefront.common.Pair; -import com.yammer.metrics.core.Metric; -import com.yammer.metrics.core.MetricName; - -import java.util.function.Function; - -/** - * @author Mori Bellamy (mori@wavefront.com) - */ -public interface MetricTranslator extends Function, Pair> { -} diff --git a/java-lib/src/main/java/com/wavefront/metrics/ReconnectingSocket.java b/java-lib/src/main/java/com/wavefront/metrics/ReconnectingSocket.java deleted file mode 100644 index 8c9ad6492..000000000 --- a/java-lib/src/main/java/com/wavefront/metrics/ReconnectingSocket.java +++ /dev/null @@ -1,187 +0,0 @@ -package com.wavefront.metrics; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Throwables; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.net.Socket; -import java.net.SocketException; -import java.net.UnknownHostException; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.annotation.Nullable; -import javax.net.SocketFactory; - -/** - * Creates a TCP client suitable for the WF proxy. That is: a client which is long-lived and semantically one-way. - * This client tries persistently to reconnect to the given host and port if a connection is ever broken. If the server - * (in practice, the WF proxy) sends a TCP FIN or TCP RST, we will treat it as a "broken connection" and just try - * to connect again on the next call to write(). This means each ReconnectingSocket has a polling thread for the server - * to listen for connection resets. - * - * @author Mori Bellamy (mori@wavefront.com) - */ -public class ReconnectingSocket { - protected static final Logger logger = Logger.getLogger(ReconnectingSocket.class.getCanonicalName()); - - private static final int - SERVER_READ_TIMEOUT_MILLIS = 2000, - SERVER_POLL_INTERVAL_MILLIS = 4000; - - private final String host; - private final int port; - private final long connectionTimeToLiveMillis; - private final Supplier timeSupplier; - private final SocketFactory socketFactory; - private volatile boolean serverTerminated; - private volatile long lastConnectionTimeMillis; - private final Timer pollingTimer; - private AtomicReference underlyingSocket; - private AtomicReference socketOutputStream; - - /** - * @throws IOException When we cannot open the remote socket. - */ - public ReconnectingSocket(String host, int port, SocketFactory socketFactory) throws IOException { - this(host, port, socketFactory, null, null); - } - - /** - * @param host Hostname to connect to - * @param port Port to connect to - * @param socketFactory SocketFactory used to instantiate new sockets - * @param connectionTimeToLiveMillis Connection TTL, with expiration checked after each flush. When null, - * TTL is not enforced. - * @param timeSupplier Get current timestamp in millis - * @throws IOException When we cannot open the remote socket. - */ - public ReconnectingSocket(String host, int port, SocketFactory socketFactory, - @Nullable Long connectionTimeToLiveMillis, @Nullable Supplier timeSupplier) - throws IOException { - this.host = host; - this.port = port; - this.serverTerminated = false; - this.socketFactory = socketFactory; - this.connectionTimeToLiveMillis = connectionTimeToLiveMillis == null ? Long.MAX_VALUE : connectionTimeToLiveMillis; - this.timeSupplier = timeSupplier == null ? System::currentTimeMillis : timeSupplier; - - this.underlyingSocket = new AtomicReference<>(socketFactory.createSocket(host, port)); - this.underlyingSocket.get().setSoTimeout(SERVER_READ_TIMEOUT_MILLIS); - this.socketOutputStream = new AtomicReference<>(new BufferedOutputStream(underlyingSocket.get().getOutputStream())); - this.lastConnectionTimeMillis = this.timeSupplier.get(); - - this.pollingTimer = new Timer(); - - pollingTimer.scheduleAtFixedRate(new TimerTask() { - @Override - public void run() { - maybeReconnect(); - } - }, SERVER_POLL_INTERVAL_MILLIS, SERVER_POLL_INTERVAL_MILLIS); - - } - - @VisibleForTesting - void maybeReconnect() { - try { - byte[] message = new byte[1000]; - int bytesRead; - try { - bytesRead = underlyingSocket.get().getInputStream().read(message); - } catch (IOException e) { - // Read timeout, just try again later. Important to set SO_TIMEOUT elsewhere. - return; - } - if (bytesRead == -1) { - serverTerminated = true; - } - } catch (Exception e) { - logger.log(Level.SEVERE, "Cannot poll server for TCP FIN."); - } - } - - public ReconnectingSocket(String host, int port) throws IOException { - this(host, port, SocketFactory.getDefault()); - } - - /** - * Closes the outputStream best-effort. Tries to re-instantiate the outputStream. - * - * @throws IOException If we cannot close a outputStream we had opened before. - * @throws UnknownHostException When {@link #host} and {@link #port} are bad. - */ - private synchronized void resetSocket() throws IOException { - try { - BufferedOutputStream old = socketOutputStream.get(); - if (old != null) old.close(); - } catch (SocketException e) { - logger.log(Level.INFO, "Could not flush to socket.", e); - } finally { - serverTerminated = false; - try { - underlyingSocket.getAndSet(socketFactory.createSocket(host, port)).close(); - } catch (SocketException e) { - logger.log(Level.WARNING, "Could not close old socket.", e); - } - underlyingSocket.get().setSoTimeout(SERVER_READ_TIMEOUT_MILLIS); - socketOutputStream.set(new BufferedOutputStream(underlyingSocket.get().getOutputStream())); - lastConnectionTimeMillis = timeSupplier.get(); - logger.log(Level.INFO, String.format("Successfully reset connection to %s:%d", host, port)); - } - } - - /** - * Try to send the given message. On failure, reset and try again. If _that_ fails, - * just rethrow the exception. - * - * @throws Exception when a single retry is not enough to have a successful write to the remote host. - */ - public void write(String message) throws Exception { - try { - if (serverTerminated) { - throw new Exception("Remote server terminated."); // Handled below. - } - // Might be NPE due to previously failed call to resetSocket. - socketOutputStream.get().write(message.getBytes()); - } catch (Exception e) { - try { - logger.log(Level.WARNING, "Attempting to reset socket connection.", e); - resetSocket(); - socketOutputStream.get().write(message.getBytes()); - } catch (Exception e2) { - throw Throwables.propagate(e2); - } - } - } - - /** - * Flushes the outputStream best-effort. If that fails, we reset the connection. - */ - public synchronized void flush() throws IOException { - try { - socketOutputStream.get().flush(); - } catch (Exception e) { - logger.log(Level.WARNING, "Attempting to reset socket connection.", e); - resetSocket(); - } - if (timeSupplier.get() - lastConnectionTimeMillis > connectionTimeToLiveMillis) { - logger.info("Connection TTL expired, reconnecting"); - resetSocket(); - } - } - - public void close() throws IOException { - try { - flush(); - } finally { - pollingTimer.cancel(); - socketOutputStream.get().close(); - } - } -} diff --git a/java-lib/src/main/java/com/yammer/metrics/core/DeltaCounter.java b/java-lib/src/main/java/com/yammer/metrics/core/DeltaCounter.java deleted file mode 100644 index 010a36396..000000000 --- a/java-lib/src/main/java/com/yammer/metrics/core/DeltaCounter.java +++ /dev/null @@ -1,123 +0,0 @@ -package com.yammer.metrics.core; - -import com.google.common.annotations.VisibleForTesting; -import com.wavefront.common.MetricConstants; -import com.yammer.metrics.Metrics; - -/** - * A counter for Wavefront delta metrics. - * - * Differs from a counter in that it is reset every time the value is reported. - * - * (This is similar to how {@link com.yammer.metrics.core.WavefrontHistogram} is implemented) - * - * @author Suranjan Pramanik (suranjan@wavefront.com) - */ -public class DeltaCounter extends Counter { - - private DeltaCounter() { - // nothing to do. keeping it private so that it can't be instantiated directly - } - - /** - * A static factory method to create an instance of Delta Counter. - * - * @param registry The MetricRegistry to use - * @param metricName The MetricName to use - * @return An instance of DeltaCounter - */ - @VisibleForTesting - public static synchronized DeltaCounter get(MetricsRegistry registry, MetricName metricName) { - if (registry == null || metricName == null || metricName.getName().isEmpty()) { - throw new IllegalArgumentException("Invalid arguments"); - } - - DeltaCounter counter = new DeltaCounter(); - MetricName newMetricName = getDeltaCounterMetricName(metricName); - registry.getOrAdd(newMetricName, counter); - return counter; - } - - /** - * A static factory method to create an instance of DeltaCounter. It uses the default - * MetricsRegistry. - * - * @param metricName The MetricName to use - * @return An instance of DeltaCounter - */ - public static DeltaCounter get(MetricName metricName) { - return get(Metrics.defaultRegistry(), metricName); - } - - /** - * This method returns the current count of the DeltaCounter and resets the counter. - * - * @param counter The DeltaCounter whose value is requested - * @return The current count of the DeltaCounter - */ - public static long processDeltaCounter(DeltaCounter counter) { - long count = counter.count(); - counter.dec(count); - return count; - } - - /** - * This method transforms the MetricName into a new MetricName that represents a DeltaCounter. - * The transformation includes prepending a "\u2206" character to the name. - * - * @param metricName The MetricName which needs to be transformed - * @return The new MetricName representing a DeltaCounter - */ - public static MetricName getDeltaCounterMetricName(MetricName metricName) { - if (isDelta(metricName.getName())) { - return metricName; - } else { - String name = getDeltaCounterName(metricName.getName()); - return new MetricName(metricName.getGroup(), metricName.getType(), name, - metricName.getScope()); - } - } - - /** - * A helper function to transform a counter name to a DeltaCounter name. The transformation - * includes prepending a "\u2206" character to the name. - * - * @param name The name which needs to be transformed - * @return The new name representing a DeltaCounter - */ - public static String getDeltaCounterName(String name) { - if (!isDelta(name)) { - return MetricConstants.DELTA_PREFIX + name; - } else { - return name; - } - } - - /** - * This method checks whether the name is a valid DeltaCounter name. - * - * @param name The name which needs to be checked - * @return True if the name is a valid DeltaCounter name, otherwise returns false - */ - public static boolean isDelta(String name) { - return name.startsWith(MetricConstants.DELTA_PREFIX) || - name.startsWith(MetricConstants.DELTA_PREFIX_2); - } - - /** - * A helper function to transform the name from a DeltaCounter name to a new name by removing - * the "\u2206" prefix. If the name is not a DeltaCounter name then the input name is returned. - * - * @param name The name which needs to be transformed - * @return The transformed name - */ - public static String getNameWithoutDeltaPrefix(String name) { - if (name.startsWith(MetricConstants.DELTA_PREFIX)) { - return name.substring(MetricConstants.DELTA_PREFIX.length()); - } else if (name.startsWith(MetricConstants.DELTA_PREFIX_2)) { - return name.substring(MetricConstants.DELTA_PREFIX_2.length()); - } else { - return name; - } - } -} diff --git a/java-lib/src/main/java/com/yammer/metrics/core/SafeVirtualMachineMetrics.java b/java-lib/src/main/java/com/yammer/metrics/core/SafeVirtualMachineMetrics.java deleted file mode 100644 index b21f64e00..000000000 --- a/java-lib/src/main/java/com/yammer/metrics/core/SafeVirtualMachineMetrics.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.yammer.metrics.core; - -import com.sun.management.UnixOperatingSystemMXBean; - -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.MemoryPoolMXBean; -import java.lang.management.OperatingSystemMXBean; -import java.lang.management.RuntimeMXBean; -import java.lang.management.ThreadMXBean; -import java.util.List; - -import javax.management.MBeanServer; - -/** - * Java 9 compatible implementation of {@link VirtualMachineMetrics} that doesn't use reflection - * and is not susceptible to a InaccessibleObjectException in fileDescriptorUsage()) - * - * @author Vasily Vorontsov (vasily@wavefront.com) - */ -public class SafeVirtualMachineMetrics extends VirtualMachineMetrics { - private static final VirtualMachineMetrics INSTANCE = new SafeVirtualMachineMetrics( - ManagementFactory.getMemoryMXBean(), ManagementFactory.getMemoryPoolMXBeans(), - ManagementFactory.getOperatingSystemMXBean(), ManagementFactory.getThreadMXBean(), - ManagementFactory.getGarbageCollectorMXBeans(), ManagementFactory.getRuntimeMXBean(), - ManagementFactory.getPlatformMBeanServer()); - private final OperatingSystemMXBean os; - - /** - * The default instance of {@link SafeVirtualMachineMetrics}. - * - * @return the default {@link SafeVirtualMachineMetrics instance} - */ - public static VirtualMachineMetrics getInstance() { - return INSTANCE; - } - - private SafeVirtualMachineMetrics(MemoryMXBean memory, List memoryPools, OperatingSystemMXBean os, - ThreadMXBean threads, List garbageCollectors, - RuntimeMXBean runtime, MBeanServer mBeanServer) { - super(memory, memoryPools, os, threads, garbageCollectors, runtime, mBeanServer); - this.os = os; - } - - /** - * Returns the percentage of available file descriptors which are currently in use. - * - * @return the percentage of available file descriptors which are currently in use, or {@code - * NaN} if the running JVM does not have access to this information - */ - @Override - public double fileDescriptorUsage() { - if (!(this.os instanceof UnixOperatingSystemMXBean)) { - return Double.NaN; - } - Long openFds = ((UnixOperatingSystemMXBean)os).getOpenFileDescriptorCount(); - Long maxFds = ((UnixOperatingSystemMXBean)os).getMaxFileDescriptorCount(); - return openFds.doubleValue() / maxFds.doubleValue(); - } - -} diff --git a/java-lib/src/main/java/com/yammer/metrics/core/WavefrontHistogram.java b/java-lib/src/main/java/com/yammer/metrics/core/WavefrontHistogram.java deleted file mode 100644 index b8c8314b7..000000000 --- a/java-lib/src/main/java/com/yammer/metrics/core/WavefrontHistogram.java +++ /dev/null @@ -1,324 +0,0 @@ -package com.yammer.metrics.core; - -import com.google.common.annotations.VisibleForTesting; - -import com.tdunning.math.stats.AVLTreeDigest; -import com.tdunning.math.stats.Centroid; -import com.tdunning.math.stats.TDigest; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.stats.Sample; -import com.yammer.metrics.stats.Snapshot; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.function.Supplier; - -import static com.google.common.collect.Iterables.getFirst; -import static com.google.common.collect.Iterables.getLast; -import static java.lang.Double.MAX_VALUE; -import static java.lang.Double.MIN_VALUE; -import static java.lang.Double.NaN; - -/** - * Wavefront implementation of {@link Histogram}. - * - * @author Tim Schmidt (tim@wavefront.com). - */ -public class WavefrontHistogram extends Histogram implements Metric { - private final static int ACCURACY = 100; - private final static int MAX_BINS = 10; - private final Supplier millis; - - private final ConcurrentMap> perThreadHistogramBins = new ConcurrentHashMap<>(); - - private WavefrontHistogram(TDigestSample sample, Supplier millis) { - super(sample); - this.millis = millis; - } - - public static WavefrontHistogram get(MetricName metricName) { - return get(Metrics.defaultRegistry(), metricName); - } - - public static WavefrontHistogram get(MetricsRegistry registry, MetricName metricName) { - return get(registry, metricName, System::currentTimeMillis); - } - - @VisibleForTesting - public static WavefrontHistogram get(MetricsRegistry registry, - MetricName metricName, - Supplier clock) { - // Awkward construction trying to fit in with Yammer Histograms - TDigestSample sample = new TDigestSample(); - WavefrontHistogram tDigestHistogram = new WavefrontHistogram(sample, clock); - sample.set(tDigestHistogram); - return registry.getOrAdd(metricName, tDigestHistogram); - } - - /** - * Aggregates all the bins prior to the current minute - * This is because threads might be updating the current minute bin while the bins() method is invoked - * - * @param clear if set to true, will clear the older bins - * @return returns aggregated collection of all the bins prior to the current minute - */ - public List bins(boolean clear) { - List result = new ArrayList<>(); - final long cutoffMillis = minMillis(); - perThreadHistogramBins.values().stream().flatMap(List::stream). - filter(i -> i.getMinMillis() < cutoffMillis).forEach(result::add); - - if (clear) { - clearPriorCurrentMinuteBin(cutoffMillis); - } - - return result; - } - - private long minMillis() { - long currMillis; - if (millis == null) { - // happens because WavefrontHistogram.get() invokes the super() Histogram constructor - // which invokes clear() method which in turn invokes this method - currMillis = System.currentTimeMillis(); - } else { - currMillis = millis.get(); - } - return (currMillis / 60000L) * 60000L; - } - - @Override - public void update(int value) { - update((double) value); - } - - /** - * Helper to retrieve the current bin. Will be invoked per thread. - */ - private MinuteBin getCurrent() { - long key = Thread.currentThread().getId(); - LinkedList bins = perThreadHistogramBins.get(key); - if (bins == null) { - bins = new LinkedList<>(); - LinkedList existing = perThreadHistogramBins.putIfAbsent(key, bins); - if (existing != null) { - bins = existing; - } - } - - long currMinMillis = minMillis(); - - // bins with clear == true flag will drain (CONSUMER) the list, - // so synchronize the access to the respective 'bins' list - synchronized (bins) { - if (bins.isEmpty() || bins.getLast().minMillis != currMinMillis) { - bins.add(new MinuteBin(currMinMillis)); - if (bins.size() > MAX_BINS) { - bins.removeFirst(); - } - } - return bins.getLast(); - } - } - - /** - * Bulk-update this histogram with a set of centroids. - * - * @param means the centroid values - * @param counts the centroid weights/sample counts - */ - public void bulkUpdate(List means, List counts) { - if (means != null && counts != null) { - int n = Math.min(means.size(), counts.size()); - MinuteBin current = getCurrent(); - for (int i = 0; i < n; ++i) { - current.dist.add(means.get(i), counts.get(i)); - } - } - } - - public void update(double value) { - getCurrent().dist.add(value); - } - - @Override - public void update(long value) { - update((double) value); - } - - @Override - public double mean() { - Collection centroids = snapshot().centroids(); - return centroids.size() == 0 ? - Double.NaN : - centroids.stream().mapToDouble(c -> (c.count() * c.mean()) / centroids.size()).sum(); - } - - public double min() { - // This is a lie if the winning centroid's weight > 1 - return perThreadHistogramBins.values().stream().flatMap(List::stream).map(b -> b.dist.centroids()). - mapToDouble(cs -> getFirst(cs, new Centroid(MAX_VALUE)).mean()).min().orElse(NaN); - } - - public double max() { - //This is a lie if the winning centroid's weight > 1 - return perThreadHistogramBins.values().stream().flatMap(List::stream).map(b -> b.dist.centroids()). - mapToDouble(cs -> getLast(cs, new Centroid(MIN_VALUE)).mean()).max().orElse(NaN); - } - - @Override - public long count() { - return perThreadHistogramBins.values().stream().flatMap(List::stream).mapToLong(bin -> bin.dist.size()).sum(); - } - - @Override - public double sum() { - return Double.NaN; - } - - @Override - public double stdDev() { - return Double.NaN; - } - - /** - * Note - We override the behavior of the clear() method. - * In the super class, we would clear all the recorded values. - */ - @Override - public void clear() { - // More awkwardness - clearPriorCurrentMinuteBin(minMillis()); - } - - private void clearPriorCurrentMinuteBin(long cutoffMillis) { - if (perThreadHistogramBins == null) { - // will happen if WavefrontHistogram.super() constructor will be invoked - // before WavefrontHistogram object is fully instantiated, - // which will be invoke clear() method - return; - } - for (LinkedList bins : perThreadHistogramBins.values()) { - // getCurrent() method will add (PRODUCER) item to the bins list, so synchronize the access - synchronized (bins) { - bins.removeIf(minuteBin -> minuteBin.getMinMillis() < cutoffMillis); - } - } - } - - // TODO - how to ensure thread safety? do we care? - private TDigest snapshot() { - final TDigest snapshot = new AVLTreeDigest(ACCURACY); - perThreadHistogramBins.values().stream().flatMap(List::stream).forEach(bin -> snapshot.add(bin.dist)); - return snapshot; - } - - @Override - public Snapshot getSnapshot() { - final TDigest snapshot = snapshot(); - - return new Snapshot(new double[0]) { - @Override - public double get75thPercentile() { - return getValue(.75); - } - - @Override - public double get95thPercentile() { - return getValue(.95); - } - - @Override - public double get98thPercentile() { - return getValue(.98); - } - - @Override - public double get999thPercentile() { - return getValue(.999); - } - - @Override - public double get99thPercentile() { - return getValue(.99); - } - - @Override - public double getMedian() { - return getValue(.50); - } - - @Override - public double getValue(double quantile) { - return snapshot.quantile(quantile); - } - - @Override - public double[] getValues() { - return new double[0]; - } - - @Override - public int size() { - return (int) snapshot.size(); - } - }; - } - - @Override - public void processWith(MetricProcessor metricProcessor, MetricName metricName, T t) throws Exception { - metricProcessor.processHistogram(metricName, this, t); - } - - public static class MinuteBin { - private final TDigest dist; - private final long minMillis; - - MinuteBin(long minMillis) { - dist = new AVLTreeDigest(ACCURACY); - this.minMillis = minMillis; - } - - public TDigest getDist() { - return dist; - } - - public long getMinMillis() { - return minMillis; - } - } - - private static class TDigestSample implements Sample { - - private WavefrontHistogram wfHist; - - void set(WavefrontHistogram tdm) { - this.wfHist = tdm; - } - - @Override - public void clear() { - wfHist.clear(); - } - - @Override - public int size() { - return (int) wfHist.count(); - } - - @Override - public void update(long l) { - wfHist.update(l); - } - - @Override - public Snapshot getSnapshot() { - return wfHist.getSnapshot(); - } - - } -} diff --git a/java-lib/src/main/java/io/dropwizard/metrics5/DeltaCounter.java b/java-lib/src/main/java/io/dropwizard/metrics5/DeltaCounter.java deleted file mode 100644 index af9ef8d0b..000000000 --- a/java-lib/src/main/java/io/dropwizard/metrics5/DeltaCounter.java +++ /dev/null @@ -1,29 +0,0 @@ -package io.dropwizard.metrics5; - -import com.google.common.annotations.VisibleForTesting; -import com.wavefront.common.MetricConstants; - -/** - * A counter for Wavefront delta metrics. - * - * Differs from a counter in that it is reset in the WavefrontReporter every time the value is reported. - * - * @author Vikram Raman (vikram@wavefront.com) - */ -public class DeltaCounter extends Counter { - - @VisibleForTesting - public static synchronized DeltaCounter get(MetricRegistry registry, String metricName) { - - if (registry == null || metricName == null || metricName.isEmpty()) { - throw new IllegalArgumentException("Invalid arguments"); - } - - if (!(metricName.startsWith(MetricConstants.DELTA_PREFIX) || metricName.startsWith(MetricConstants.DELTA_PREFIX_2))) { - metricName = MetricConstants.DELTA_PREFIX + metricName; - } - DeltaCounter counter = new DeltaCounter(); - registry.register(metricName, counter); - return counter; - } -} \ No newline at end of file diff --git a/java-lib/src/main/java/io/dropwizard/metrics5/jvm/SafeFileDescriptorRatioGauge.java b/java-lib/src/main/java/io/dropwizard/metrics5/jvm/SafeFileDescriptorRatioGauge.java deleted file mode 100644 index 819648194..000000000 --- a/java-lib/src/main/java/io/dropwizard/metrics5/jvm/SafeFileDescriptorRatioGauge.java +++ /dev/null @@ -1,47 +0,0 @@ -package io.dropwizard.metrics5.jvm; - -import io.dropwizard.metrics5.RatioGauge; -import com.sun.management.UnixOperatingSystemMXBean; - -import java.lang.management.ManagementFactory; -import java.lang.management.OperatingSystemMXBean; - -/** - * Java 9 compatible implementation of FileDescriptorRatioGauge that doesn't use reflection - * and is not susceptible to an InaccessibleObjectException - * - * The gauge represents a ratio of used to total file descriptors. - * - * @author Vasily Vorontsov (vasily@wavefront.com) - */ -public class SafeFileDescriptorRatioGauge extends RatioGauge { - private final OperatingSystemMXBean os; - - /** - * Creates a new gauge using the platform OS bean. - */ - public SafeFileDescriptorRatioGauge() { - this(ManagementFactory.getOperatingSystemMXBean()); - } - - /** - * Creates a new gauge using the given OS bean. - * - * @param os an {@link OperatingSystemMXBean} - */ - public SafeFileDescriptorRatioGauge(OperatingSystemMXBean os) { - this.os = os; - } - - /** - * @return {@link com.codahale.metrics.RatioGauge.Ratio} of used to total file descriptors. - */ - protected Ratio getRatio() { - if (!(this.os instanceof UnixOperatingSystemMXBean)) { - return Ratio.of(Double.NaN, Double.NaN); - } - Long openFds = ((UnixOperatingSystemMXBean)os).getOpenFileDescriptorCount(); - Long maxFds = ((UnixOperatingSystemMXBean)os).getMaxFileDescriptorCount(); - return Ratio.of(openFds.doubleValue(), maxFds.doubleValue()); - } -} diff --git a/java-lib/src/main/resources/wavefront/templates/enum.vm b/java-lib/src/main/resources/wavefront/templates/enum.vm deleted file mode 100644 index ab37daf14..000000000 --- a/java-lib/src/main/resources/wavefront/templates/enum.vm +++ /dev/null @@ -1,34 +0,0 @@ -## -## Licensed to the Apache Software Foundation (ASF) under one -## or more contributor license agreements. See the NOTICE file -## distributed with this work for additional information -## regarding copyright ownership. The ASF licenses this file -## to you under the Apache License, Version 2.0 (the -## "License"); you may not use this file except in compliance -## with the License. You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## -#if ($schema.getNamespace()) -package $schema.getNamespace(); -#end -@SuppressWarnings("all") -#if ($schema.getDoc()) -/** $schema.getDoc() */ -#end -#foreach ($annotation in $this.javaAnnotations($schema)) -@$annotation -#end -@org.apache.avro.specific.AvroGenerated -public enum ${this.mangle($schema.getName())} { - #foreach ($symbol in ${schema.getEnumSymbols()})${this.mangle($symbol)}#if ($velocityHasNext), #end#end - ; - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.javaEscape($schema.toString())}"); - public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } -} \ No newline at end of file diff --git a/java-lib/src/main/resources/wavefront/templates/fixed.vm b/java-lib/src/main/resources/wavefront/templates/fixed.vm deleted file mode 100644 index 02b350681..000000000 --- a/java-lib/src/main/resources/wavefront/templates/fixed.vm +++ /dev/null @@ -1,65 +0,0 @@ -## -## Licensed to the Apache Software Foundation (ASF) under one -## or more contributor license agreements. See the NOTICE file -## distributed with this work for additional information -## regarding copyright ownership. The ASF licenses this file -## to you under the Apache License, Version 2.0 (the -## "License"); you may not use this file except in compliance -## with the License. You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## -#if ($schema.getNamespace()) -package $schema.getNamespace(); -#end -@SuppressWarnings("all") -#if ($schema.getDoc()) -/** $schema.getDoc() */ -#end -#foreach ($annotation in $this.javaAnnotations($schema)) -@$annotation -#end -@org.apache.avro.specific.FixedSize($schema.getFixedSize()) -@org.apache.avro.specific.AvroGenerated -public class ${this.mangle($schema.getName())} extends org.apache.avro.specific.SpecificFixed { - private static final long serialVersionUID = ${this.fingerprint64($schema)}L; - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.javaEscape($schema.toString())}"); - public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } - public org.apache.avro.Schema getSchema() { return SCHEMA$; } - - /** Creates a new ${this.mangle($schema.getName())} */ - public ${this.mangle($schema.getName())}() { - super(); - } - - /** - * Creates a new ${this.mangle($schema.getName())} with the given bytes. - * @param bytes The bytes to create the new ${this.mangle($schema.getName())}. - */ - public ${this.mangle($schema.getName())}(byte[] bytes) { - super(bytes); - } - - private static final org.apache.avro.io.DatumWriter - WRITER$ = new org.apache.avro.specific.SpecificDatumWriter<${this.mangle($schema.getName())}>(SCHEMA$); - - @Override public void writeExternal(java.io.ObjectOutput out) - throws java.io.IOException { - WRITER$.write(this, org.apache.avro.specific.SpecificData.getEncoder(out)); - } - - private static final org.apache.avro.io.DatumReader - READER$ = new org.apache.avro.specific.SpecificDatumReader<${this.mangle($schema.getName())}>(SCHEMA$); - - @Override public void readExternal(java.io.ObjectInput in) - throws java.io.IOException { - READER$.read(this, org.apache.avro.specific.SpecificData.getDecoder(in)); - } - -} \ No newline at end of file diff --git a/java-lib/src/main/resources/wavefront/templates/protocol.vm b/java-lib/src/main/resources/wavefront/templates/protocol.vm deleted file mode 100644 index 4077c8428..000000000 --- a/java-lib/src/main/resources/wavefront/templates/protocol.vm +++ /dev/null @@ -1,96 +0,0 @@ -## -## Licensed to the Apache Software Foundation (ASF) under one -## or more contributor license agreements. See the NOTICE file -## distributed with this work for additional information -## regarding copyright ownership. The ASF licenses this file -## to you under the Apache License, Version 2.0 (the -## "License"); you may not use this file except in compliance -## with the License. You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## -#if ($protocol.getNamespace()) -package $protocol.getNamespace(); -#end - -@SuppressWarnings("all") -#if ($protocol.getDoc()) -/** $protocol.getDoc() */ -#end -#foreach ($annotation in $this.javaAnnotations($protocol)) -@$annotation -#end -@org.apache.avro.specific.AvroGenerated -public interface $this.mangle($protocol.getName()) { - public static final org.apache.avro.Protocol PROTOCOL = org.apache.avro.Protocol.parse(${this.javaSplit($protocol.toString())}); -#foreach ($e in $protocol.getMessages().entrySet()) -#set ($name = $e.getKey()) -#set ($message = $e.getValue()) -#set ($response = $message.getResponse()) - /** -#if ($message.getDoc()) - * $this.escapeForJavadoc($message.getDoc()) -#end -#foreach ($p in $message.getRequest().getFields())## -#if ($p.doc()) * @param ${this.mangle($p.name())} $p.doc() -#end -#end - */ -#foreach ($annotation in $this.javaAnnotations($message)) - @$annotation -#end - #if ($message.isOneWay())void#else${this.javaUnbox($response)}#end - ${this.mangle($name)}(## -#foreach ($p in $message.getRequest().getFields())## -#* *#${this.javaUnbox($p.schema())} ${this.mangle($p.name())}#if ($velocityHasNext), #end -#end -)#if (! $message.isOneWay()) - throws org.apache.avro.AvroRemoteException## -## The first error is always "string", so we skip it. -#foreach ($error in $message.getErrors().getTypes().subList(1, $message.getErrors().getTypes().size())) -, ${this.mangle($error.getFullName())}## -#end## (error list) -#end## (one way) -; -#end## (requests) - -## Generate nested callback API - @SuppressWarnings("all") -#if ($protocol.getDoc()) - /** $protocol.getDoc() */ -#end - public interface Callback extends $this.mangle($protocol.getName()) { - public static final org.apache.avro.Protocol PROTOCOL = #if ($protocol.getNamespace())$protocol.getNamespace().#end${this.mangle($protocol.getName())}.PROTOCOL; -#foreach ($e in $protocol.getMessages().entrySet()) -#set ($name = $e.getKey()) -#set ($message = $e.getValue()) -#set ($response = $message.getResponse()) -## Generate callback method if the message is not one-way: -#if (! $message.isOneWay()) - /** -#if ($message.getDoc()) - * $this.escapeForJavadoc($message.getDoc()) -#end -#foreach ($p in $message.getRequest().getFields())## -#if ($p.doc()) * @param ${this.mangle($p.name())} $p.doc() -#end -#end - * @throws java.io.IOException The async call could not be completed. - */ - void ${this.mangle($name)}(## -#foreach ($p in $message.getRequest().getFields())## -#* *#${this.javaUnbox($p.schema())} ${this.mangle($p.name())}#if ($velocityHasNext), #end -#end -#if ($message.getRequest().getFields().size() > 0), #end -org.apache.avro.ipc.Callback<${this.javaType($response)}> callback) throws java.io.IOException; -#end## (generate callback method) -#end## (requests) - }## End of Callback interface - -}## End of protocol interface \ No newline at end of file diff --git a/java-lib/src/main/resources/wavefront/templates/record.vm b/java-lib/src/main/resources/wavefront/templates/record.vm deleted file mode 100644 index 01a78d784..000000000 --- a/java-lib/src/main/resources/wavefront/templates/record.vm +++ /dev/null @@ -1,505 +0,0 @@ -## -## Licensed to the Apache Software Foundation (ASF) under one -## or more contributor license agreements. See the NOTICE file -## distributed with this work for additional information -## regarding copyright ownership. The ASF licenses this file -## to you under the Apache License, Version 2.0 (the -## "License"); you may not use this file except in compliance -## with the License. You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## -#if ($schema.getNamespace()) -package $schema.getNamespace(); -#end - -import org.apache.avro.specific.SpecificData; -#if (!$schema.isError()) -import org.apache.avro.message.BinaryMessageEncoder; -import org.apache.avro.message.BinaryMessageDecoder; -import org.apache.avro.message.SchemaStore; -#end - -@SuppressWarnings("all") -#if ($schema.getDoc()) -/** $schema.getDoc() */ -#end -#foreach ($annotation in $this.javaAnnotations($schema)) -@$annotation -#end -@org.apache.avro.specific.AvroGenerated -public class ${this.mangle($schema.getName())}#if ($schema.isError()) extends org.apache.avro.specific.SpecificExceptionBase#else extends org.apache.avro.specific.SpecificRecordBase#end implements org.apache.avro.specific.SpecificRecord { - private static final long serialVersionUID = ${this.fingerprint64($schema)}L; - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse(${this.javaSplit($schema.toString())}); - public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } - - private static SpecificData MODEL$ = new SpecificData(); - -#if (!$schema.isError()) - private static final BinaryMessageEncoder<${this.mangle($schema.getName())}> ENCODER = - new BinaryMessageEncoder<${this.mangle($schema.getName())}>(MODEL$, SCHEMA$); - - private static final BinaryMessageDecoder<${this.mangle($schema.getName())}> DECODER = - new BinaryMessageDecoder<${this.mangle($schema.getName())}>(MODEL$, SCHEMA$); - - /** - * Return the BinaryMessageDecoder instance used by this class. - */ - public static BinaryMessageDecoder<${this.mangle($schema.getName())}> getDecoder() { - return DECODER; - } - - /** - * Create a new BinaryMessageDecoder instance for this class that uses the specified {@link SchemaStore}. - * @param resolver a {@link SchemaStore} used to find schemas by fingerprint - */ - public static BinaryMessageDecoder<${this.mangle($schema.getName())}> createDecoder(SchemaStore resolver) { - return new BinaryMessageDecoder<${this.mangle($schema.getName())}>(MODEL$, SCHEMA$, resolver); - } - - /** Serializes this ${schema.getName()} to a ByteBuffer. */ - public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException { - return ENCODER.encode(this); - } - - /** Deserializes a ${schema.getName()} from a ByteBuffer. */ - public static ${this.mangle($schema.getName())} fromByteBuffer( - java.nio.ByteBuffer b) throws java.io.IOException { - return DECODER.decode(b); - } -#end - -#foreach ($field in $schema.getFields()) -#if ($field.doc()) - /** $field.doc() */ -#end -#foreach ($annotation in $this.javaAnnotations($field)) - @$annotation -#end - #if (${this.deprecatedFields()})@Deprecated#end #if (${this.publicFields()})public#elseif (${this.privateFields()})private#end ${this.javaUnbox($field.schema())} ${this.mangle($field.name(), $schema.isError())}; -#end -#if ($schema.isError()) - - public ${this.mangle($schema.getName())}() { - super(); - } - - public ${this.mangle($schema.getName())}(Object value) { - super(value); - } - - public ${this.mangle($schema.getName())}(Throwable cause) { - super(cause); - } - - public ${this.mangle($schema.getName())}(Object value, Throwable cause) { - super(value, cause); - } - -#else -#if ($schema.getFields().size() > 0) - - /** - * Default constructor. Note that this does not initialize fields - * to their default values from the schema. If that is desired then - * one should use newBuilder(). - */ - public ${this.mangle($schema.getName())}() {} -#if ($this.isCreateAllArgsConstructor()) - - /** - * All-args constructor. -#foreach ($field in $schema.getFields()) -#if ($field.doc()) * @param ${this.mangle($field.name())} $field.doc() -#else * @param ${this.mangle($field.name())} The new value for ${field.name()} -#end -#end - */ - public ${this.mangle($schema.getName())}(#foreach($field in $schema.getFields())${this.javaType($field.schema())} ${this.mangle($field.name())}#if($velocityCount < $schema.getFields().size()), #end#end) { -#foreach ($field in $schema.getFields()) - this.${this.mangle($field.name())} = ${this.mangle($field.name())}; -#end - } -#else - /** - * This schema contains more than 254 fields which exceeds the maximum number - * of permitted constructor parameters in the JVM. An all-args constructor - * will not be generated. Please use newBuilder() to instantiate - * objects instead. - */ -#end -#end - -#end - public org.apache.avro.Schema getSchema() { return SCHEMA$; } - // Used by DatumWriter. Applications should not call. - public java.lang.Object get(int field$) { - switch (field$) { -#set ($i = 0) -#foreach ($field in $schema.getFields()) - case $i: return ${this.mangle($field.name(), $schema.isError())}; -#set ($i = $i + 1) -#end - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); - } - } - -#if ($this.hasLogicalTypeField($schema)) - protected static final org.apache.avro.data.TimeConversions.DateConversion DATE_CONVERSION = new org.apache.avro.data.TimeConversions.DateConversion(); - protected static final org.apache.avro.data.TimeConversions.TimeConversion TIME_CONVERSION = new org.apache.avro.data.TimeConversions.TimeConversion(); - protected static final org.apache.avro.data.TimeConversions.TimestampConversion TIMESTAMP_CONVERSION = new org.apache.avro.data.TimeConversions.TimestampConversion(); - protected static final org.apache.avro.Conversions.DecimalConversion DECIMAL_CONVERSION = new org.apache.avro.Conversions.DecimalConversion(); - - private static final org.apache.avro.Conversion[] conversions = - new org.apache.avro.Conversion[] { -#foreach ($field in $schema.getFields()) - ${this.conversionInstance($field.schema())}, -#end - null - }; - - @Override - public org.apache.avro.Conversion getConversion(int field) { - return conversions[field]; - } - -#end - // Used by DatumReader. Applications should not call. - @SuppressWarnings(value="unchecked") - public void put(int field$, java.lang.Object value$) { - switch (field$) { -#set ($i = 0) -#foreach ($field in $schema.getFields()) - case $i: ${this.mangle($field.name(), $schema.isError())} = (${this.javaType($field.schema())})value$; break; -#set ($i = $i + 1) -#end - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); - } - } - -#foreach ($field in $schema.getFields()) - /** - * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * @return $field.doc() -#else * @return The value of the '${this.mangle($field.name(), $schema.isError())}' field. -#end - */ - public ${this.javaType($field.schema())} ${this.generateGetMethod($schema, $field)}() { - return ${this.mangle($field.name(), $schema.isError())}; - } - -#if ($this.createSetters) - /** - * Sets the value of the '${this.mangle($field.name(), $schema.isError())}' field. - #if ($field.doc()) * $field.doc()#end - * @param value the value to set. - */ - #if ($field.schema().getType().name() == "UNION" && $this.javaType($field.schema()) == "java.lang.Object") - #foreach ($type in $field.schema().getTypes()) - public void ${this.generateSetMethod($schema, $field)}(${this.javaType($type)} value) { - this.${this.mangle($field.name(), $schema.isError())} = value; - } - #end - #else - public void ${this.generateSetMethod($schema, $field)}(${this.javaType($field.schema())} value) { - this.${this.mangle($field.name(), $schema.isError())} = value; - } - #end -#end - -#end - /** - * Creates a new ${this.mangle($schema.getName())} RecordBuilder. - * @return A new ${this.mangle($schema.getName())} RecordBuilder - */ - public static #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder newBuilder() { - return new #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder(); - } - - /** - * Creates a new ${this.mangle($schema.getName())} RecordBuilder by copying an existing Builder. - * @param other The existing builder to copy. - * @return A new ${this.mangle($schema.getName())} RecordBuilder - */ - public static #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder newBuilder(#if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder other) { - if (other == null) { - return new #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder(); - } else { - return new #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder(other); - } - } - - /** - * Creates a new ${this.mangle($schema.getName())} RecordBuilder by copying an existing $this.mangle($schema.getName()) instance. - * @param other The existing instance to copy. - * @return A new ${this.mangle($schema.getName())} RecordBuilder - */ - public static #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder newBuilder(#if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())} other) { - if (other == null) { - return new #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder(); - } else { - return new #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder(other); - } - } - - /** - * RecordBuilder for ${this.mangle($schema.getName())} instances. - */ - public static class Builder extends#if ($schema.isError()) org.apache.avro.specific.SpecificErrorBuilderBase<${this.mangle($schema.getName())}>#else org.apache.avro.specific.SpecificRecordBuilderBase<${this.mangle($schema.getName())}>#end - - implements#if ($schema.isError()) org.apache.avro.data.ErrorBuilder<${this.mangle($schema.getName())}>#else org.apache.avro.data.RecordBuilder<${this.mangle($schema.getName())}>#end { - -#foreach ($field in $schema.getFields()) -#if ($field.doc()) - /** $field.doc() */ -#end - private ${this.javaUnbox($field.schema())} ${this.mangle($field.name(), $schema.isError())}; -#if (${this.hasBuilder($field.schema())}) - private ${this.javaUnbox($field.schema())}.Builder ${this.mangle($field.name(), $schema.isError())}Builder; -#end -#end - - /** Creates a new Builder */ - private Builder() { - super(SCHEMA$); - } - - /** - * Creates a Builder by copying an existing Builder. - * @param other The existing Builder to copy. - */ - private Builder(#if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder other) { - super(other); -#foreach ($field in $schema.getFields()) - if (isValidValue(fields()[$field.pos()], other.${this.mangle($field.name(), $schema.isError())})) { - this.${this.mangle($field.name(), $schema.isError())} = data().deepCopy(fields()[$field.pos()].schema(), other.${this.mangle($field.name(), $schema.isError())}); - fieldSetFlags()[$field.pos()] = other.fieldSetFlags()[$field.pos()]; - } -#if (${this.hasBuilder($field.schema())}) - if (other.${this.generateHasBuilderMethod($schema, $field)}()) { - this.${this.mangle($field.name(), $schema.isError())}Builder = ${this.javaType($field.schema())}.newBuilder(other.${this.generateGetBuilderMethod($schema, $field)}()); - } -#end -#end - } - - /** - * Creates a Builder by copying an existing $this.mangle($schema.getName()) instance - * @param other The existing instance to copy. - */ - private Builder(#if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())} other) { -#if ($schema.isError()) super(other)#else - super(SCHEMA$)#end; -#foreach ($field in $schema.getFields()) - if (isValidValue(fields()[$field.pos()], other.${this.mangle($field.name(), $schema.isError())})) { - this.${this.mangle($field.name(), $schema.isError())} = data().deepCopy(fields()[$field.pos()].schema(), other.${this.mangle($field.name(), $schema.isError())}); - fieldSetFlags()[$field.pos()] = true; - } -#if (${this.hasBuilder($field.schema())}) - this.${this.mangle($field.name(), $schema.isError())}Builder = null; -#end -#end - } -#if ($schema.isError()) - - @Override - public #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder setValue(Object value) { - super.setValue(value); - return this; - } - - @Override - public #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder clearValue() { - super.clearValue(); - return this; - } - - @Override - public #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder setCause(Throwable cause) { - super.setCause(cause); - return this; - } - - @Override - public #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder clearCause() { - super.clearCause(); - return this; - } -#end - -#foreach ($field in $schema.getFields()) - /** - * Gets the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * $field.doc() -#end - * @return The value. - */ - public ${this.javaType($field.schema())} ${this.generateGetMethod($schema, $field)}() { - return ${this.mangle($field.name(), $schema.isError())}; - } - #if ($field.schema().getType().name() == "UNION" && $this.javaType($field.schema()) == "java.lang.Object") - #foreach ($type in $field.schema().getTypes()) - - /** - * Sets the value of the '${this.mangle($field.name(), $schema.isError())}' field. - #if ($field.doc()) * $field.doc() - #end - * @param value The value of '${this.mangle($field.name(), $schema.isError())}'. - * @return This builder. - */ - public #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder ${this.generateSetMethod($schema, $field)}(${this.javaUnbox($type)} value) { - validate(fields()[$field.pos()], value); - #if (${this.hasBuilder($field.schema())}) - this.${this.mangle($field.name(), $schema.isError())}Builder = null; - #end - this.${this.mangle($field.name(), $schema.isError())} = value; - fieldSetFlags()[$field.pos()] = true; - return this; - } - #end - #else - - /** - * Sets the value of the '${this.mangle($field.name(), $schema.isError())}' field. - #if ($field.doc()) * $field.doc() - #end - * @param value The value of '${this.mangle($field.name(), $schema.isError())}'. - * @return This builder. - */ - public #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder ${this.generateSetMethod($schema, $field)}(${this.javaUnbox($field.schema())} value) { - validate(fields()[$field.pos()], value); - #if (${this.hasBuilder($field.schema())}) - this.${this.mangle($field.name(), $schema.isError())}Builder = null; - #end - this.${this.mangle($field.name(), $schema.isError())} = value; - fieldSetFlags()[$field.pos()] = true; - return this; - } - #end - - /** - * Checks whether the '${this.mangle($field.name(), $schema.isError())}' field has been set. -#if ($field.doc()) * $field.doc() -#end - * @return True if the '${this.mangle($field.name(), $schema.isError())}' field has been set, false otherwise. - */ - public boolean ${this.generateHasMethod($schema, $field)}() { - return fieldSetFlags()[$field.pos()]; - } - -#if (${this.hasBuilder($field.schema())}) - /** - * Gets the Builder instance for the '${this.mangle($field.name(), $schema.isError())}' field and creates one if it doesn't exist yet. -#if ($field.doc()) * $field.doc() -#end - * @return This builder. - */ - public ${this.javaType($field.schema())}.Builder ${this.generateGetBuilderMethod($schema, $field)}() { - if (${this.mangle($field.name(), $schema.isError())}Builder == null) { - if (${this.generateHasMethod($schema, $field)}()) { - ${this.generateSetBuilderMethod($schema, $field)}(${this.javaType($field.schema())}.newBuilder(${this.mangle($field.name(), $schema.isError())})); - } else { - ${this.generateSetBuilderMethod($schema, $field)}(${this.javaType($field.schema())}.newBuilder()); - } - } - return ${this.mangle($field.name(), $schema.isError())}Builder; - } - - /** - * Sets the Builder instance for the '${this.mangle($field.name(), $schema.isError())}' field -#if ($field.doc()) * $field.doc() -#end - * @param value The builder instance that must be set. - * @return This builder. - */ - public #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder ${this.generateSetBuilderMethod($schema, $field)}(${this.javaUnbox($field.schema())}.Builder value) { - ${this.generateClearMethod($schema, $field)}(); - ${this.mangle($field.name(), $schema.isError())}Builder = value; - return this; - } - - /** - * Checks whether the '${this.mangle($field.name(), $schema.isError())}' field has an active Builder instance -#if ($field.doc()) * $field.doc() -#end - * @return True if the '${this.mangle($field.name(), $schema.isError())}' field has an active Builder instance - */ - public boolean ${this.generateHasBuilderMethod($schema, $field)}() { - return ${this.mangle($field.name(), $schema.isError())}Builder != null; - } -#end - - /** - * Clears the value of the '${this.mangle($field.name(), $schema.isError())}' field. -#if ($field.doc()) * $field.doc() -#end - * @return This builder. - */ - public #if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.Builder ${this.generateClearMethod($schema, $field)}() { -#if (${this.isUnboxedJavaTypeNullable($field.schema())}) - ${this.mangle($field.name(), $schema.isError())} = null; -#end -#if (${this.hasBuilder($field.schema())}) - ${this.mangle($field.name(), $schema.isError())}Builder = null; -#end - fieldSetFlags()[$field.pos()] = false; - return this; - } - -#end - @Override - @SuppressWarnings("unchecked") - public ${this.mangle($schema.getName())} build() { - try { - ${this.mangle($schema.getName())} record = new ${this.mangle($schema.getName())}(#if ($schema.isError())getValue(), getCause()#end); -#foreach ($field in $schema.getFields()) -#if (${this.hasBuilder($field.schema())}) - if (${this.mangle($field.name(), $schema.isError())}Builder != null) { - record.${this.mangle($field.name(), $schema.isError())} = this.${this.mangle($field.name(), $schema.isError())}Builder.build(); - } else { -#if ($this.hasLogicalTypeField($schema)) - record.${this.mangle($field.name(), $schema.isError())} = fieldSetFlags()[$field.pos()] ? this.${this.mangle($field.name(), $schema.isError())} : (${this.javaType($field.schema())}) defaultValue(fields()[$field.pos()], record.getConversion($field.pos())); -#else - record.${this.mangle($field.name(), $schema.isError())} = fieldSetFlags()[$field.pos()] ? this.${this.mangle($field.name(), $schema.isError())} : (${this.javaType($field.schema())}) defaultValue(fields()[$field.pos()]); -#end - } -#else -#if ($this.hasLogicalTypeField($schema)) - record.${this.mangle($field.name(), $schema.isError())} = fieldSetFlags()[$field.pos()] ? this.${this.mangle($field.name(), $schema.isError())} : (${this.javaType($field.schema())}) defaultValue(fields()[$field.pos()], record.getConversion($field.pos())); -#else - record.${this.mangle($field.name(), $schema.isError())} = fieldSetFlags()[$field.pos()] ? this.${this.mangle($field.name(), $schema.isError())} : (${this.javaType($field.schema())}) defaultValue(fields()[$field.pos()]); -#end -#end -#end - return record; - } catch (java.lang.Exception e) { - throw new org.apache.avro.AvroRuntimeException(e); - } - } - } - - @SuppressWarnings("unchecked") - private static final org.apache.avro.io.DatumWriter<${this.mangle($schema.getName())}> - WRITER$ = (org.apache.avro.io.DatumWriter<${this.mangle($schema.getName())}>)MODEL$.createDatumWriter(SCHEMA$); - - @Override public void writeExternal(java.io.ObjectOutput out) - throws java.io.IOException { - WRITER$.write(this, SpecificData.getEncoder(out)); - } - - @SuppressWarnings("unchecked") - private static final org.apache.avro.io.DatumReader<${this.mangle($schema.getName())}> - READER$ = (org.apache.avro.io.DatumReader<${this.mangle($schema.getName())}>)MODEL$.createDatumReader(SCHEMA$); - - @Override public void readExternal(java.io.ObjectInput in) - throws java.io.IOException { - READER$.read(this, SpecificData.getDecoder(in)); - } - -} \ No newline at end of file diff --git a/java-lib/src/test/java/com/wavefront/common/MetricManglerTest.java b/java-lib/src/test/java/com/wavefront/common/MetricManglerTest.java deleted file mode 100644 index 59f3a9564..000000000 --- a/java-lib/src/test/java/com/wavefront/common/MetricManglerTest.java +++ /dev/null @@ -1,82 +0,0 @@ -package com.wavefront.common; - -import static org.junit.Assert.*; -import org.junit.Test; - -public class MetricManglerTest { - @Test - public void testSourceMetricParsing() { - String testInput = "hosts.sjc123.cpu.loadavg.1m"; - String expectedOutput = "cpu.loadavg.1m"; - - { - MetricMangler mangler = new MetricMangler("2", "", "1"); - MetricMangler.MetricComponents c = mangler.extractComponents(testInput); - assertEquals(expectedOutput, c.metric); - assertEquals("sjc123", c.source); - } - { - MetricMangler mangler = new MetricMangler("2", null, "1"); - MetricMangler.MetricComponents c = mangler.extractComponents(testInput); - assertEquals(expectedOutput, c.metric); - assertEquals("sjc123", c.source); - } - } - - @Test - public void testSourceMetricParsingNoRemove() { - String testInput = "hosts.sjc123.cpu.loadavg.1m"; - String expectedOutput = "hosts.cpu.loadavg.1m"; - - { - MetricMangler mangler = new MetricMangler("2", "", ""); - MetricMangler.MetricComponents c = mangler.extractComponents(testInput); - assertEquals(expectedOutput, c.metric); - assertEquals("sjc123", c.source); - } - { - MetricMangler mangler = new MetricMangler("2", null, null); - MetricMangler.MetricComponents c = mangler.extractComponents(testInput); - assertEquals(expectedOutput, c.metric); - assertEquals("sjc123", c.source); - } - } - - @Test - public void testSourceMetricParsingNoSourceAndNoRemove() { - String testInput = "hosts.sjc123.cpu.loadavg.1m"; - String expectedOutput = "hosts.sjc123.cpu.loadavg.1m"; - - { - MetricMangler mangler = new MetricMangler("", "", ""); - MetricMangler.MetricComponents c = mangler.extractComponents(testInput); - assertEquals(expectedOutput, c.metric); - assertEquals(null, c.source); - } - { - MetricMangler mangler = new MetricMangler(null, null, null); - MetricMangler.MetricComponents c = mangler.extractComponents(testInput); - assertEquals(expectedOutput, c.metric); - assertEquals(null, c.source); - } - } - - @Test - public void testSourceMetricParsingWithTags() { - String testInput = "hosts.sjc123.cpu.loadavg.1m;foo=bar;boo=baz"; - String expectedOutput = "cpu.loadavg.1m"; - - { - MetricMangler mangler = new MetricMangler("2", "", "1"); - MetricMangler.MetricComponents c = mangler.extractComponents(testInput); - assertEquals(expectedOutput, c.metric); - assertEquals("sjc123", c.source); - assertNotEquals(null, c.annotations); - assertEquals(2, c.annotations.length); - assertEquals("foo=bar", c.annotations[0]); - assertEquals("boo=baz", c.annotations[1]); - } - } - - -} diff --git a/java-lib/src/test/java/com/wavefront/common/ReportPointTest.java b/java-lib/src/test/java/com/wavefront/common/ReportPointTest.java deleted file mode 100644 index d47dc6e00..000000000 --- a/java-lib/src/test/java/com/wavefront/common/ReportPointTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.wavefront.common; - -import org.junit.Test; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import wavefront.report.ReportPoint; - -import static org.junit.Assert.assertEquals; - -public class ReportPointTest { - - /** - * This captures an issue where the latest Avro vm templates does not generate overloaded settings for Avro objects - * and builders. - */ - @Test - public void testReportPointBuilderSetters() throws IOException { - // test integer to long widening conversion. - ReportPoint rp = ReportPoint.newBuilder(). - setValue(1). - setMetric("hello"). - setTimestamp(12345). - build(); - ByteBuffer byteBuffer = rp.toByteBuffer(); - byteBuffer.rewind(); - assertEquals(1L, ReportPoint.fromByteBuffer(byteBuffer).getValue()); - - // test long (no widening conversion. - rp = ReportPoint.newBuilder(). - setValue(1L). - setMetric("hello"). - setTimestamp(12345). - build(); - byteBuffer = rp.toByteBuffer(); - byteBuffer.rewind(); - assertEquals(1L, ReportPoint.fromByteBuffer(byteBuffer).getValue()); - - // test float to double widening conversion. - rp = ReportPoint.newBuilder(). - setValue(1f). - setMetric("hello"). - setTimestamp(12345). - build(); - byteBuffer = rp.toByteBuffer(); - byteBuffer.rewind(); - assertEquals(1.0D, ReportPoint.fromByteBuffer(byteBuffer).getValue()); - - // test long (no widening conversion. - rp = ReportPoint.newBuilder(). - setValue(1.0D). - setMetric("hello"). - setTimestamp(12345). - build(); - byteBuffer = rp.toByteBuffer(); - byteBuffer.rewind(); - assertEquals(1.0D, ReportPoint.fromByteBuffer(byteBuffer).getValue()); - } -} diff --git a/java-lib/src/test/java/com/wavefront/data/ValidationTest.java b/java-lib/src/test/java/com/wavefront/data/ValidationTest.java deleted file mode 100644 index b05b1ff2f..000000000 --- a/java-lib/src/test/java/com/wavefront/data/ValidationTest.java +++ /dev/null @@ -1,127 +0,0 @@ -package com.wavefront.data; - -import com.wavefront.ingester.HistogramDecoder; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import wavefront.report.ReportPoint; - -/** - * @author vasily@wavefront.com - */ -public class ValidationTest { - - @Test - public void testPointIllegalChars() { - String input = "metric1"; - Assert.assertTrue(Validation.charactersAreValid(input)); - - input = "good.metric2"; - Assert.assertTrue(Validation.charactersAreValid(input)); - - input = "good-metric3"; - Assert.assertTrue(Validation.charactersAreValid(input)); - - input = "good_metric4"; - Assert.assertTrue(Validation.charactersAreValid(input)); - - input = "good,metric5"; - Assert.assertTrue(Validation.charactersAreValid(input)); - - input = "good/metric6"; - Assert.assertTrue(Validation.charactersAreValid(input)); - - // first character can no longer be ~ - input = "~good.metric7"; - Assert.assertTrue(Validation.charactersAreValid(input)); - - // first character can be ∆ (\u2206) - input = "∆delta.metric8"; - Assert.assertTrue(Validation.charactersAreValid(input)); - - // first character can be Δ (\u0394) - input = "Δdelta.metric9"; - Assert.assertTrue(Validation.charactersAreValid(input)); - - // non-first character cannot be ~ - input = "~good.~metric"; - Assert.assertFalse(Validation.charactersAreValid(input)); - - // non-first character cannot be ∆ (\u2206) - input = "∆delta.∆metric"; - Assert.assertFalse(Validation.charactersAreValid(input)); - - // non-first character cannot be Δ (\u0394) - input = "∆delta.Δmetric"; - Assert.assertFalse(Validation.charactersAreValid(input)); - - // cannot end in ~ - input = "good.metric.~"; - Assert.assertFalse(Validation.charactersAreValid(input)); - - // cannot end in ∆ (\u2206) - input = "delta.metric.∆"; - Assert.assertFalse(Validation.charactersAreValid(input)); - - // cannot end in Δ (\u0394) - input = "delta.metric.Δ"; - Assert.assertFalse(Validation.charactersAreValid(input)); - - input = "abcdefghijklmnopqrstuvwxyz.0123456789,/_-ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - Assert.assertTrue(Validation.charactersAreValid(input)); - - input = "abcdefghijklmnopqrstuvwxyz.0123456789,/_-ABCDEFGHIJKLMNOPQRSTUVWXYZ~"; - Assert.assertFalse(Validation.charactersAreValid(input)); - - input = "as;df"; - Assert.assertFalse(Validation.charactersAreValid(input)); - - input = "as:df"; - Assert.assertFalse(Validation.charactersAreValid(input)); - - input = "as df"; - Assert.assertFalse(Validation.charactersAreValid(input)); - - input = "as'df"; - Assert.assertFalse(Validation.charactersAreValid(input)); - } - - @Test - public void testPointAnnotationKeyValidation() { - Map goodMap = new HashMap(); - goodMap.put("key", "value"); - - Map badMap = new HashMap(); - badMap.put("k:ey", "value"); - - ReportPoint rp = new ReportPoint("some metric", System.currentTimeMillis(), 10L, "host", "table", - goodMap); - Assert.assertTrue(Validation.annotationKeysAreValid(rp)); - - rp.setAnnotations(badMap); - Assert.assertFalse(Validation.annotationKeysAreValid(rp)); - } - - @Test - public void testValidHistogram() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - decoder.decodeReportPoints("!M 1533849540 #1 0.0 #2 1.0 #3 3.0 TestMetric source=Test key=value", out, "dummy"); - Validation.validatePoint(out.get(0), "test", Validation.Level.NUMERIC_ONLY); - } - - @Test(expected = IllegalArgumentException.class) - public void testEmptyHistogramThrows() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - decoder.decodeReportPoints("!M 1533849540 #0 0.0 #0 1.0 #0 3.0 TestMetric source=Test key=value", out, "dummy"); - Validation.validatePoint(out.get(0), "test", Validation.Level.NUMERIC_ONLY); - Assert.fail("Empty Histogram should fail validation!"); - } -} diff --git a/java-lib/src/test/java/com/wavefront/ingester/GraphiteDecoderTest.java b/java-lib/src/test/java/com/wavefront/ingester/GraphiteDecoderTest.java deleted file mode 100644 index 85b45c53a..000000000 --- a/java-lib/src/test/java/com/wavefront/ingester/GraphiteDecoderTest.java +++ /dev/null @@ -1,519 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.collect.Lists; - -import org.junit.Ignore; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import wavefront.report.ReportPoint; - -import static junit.framework.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * TODO: Edit this

User: sam Date: 7/13/13 Time: 11:34 AM - */ -public class GraphiteDecoderTest { - - private final static List emptyCustomSourceTags = Collections.emptyList(); - - @Test - public void testDoubleFormat() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93.123e3 host=vehicle_2554", out); - ReportPoint point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93123.0, point.getValue()); - assertEquals("vehicle_2554", point.getHost()); - - out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level -93.123e3 host=vehicle_2554", out); - point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(-93123.0, point.getValue()); - assertEquals("vehicle_2554", point.getHost()); - - out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level -93.123e3", out); - point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(-93123.0, point.getValue()); - assertEquals("localhost", point.getHost()); - assertNotNull(point.getAnnotations()); - assertTrue(point.getAnnotations().isEmpty()); - - out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93.123e-3 host=vehicle_2554", out); - point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(0.093123, point.getValue()); - assertEquals("vehicle_2554", point.getHost()); - - out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level -93.123e-3 host=vehicle_2554", out); - point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(-0.093123, point.getValue()); - assertEquals("vehicle_2554", point.getHost()); - - List points = Lists.newArrayList(); - decoder.decodeReportPoints("test.devnag.10 100 host=ip1", points, "tsdb"); - point = points.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("test.devnag.10", point.getMetric()); - assertEquals(100.0, point.getValue()); - assertEquals("ip1", point.getHost()); - - points = Lists.newArrayList(); - decoder.decodeReportPoints("∆test.devnag.10 100 host=ip1", points, "tsdb"); - point = points.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("∆test.devnag.10", point.getMetric()); - assertEquals(100.0, point.getValue()); - assertEquals("ip1", point.getHost()); - - points.clear(); - decoder.decodeReportPoints("test.devnag.10 100 host=ip1 a=500", points, "tsdb"); - point = points.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("test.devnag.10", point.getMetric()); - assertEquals(100.0, point.getValue()); - assertEquals("ip1", point.getHost()); - assertEquals(1, point.getAnnotations().size()); - assertEquals("500", point.getAnnotations().get("a")); - points.clear(); - decoder.decodeReportPoints("test.devnag.10 100 host=ip1 b=500", points, "tsdb"); - point = points.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("test.devnag.10", point.getMetric()); - assertEquals(100.0, point.getValue()); - assertEquals("ip1", point.getHost()); - assertEquals(1, point.getAnnotations().size()); - assertEquals("500", point.getAnnotations().get("b")); - points.clear(); - decoder.decodeReportPoints("test.devnag.10 100 host=ip1 A=500", points, "tsdb"); - point = points.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("test.devnag.10", point.getMetric()); - assertEquals(100.0, point.getValue()); - assertEquals("ip1", point.getHost()); - assertEquals(1, point.getAnnotations().size()); - assertEquals("500", point.getAnnotations().get("A")); - } - - @Test - public void testTagVWithDigitAtBeginning() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93 host=vehicle_2554 version=1_0", out); - ReportPoint point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals("vehicle_2554", point.getHost()); - assertEquals("1_0", point.getAnnotations().get("version")); - } - - @Test - public void testFormat() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93 host=vehicle_2554", out); - ReportPoint point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals("vehicle_2554", point.getHost()); - } - - @Test - public void testIpV4Host() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93 host=10.0.0.1", out); - ReportPoint point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals("10.0.0.1", point.getHost()); - } - - @Test - public void testIpV6Host() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93 host=2001:db8:3333:4444:5555:6666:7777:8888", out); - ReportPoint point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals("2001:db8:3333:4444:5555:6666:7777:8888", point.getHost()); - } - - @Test - public void testFormatWithTimestamp() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93 1234567890.246 host=vehicle_2554", out); - ReportPoint point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals(1234567890246L, point.getTimestamp().longValue()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals("vehicle_2554", point.getHost()); - } - - @Test - public void testFormatWithNoTags() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93", out); - ReportPoint point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - } - - @Test - public void testFormatWithNoTagsAndTimestamp() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93 1234567890.246", out); - ReportPoint point = out.get(0); - assertEquals("tsdb", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals(1234567890246L, point.getTimestamp().longValue()); - } - - @Test - public void testDecodeWithNoCustomer() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("vehicle.charge.battery_level 93 host=vehicle_2554", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals("vehicle_2554", point.getHost()); - } - - @Test - public void testDecodeWithNoCustomerWithTimestamp() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("vehicle.charge.battery_level 93 1234567890.246 host=vehicle_2554", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals(1234567890246L, point.getTimestamp().longValue()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals("vehicle_2554", point.getHost()); - } - - @Test - public void testDecodeWithNoCustomerWithNoTags() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("vehicle.charge.battery_level 93", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - } - - @Test - public void testDecodeWithNoCustomerWithNoTagsAndTimestamp() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("vehicle.charge.battery_level 93 1234567890.246", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals(1234567890246L, point.getTimestamp().longValue()); - } - - @Test - public void testDecodeWithMillisTimestamp() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("vehicle.charge.battery_level 93 1234567892468", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals(1234567892468L, point.getTimestamp().longValue()); - } - - @Test - public void testMetricWithNumberStarting() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("1vehicle.charge.battery_level 93 1234567890.246", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("1vehicle.charge.battery_level", point.getMetric()); - assertEquals(93.0, point.getValue()); - assertEquals(1234567890246L, point.getTimestamp().longValue()); - } - - @Test - public void testQuotedMetric() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("\"1vehicle.charge.$()+battery_level\" 93 1234567890.246 host=12345 " + - "blah=\"test hello\" \"hello world\"=test", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("1vehicle.charge.$()+battery_level", point.getMetric()); - assertEquals("12345", point.getHost()); - assertEquals(93.0, point.getValue()); - assertEquals(1234567890246L, point.getTimestamp().longValue()); - assertEquals("test hello", point.getAnnotations().get("blah")); - assertEquals("test", point.getAnnotations().get("hello world")); - } - - @Test - public void testMetricWithAnnotationQuoted() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("1vehicle.charge.battery_level 93 1234567890.246 host=12345 blah=\"test hello\" " + - "\"hello world\"=test", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("1vehicle.charge.battery_level", point.getMetric()); - assertEquals("12345", point.getHost()); - assertEquals(93.0, point.getValue()); - assertEquals(1234567890246L, point.getTimestamp().longValue()); - assertEquals("test hello", point.getAnnotations().get("blah")); - assertEquals("test", point.getAnnotations().get("hello world")); - } - - @Test - public void testQuotes() { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("\"1vehicle.charge.'battery_level\" 93 1234567890.246 " + - "host=12345 blah=\"test'\\\"hello\" \"hello world\"=test", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("1vehicle.charge.'battery_level", point.getMetric()); - assertEquals("12345", point.getHost()); - assertEquals(93.0, point.getValue()); - assertEquals(1234567890246L, point.getTimestamp().longValue()); - assertEquals("test'\"hello", point.getAnnotations().get("blah")); - assertEquals("test", point.getAnnotations().get("hello world")); - } - - @Test - public void testSimple() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("test 1 host=test", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("test", point.getMetric()); - assertEquals("test", point.getHost()); - assertEquals(1.0, point.getValue()); - } - - @Test - public void testSource() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("test 1 source=test", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("test", point.getMetric()); - assertEquals("test", point.getHost()); - assertEquals(1.0, point.getValue()); - } - - @Test - public void testSourcePriority() throws Exception { - List customSourceTags = new ArrayList(); - customSourceTags.add("fqdn"); - GraphiteDecoder decoder = new GraphiteDecoder(customSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("test 1 source=test host=bar fqdn=foo", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("test", point.getMetric()); - assertEquals("test", point.getHost()); - assertEquals("bar", point.getAnnotations().get("_host")); - assertEquals("foo", point.getAnnotations().get("fqdn")); - assertEquals(1.0, point.getValue()); - } - - @Test - public void testFQDNasSource() throws Exception { - List customSourceTags = new ArrayList(); - customSourceTags.add("fqdn"); - customSourceTags.add("hostname"); - GraphiteDecoder decoder = new GraphiteDecoder(customSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("test 1 hostname=machine fqdn=machine.company.com", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("test", point.getMetric()); - assertEquals("machine.company.com", point.getHost()); - assertEquals("machine.company.com", point.getAnnotations().get("fqdn")); - assertEquals("machine", point.getAnnotations().get("hostname")); - assertEquals(1.0, point.getValue()); - } - - - @Test - public void testUserPrefOverridesDefault() throws Exception { - List customSourceTags = new ArrayList(); - customSourceTags.add("fqdn"); - GraphiteDecoder decoder = new GraphiteDecoder("localhost", customSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("test 1 fqdn=machine.company.com", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("test", point.getMetric()); - assertEquals("machine.company.com", point.getHost()); - assertEquals("machine.company.com", point.getAnnotations().get("fqdn")); - assertEquals(1.0, point.getValue()); - } - - @Test - public void testTagRewrite() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("test 1 source=test tag=bar", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("test", point.getMetric()); - assertEquals("test", point.getHost()); - assertEquals("bar", point.getAnnotations().get("_tag")); - assertEquals(1.0, point.getValue()); - } - - @Test - public void testMONIT2576() { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("vm.guest.virtualDisk.mediumSeeks.latest 4.00 1439250320 " + - "host=iadprdhyp02.iad.corp.com guest=47173170-2069-4bcc-9bd4-041055b554ec " + - "instance=ide0_0", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("vm.guest.virtualDisk.mediumSeeks.latest", point.getMetric()); - assertEquals("iadprdhyp02.iad.corp.com", point.getHost()); - assertEquals("47173170-2069-4bcc-9bd4-041055b554ec", point.getAnnotations().get("guest")); - assertEquals("ide0_0", point.getAnnotations().get("instance")); - assertEquals(4.0, point.getValue()); - - out = new ArrayList<>(); - try { - decoder.decodeReportPoints("test.metric 1 host=test test=\"", out, "customer"); - fail("should throw"); - } catch (Exception ignored) { - } - } - - @Test - public void testNumberLookingTagValue() { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("vm.guest.virtualDisk.mediumSeeks.latest 4.00 1439250320 " + - "host=iadprdhyp02.iad.corp.com version=\"1.0.0-030051.d0e485f\"", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("vm.guest.virtualDisk.mediumSeeks.latest", point.getMetric()); - assertEquals("iadprdhyp02.iad.corp.com", point.getHost()); - assertEquals("1.0.0-030051.d0e485f", point.getAnnotations().get("version")); - assertEquals(4.0, point.getValue()); - } - - @Test - public void testNumberLookingTagValue2() { - GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags); - List out = Lists.newArrayList(); - decoder.decodeReportPoints("vm.guest.virtualDisk.mediumSeeks.latest 4.00 1439250320 " + - "host=iadprdhyp02.iad.corp.com version=\"1.0.0\\\"-030051.d0e485f\"", out, "customer"); - ReportPoint point = out.get(0); - assertEquals("customer", point.getTable()); - assertEquals("vm.guest.virtualDisk.mediumSeeks.latest", point.getMetric()); - assertEquals("iadprdhyp02.iad.corp.com", point.getHost()); - assertEquals("1.0.0\"-030051.d0e485f", point.getAnnotations().get("version")); - assertEquals(4.0, point.getValue()); - } - - @Ignore - @Test - public void testBenchmark() throws Exception { - List customSourceTags = new ArrayList(); - customSourceTags.add("fqdn"); - customSourceTags.add("hostname"); - customSourceTags.add("highcardinalitytag1"); - customSourceTags.add("highcardinalitytag2"); - customSourceTags.add("highcardinalitytag3"); - customSourceTags.add("highcardinalitytag4"); - customSourceTags.add("highcardinalitytag5"); - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - int ITERATIONS = 1000000; - for (int i = 0; i < ITERATIONS / 1000; i++) { - List out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93 123456 highcardinalitytag5=vehicle_2554", out); - } - long start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - List out = new ArrayList<>(); - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 93 123456 highcardinalitytag5=vehicle_2554", out); - } - double end = System.currentTimeMillis(); - System.out.println(ITERATIONS / ((end - start) / 1000) + " DPS"); - } - - @Test - public void testNegativeDeltas() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - try { - decoder.decodeReportPoints("∆request.count -1 source=test.wavefront.com", out); - decoder.decodeReportPoints("Δrequest.count -1 source=test.wavefront.com", out); - fail("should throw"); - } catch (RuntimeException ignored) { - } - } - - @Test - public void testPositiveDeltas() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - try { - decoder.decodeReportPoints("∆request.count 1 source=test.wavefront.com", out); - decoder.decodeReportPoints("Δrequest.count 1 source=test.wavefront.com", out); - } catch (RuntimeException e) { - fail("should not throw"); - } - } - - @Test - public void testZeroDeltaValue() throws Exception { - GraphiteDecoder decoder = new GraphiteDecoder("localhost", emptyCustomSourceTags); - List out = new ArrayList<>(); - try { - decoder.decodeReportPoints("∆request.count 0 source=test.wavefront.com", out); - decoder.decodeReportPoints("Δrequest.count 0 source=test.wavefront.com", out); - fail("should throw"); - } catch (RuntimeException ignored) { - } - } -} diff --git a/java-lib/src/test/java/com/wavefront/ingester/GraphiteHostAnnotatorTest.java b/java-lib/src/test/java/com/wavefront/ingester/GraphiteHostAnnotatorTest.java deleted file mode 100644 index 042ad0460..000000000 --- a/java-lib/src/test/java/com/wavefront/ingester/GraphiteHostAnnotatorTest.java +++ /dev/null @@ -1,100 +0,0 @@ -package com.wavefront.ingester; - -import org.junit.Ignore; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -import static junit.framework.Assert.assertEquals; - -/** - * Tests for GraphiteHostAnnotator. - * - * @author Conor Beverland (conor@wavefront.com). - */ -public class GraphiteHostAnnotatorTest { - - private final static List emptyCustomSourceTags = Collections.emptyList(); - - @Test - public void testHostMatches() throws Exception { - GraphiteHostAnnotator handler = new GraphiteHostAnnotator("test.host.com", emptyCustomSourceTags); - List out = new LinkedList(); - String msg = "test.metric 1 host=foo"; - handler.decode(null, msg, out); - assertEquals(msg, out.get(0)); - } - - @Test - public void testSourceMatches() throws Exception { - GraphiteHostAnnotator handler = new GraphiteHostAnnotator("test.host.com", emptyCustomSourceTags); - List out = new LinkedList(); - String msg = "test.metric 1 source=foo"; - handler.decode(null, msg, out); - assertEquals(msg, out.get(0)); - } - - @Test - public void testSourceAdded() throws Exception { - GraphiteHostAnnotator handler = new GraphiteHostAnnotator("test.host.com", emptyCustomSourceTags); - List out = new LinkedList(); - String msg = "test.metric 1"; - handler.decode(null, msg, out); - assertEquals("test.metric 1 source=\"test.host.com\"", out.get(0)); - } - - @Test - public void testCustomTagMatches() throws Exception { - List customSourceTags = new ArrayList(); - customSourceTags.add("fqdn"); - customSourceTags.add("hostname"); - GraphiteHostAnnotator handler = new GraphiteHostAnnotator("test.host.com", customSourceTags); - List out = new LinkedList(); - String msg = "test.metric 1 fqdn=test"; - handler.decode(null, msg, out); - assertEquals(msg, out.get(0)); - } - - @Test - public void testSourceAddedWithCustomTags() throws Exception { - List customSourceTags = new ArrayList(); - customSourceTags.add("fqdn"); - customSourceTags.add("hostname"); - GraphiteHostAnnotator handler = new GraphiteHostAnnotator("test.host.com", customSourceTags); - List out = new LinkedList(); - String msg = "test.metric 1 foo=bar"; - handler.decode(null, msg, out); - assertEquals("test.metric 1 foo=bar source=\"test.host.com\"", out.get(0)); - } - - @Ignore - @Test - public void testBenchmark() throws Exception { - List customSourceTags = new ArrayList(); - customSourceTags.add("fqdn"); - customSourceTags.add("hostname"); - customSourceTags.add("highcardinalitytag1"); - customSourceTags.add("highcardinalitytag2"); - customSourceTags.add("highcardinalitytag3"); - customSourceTags.add("highcardinalitytag4"); - customSourceTags.add("highcardinalitytag5"); - - GraphiteHostAnnotator handler = new GraphiteHostAnnotator("test.host.com", customSourceTags); - int ITERATIONS = 1000000; - - for (int i = 0; i < ITERATIONS / 1000; i++) { - List out = new ArrayList<>(); - handler.decode(null, "tsdb.vehicle.charge.battery_level 93 123456 host=vehicle_2554", out); - } - long start = System.currentTimeMillis(); - for (int i = 0; i < ITERATIONS; i++) { - List out = new ArrayList<>(); - handler.decode(null, "tsdb.vehicle.charge.battery_level 93 123456 host=vehicle_2554", out); - } - double end = System.currentTimeMillis(); - System.out.println(ITERATIONS / ((end - start) / 1000) + " DPS"); - } -} diff --git a/java-lib/src/test/java/com/wavefront/ingester/HistogramDecoderTest.java b/java-lib/src/test/java/com/wavefront/ingester/HistogramDecoderTest.java deleted file mode 100644 index ff63cc71d..000000000 --- a/java-lib/src/test/java/com/wavefront/ingester/HistogramDecoderTest.java +++ /dev/null @@ -1,311 +0,0 @@ -package com.wavefront.ingester; - -import com.wavefront.common.Clock; - -import org.apache.commons.lang.time.DateUtils; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import wavefront.report.Histogram; -import wavefront.report.ReportPoint; - -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertNotNull; - -/** - * @author Tim Schmidt (tim@wavefront.com). - */ -public class HistogramDecoderTest { - - @Test - public void testBasicMessage() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M 1471988653 #3 123.237 TestMetric source=Test key=value", out, "customer"); - - assertThat(out).isNotEmpty(); - ReportPoint p = out.get(0); - assertThat(p.getMetric()).isEqualTo("TestMetric"); - // Should be converted to Millis and pinned to the beginning of the corresponding minute - assertThat(p.getTimestamp()).isEqualTo(1471988640000L); - assertThat(p.getValue()).isNotNull(); - assertThat(p.getValue().getClass()).isEqualTo(Histogram.class); - - assertThat(p.getHost()).isEqualTo("Test"); - assertThat(p.getTable()).isEqualTo("customer"); - assertThat(p.getAnnotations()).isNotNull(); - assertThat(p.getAnnotations()).containsEntry("key", "value"); - - Histogram h = (Histogram) p.getValue(); - - assertThat(h.getDuration()).isEqualTo(DateUtils.MILLIS_PER_MINUTE); - assertThat(h.getBins()).isNotNull(); - assertThat(h.getBins()).isNotEmpty(); - assertThat(h.getBins()).containsExactly(123.237D); - assertThat(h.getCounts()).isNotNull(); - assertThat(h.getCounts()).isNotEmpty(); - assertThat(h.getCounts()).containsExactly(3); - } - - - @Test - public void testHourBin() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!H 1471988653 #3 123.237 TestMetric source=Test key=value", out, "customer"); - - assertThat(out).isNotEmpty(); - ReportPoint p = out.get(0); - // Should be converted to Millis and pinned to the beginning of the corresponding hour - assertThat(p.getTimestamp()).isEqualTo(1471986000000L); - assertThat(p.getValue()).isNotNull(); - assertThat(p.getValue().getClass()).isEqualTo(Histogram.class); - - Histogram h = (Histogram) p.getValue(); - - assertThat(h.getDuration()).isEqualTo(DateUtils.MILLIS_PER_HOUR); - } - - @Test - public void testDayBin() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!D 1471988653 #3 123.237 TestMetric source=Test key=value", out, "customer"); - - assertThat(out).isNotEmpty(); - ReportPoint p = out.get(0); - // Should be converted to Millis and pinned to the beginning of the corresponding day - assertThat(p.getTimestamp()).isEqualTo(1471910400000L); - assertThat(p.getValue()).isNotNull(); - assertThat(p.getValue().getClass()).isEqualTo(Histogram.class); - - Histogram h = (Histogram) p.getValue(); - - assertThat(h.getDuration()).isEqualTo(DateUtils.MILLIS_PER_DAY); - } - - @Test - public void testTagKey() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M 1471988653 #3 123.237 TestMetric source=Test tag=value", out, "customer"); - - assertThat(out).isNotEmpty(); - ReportPoint p = out.get(0); - - assertThat(p.getAnnotations()).isNotNull(); - assertThat(p.getAnnotations()).containsEntry("_tag", "value"); - } - - @Test - public void testMultipleBuckets() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M 1471988653 #1 3.1416 #1 2.7183 TestMetric", out, "customer"); - - assertThat(out).isNotEmpty(); - ReportPoint p = out.get(0); - assertThat(p.getValue()).isNotNull(); - assertThat(p.getValue().getClass()).isEqualTo(Histogram.class); - - Histogram h = (Histogram) p.getValue(); - - assertThat(h.getDuration()).isEqualTo(DateUtils.MILLIS_PER_MINUTE); - assertThat(h.getBins()).isNotNull(); - assertThat(h.getBins()).isNotEmpty(); - assertThat(h.getBins()).containsExactly(3.1416D, 2.7183D); - assertThat(h.getCounts()).isNotNull(); - assertThat(h.getCounts()).isNotEmpty(); - assertThat(h.getCounts()).containsExactly(1, 1); - } - - @Test - public void testNegativeMean() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M 1471988653 #1 -3.1416 TestMetric", out, "customer"); - - assertThat(out).isNotEmpty(); - ReportPoint p = out.get(0); - assertThat(p.getValue()).isNotNull(); - assertThat(p.getValue().getClass()).isEqualTo(Histogram.class); - - Histogram h = (Histogram) p.getValue(); - - assertThat(h.getDuration()).isEqualTo(DateUtils.MILLIS_PER_MINUTE); - assertThat(h.getBins()).isNotNull(); - assertThat(h.getBins()).isNotEmpty(); - assertThat(h.getBins()).containsExactly(-3.1416D); - assertThat(h.getCounts()).isNotNull(); - assertThat(h.getCounts()).isNotEmpty(); - assertThat(h.getCounts()).containsExactly(1); - } - - @Test(expected = RuntimeException.class) - public void testMissingBin() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("1471988653 #3 123.237 TestMetric source=Test tag=value", out, "customer"); - } - - @Test - public void testMissingTimestamp() { - //Note - missingTimestamp to port 40,000 is no longer invalid - see MONIT-6430 for more details - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M #3 123.237 TestMetric source=Test tag=value", out, "customer"); - - assertThat(out).isNotEmpty(); - long expectedTimestamp = Clock.now(); - - ReportPoint p = out.get(0); - assertThat(p.getMetric()).isEqualTo("TestMetric"); - - assertThat(p.getValue()).isNotNull(); - assertThat(p.getValue().getClass()).isEqualTo(Histogram.class); - - // Should be converted to Millis and pinned to the beginning of the corresponding minute - long duration = ((Histogram) p.getValue()).getDuration(); - expectedTimestamp = (expectedTimestamp / duration) * duration; - assertThat(p.getTimestamp()).isEqualTo(expectedTimestamp); - - assertThat(p.getHost()).isEqualTo("Test"); - assertThat(p.getTable()).isEqualTo("customer"); - assertThat(p.getAnnotations()).isNotNull(); - assertThat(p.getAnnotations()).containsEntry("_tag", "value"); - - Histogram h = (Histogram) p.getValue(); - - assertThat(h.getDuration()).isEqualTo(DateUtils.MILLIS_PER_MINUTE); - assertThat(h.getBins()).isNotNull(); - assertThat(h.getBins()).isNotEmpty(); - assertThat(h.getBins()).containsExactly(123.237D); - assertThat(h.getCounts()).isNotNull(); - assertThat(h.getCounts()).isNotEmpty(); - assertThat(h.getCounts()).containsExactly(3); - } - - @Test(expected = RuntimeException.class) - public void testMissingCentroids() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M 1471988653 TestMetric source=Test tag=value", out, "customer"); - } - - @Test - public void testMissingMean() { - //Note - missingTimestamp to port 40,000 is no longer invalid - see MONIT-6430 for more details - // as a side-effect of that, this test no longer fails!!! - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M #3 1471988653 TestMetric source=Test tag=value", out, "customer"); - - assertThat(out).isNotEmpty(); - long expectedTimestamp = Clock.now(); - - ReportPoint p = out.get(0); - assertThat(p.getMetric()).isEqualTo("TestMetric"); - - assertThat(p.getValue()).isNotNull(); - assertThat(p.getValue().getClass()).isEqualTo(Histogram.class); - - // Should be converted to Millis and pinned to the beginning of the corresponding minute - long duration = ((Histogram) p.getValue()).getDuration(); - expectedTimestamp = (expectedTimestamp / duration) * duration; - assertThat(p.getTimestamp()).isEqualTo(expectedTimestamp); - - assertThat(p.getHost()).isEqualTo("Test"); - assertThat(p.getTable()).isEqualTo("customer"); - assertThat(p.getAnnotations()).isNotNull(); - assertThat(p.getAnnotations()).containsEntry("_tag", "value"); - - Histogram h = (Histogram) p.getValue(); - - assertThat(h.getDuration()).isEqualTo(DateUtils.MILLIS_PER_MINUTE); - assertThat(h.getBins()).isNotNull(); - assertThat(h.getBins()).isNotEmpty(); - assertThat(h.getBins()).containsExactly(1471988653.0); - assertThat(h.getCounts()).isNotNull(); - assertThat(h.getCounts()).isNotEmpty(); - assertThat(h.getCounts()).containsExactly(3); - } - - @Test(expected = RuntimeException.class) - public void testMissingCount() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M 3.412 1471988653 TestMetric source=Test tag=value", out, "customer"); - } - - @Test(expected = RuntimeException.class) - public void testZeroCount() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M #0 3.412 1471988653 TestMetric source=Test tag=value", out, "customer"); - } - - @Test(expected = RuntimeException.class) - public void testMissingMetric() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("1471988653 #3 123.237 source=Test tag=value", out, "customer"); - } - - @Test(expected = RuntimeException.class) - public void testMissingCentroid() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M #1 TestMetric source=Test", out, "customer"); - } - - @Test(expected = RuntimeException.class) - public void testMissingCentroid2() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M #1 12345 #2 TestMetric source=Test", out, "customer"); - } - - @Test(expected = RuntimeException.class) - public void testTooManyCentroids() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M #1 12345 #2 123453 1234534 12334 TestMetric source=Test", out, "customer"); - } - - @Test(expected = RuntimeException.class) - public void testNoCentroids() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M 12334 TestMetric source=Test", out, "customer"); - } - - @Test - public void testNoSourceAnnotationsIsNotNull() { - HistogramDecoder decoder = new HistogramDecoder(); - List out = new ArrayList<>(); - - decoder.decodeReportPoints("!M #1 12334 TestMetric", out, "customer"); - - ReportPoint p = out.get(0); - assertNotNull(p.getAnnotations()); - } -} diff --git a/java-lib/src/test/java/com/wavefront/ingester/OpenTSDBDecoderTest.java b/java-lib/src/test/java/com/wavefront/ingester/OpenTSDBDecoderTest.java deleted file mode 100644 index 9141aabb5..000000000 --- a/java-lib/src/test/java/com/wavefront/ingester/OpenTSDBDecoderTest.java +++ /dev/null @@ -1,110 +0,0 @@ -package com.wavefront.ingester; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import wavefront.report.ReportPoint; - -import static junit.framework.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * Tests for OpenTSDBDecoder. - * - * @author Clement Pang (clement@wavefront.com). - */ -public class OpenTSDBDecoderTest { - - @Test - public void testDoubleFormat() throws Exception { - List customSourceTags = new ArrayList(); - customSourceTags.add("fqdn"); - OpenTSDBDecoder decoder = new OpenTSDBDecoder("localhost", customSourceTags); - List out = new ArrayList<>(); - decoder.decodeReportPoints("put tsdb.vehicle.charge.battery_level 12345.678 93.123e3 host=vehicle_2554", out); - ReportPoint point = out.get(0); - assertEquals("dummy", point.getTable()); - assertEquals("tsdb.vehicle.charge.battery_level", point.getMetric()); - assertEquals(93123.0, point.getValue()); - assertEquals(12345678L, point.getTimestamp().longValue()); - assertEquals("vehicle_2554", point.getHost()); - - try { - // need "PUT" - decoder.decodeReportPoints("tsdb.vehicle.charge.battery_level 12345.678 93.123e3 host=vehicle_2554", out); - fail(); - } catch (Exception ex) { - } - - try { - // need "timestamp" - decoder.decodeReportPoints("put tsdb.vehicle.charge.battery_level 93.123e3 host=vehicle_2554", out); - fail(); - } catch (Exception ex) { - } - - try { - // need "value" - decoder.decodeReportPoints("put tsdb.vehicle.charge.battery_level 12345.678 host=vehicle_2554", out); - fail(); - } catch (Exception ex) { - } - - out = new ArrayList<>(); - decoder.decodeReportPoints("put tsdb.vehicle.charge.battery_level 12345.678 93.123e3", out); - point = out.get(0); - assertEquals("dummy", point.getTable()); - assertEquals("tsdb.vehicle.charge.battery_level", point.getMetric()); - assertEquals(93123.0, point.getValue()); - assertEquals(12345678L, point.getTimestamp().longValue()); - assertEquals("localhost", point.getHost()); - - // adaptive timestamp (13-char timestamp is millis). - out = new ArrayList<>(); - final long now = System.currentTimeMillis(); - decoder.decodeReportPoints("put tsdb.vehicle.charge.battery_level " + now - + " 93.123e3", out); - point = out.get(0); - assertEquals("dummy", point.getTable()); - assertEquals("tsdb.vehicle.charge.battery_level", point.getMetric()); - assertEquals(93123.0, point.getValue()); - assertEquals(now, point.getTimestamp().longValue()); - assertEquals("localhost", point.getHost()); - - out = new ArrayList<>(); - decoder.decodeReportPoints("put tail.kernel.counter.errors 1447394143 0 fqdn=li250-160.members.linode.com ", out); - point = out.get(0); - assertEquals("dummy", point.getTable()); - assertEquals("tail.kernel.counter.errors", point.getMetric()); - assertEquals(0.0, point.getValue()); - assertEquals(1447394143000L, point.getTimestamp().longValue()); - assertEquals("li250-160.members.linode.com", point.getHost()); - - out = new ArrayList<>(); - decoder.decodeReportPoints("put df.home-ubuntu-efs.df_complex.free 1447985300 9.22337186120781e+18 fqdn=ip-172-20-0-236.us-west-2.compute.internal ", out); - point = out.get(0); - assertEquals("dummy", point.getTable()); - assertEquals("df.home-ubuntu-efs.df_complex.free", point.getMetric()); - assertEquals(9.22337186120781e+18, point.getValue()); - assertEquals(1447985300000L, point.getTimestamp().longValue()); - assertEquals("ip-172-20-0-236.us-west-2.compute.internal", point.getHost()); - } - - @Test - public void testOpenTSDBCharacters() { - List customSourceTags = new ArrayList<>(); - customSourceTags.add("fqdn"); - OpenTSDBDecoder decoder = new OpenTSDBDecoder("localhost", customSourceTags); - List out = new ArrayList<>(); - decoder.decodeReportPoints("put tsdb.vehicle.charge.battery_level 12345.678 93.123e3 host=/vehicle_2554-test/GOOD some_tag=/vehicle_2554-test/BAD", out); - ReportPoint point = out.get(0); - assertEquals("dummy", point.getTable()); - assertEquals("tsdb.vehicle.charge.battery_level", point.getMetric()); - assertEquals(93123.0, point.getValue()); - assertEquals(12345678L, point.getTimestamp().longValue()); - assertEquals("/vehicle_2554-test/GOOD", point.getHost()); - assertEquals("/vehicle_2554-test/BAD", point.getAnnotations().get("some_tag")); - } -} diff --git a/java-lib/src/test/java/com/wavefront/ingester/ReportPointSerializerTest.java b/java-lib/src/test/java/com/wavefront/ingester/ReportPointSerializerTest.java deleted file mode 100644 index dc930f883..000000000 --- a/java-lib/src/test/java/com/wavefront/ingester/ReportPointSerializerTest.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; - -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.time.DateUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.HashMap; -import java.util.function.Function; - -import wavefront.report.Histogram; -import wavefront.report.HistogramType; -import wavefront.report.ReportPoint; - -import static com.google.common.truth.Truth.assertThat; - -/** - * @author Andrew Kao (andrew@wavefront.com), Jason Bau (jbau@wavefront.com), vasily@wavefront.com - */ -public class ReportPointSerializerTest { - private ReportPoint histogramPoint; - - private Function serializer = new ReportPointSerializer(); - - @Before - public void setUp() { - Histogram h = Histogram.newBuilder() - .setType(HistogramType.TDIGEST) - .setBins(ImmutableList.of(10D, 20D)) - .setCounts(ImmutableList.of(2, 4)) - .setDuration((int) DateUtils.MILLIS_PER_MINUTE) - .build(); - histogramPoint = ReportPoint.newBuilder() - .setTable("customer") - .setValue(h) - .setMetric("TestMetric") - .setHost("TestSource") - .setTimestamp(1469751813000L) - .setAnnotations(ImmutableMap.of("keyA", "valueA", "keyB", "valueB")) - .build(); - } - - @Test - public void testReportPointToString() { - // Common case. - Assert.assertEquals("\"some metric\" 10 1469751813 source=\"host\" \"foo\"=\"bar\" \"boo\"=\"baz\"", - serializer.apply(new ReportPoint("some metric",1469751813000L, 10L, "host", "table", - ImmutableMap.of("foo", "bar", "boo", "baz")))); - Assert.assertEquals("\"some metric\" 10 1469751813 source=\"host\"", - serializer.apply(new ReportPoint("some metric",1469751813000L, 10L, "host", "table", - ImmutableMap.of()))); - - // Quote in metric name - Assert.assertEquals("\"some\\\"metric\" 10 1469751813 source=\"host\"", - serializer.apply(new ReportPoint("some\"metric", 1469751813000L, 10L, "host", "table", - new HashMap())) - ); - // Quote in tags - Assert.assertEquals("\"some metric\" 10 1469751813 source=\"host\" \"foo\\\"\"=\"\\\"bar\" \"bo\\\"o\"=\"baz\"", - serializer.apply(new ReportPoint("some metric", 1469751813000L, 10L, "host", "table", - ImmutableMap.of("foo\"", "\"bar", "bo\"o", "baz"))) - ); - } - - @Test - public void testReportPointToString_stringValue() { - histogramPoint.setValue("Test"); - - String subject = serializer.apply(histogramPoint); - assertThat(subject).isEqualTo("\"TestMetric\" Test 1469751813 source=\"TestSource\" \"keyA\"=\"valueA\" \"keyB\"=\"valueB\""); - } - - - @Test - public void testHistogramReportPointToString() { - String subject = serializer.apply(histogramPoint); - - assertThat(subject).isEqualTo("!M 1469751813 #2 10.0 #4 20.0 \"TestMetric\" source=\"TestSource\" \"keyA\"=\"valueA\" \"keyB\"=\"valueB\""); - } - - @Test(expected = RuntimeException.class) - public void testHistogramReportPointToString_unsupportedDuration() { - ((Histogram)histogramPoint.getValue()).setDuration(13); - - serializer.apply(histogramPoint); - } - - @Test - public void testHistogramReportPointToString_binCountMismatch() { - ((Histogram)histogramPoint.getValue()).setCounts(ImmutableList.of(10)); - - String subject = serializer.apply(histogramPoint); - assertThat(subject).isEqualTo("!M 1469751813 #10 10.0 \"TestMetric\" source=\"TestSource\" \"keyA\"=\"valueA\" \"keyB\"=\"valueB\""); - } - - @Test - public void testHistogramReportPointToString_quotesInMetric() { - histogramPoint.setMetric("Test\"Metric"); - - String subject = serializer.apply(histogramPoint); - assertThat(subject).isEqualTo("!M 1469751813 #2 10.0 #4 20.0 \"Test\\\"Metric\" source=\"TestSource\" \"keyA\"=\"valueA\" \"keyB\"=\"valueB\""); - } - - @Test - public void testHistogramReportPointToString_quotesInTags() { - histogramPoint.setAnnotations(ImmutableMap.of("K\"ey", "V\"alue")); - - String subject = serializer.apply(histogramPoint); - assertThat(subject).isEqualTo("!M 1469751813 #2 10.0 #4 20.0 \"TestMetric\" source=\"TestSource\" \"K\\\"ey\"=\"V\\\"alue\""); - } - - @Test(expected = RuntimeException.class) - public void testHistogramReportPointToString_BadValue() { - ReportPoint p = new ReportPoint("m", 1469751813L, new ArrayUtils(), "h", "c", ImmutableMap.of()); - - serializer.apply(p); - } -} diff --git a/java-lib/src/test/java/com/wavefront/ingester/ReportSourceTagDecoderTest.java b/java-lib/src/test/java/com/wavefront/ingester/ReportSourceTagDecoderTest.java deleted file mode 100644 index 30c4cf82d..000000000 --- a/java-lib/src/test/java/com/wavefront/ingester/ReportSourceTagDecoderTest.java +++ /dev/null @@ -1,176 +0,0 @@ -package com.wavefront.ingester; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; - -import wavefront.report.ReportSourceTag; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * This class is used to test the source tag points - * - * @author Suranjan Pramanik (suranjan@wavefront.com). - */ -public class ReportSourceTagDecoderTest { - private static final Logger logger = Logger.getLogger(ReportSourceTagDecoderTest.class - .getCanonicalName()); - - private static final String SOURCE_TAG = "SourceTag"; - private static final String SOURCE_DESCRIPTION = "SourceDescription"; - - @Test - public void testSimpleSourceTagFormat() throws Exception { - ReportSourceTagDecoder decoder = new ReportSourceTagDecoder(); - List out = new ArrayList<>(); - // Testwith 3sourceTags - decoder.decode(String.format("@%s %s=%s %s=aSource sourceTag1 sourceTag2 " + - "sourceTag3", SOURCE_TAG, - ReportSourceTagIngesterFormatter.ACTION, ReportSourceTagIngesterFormatter.ACTION_SAVE, - ReportSourceTagIngesterFormatter.SOURCE), out); - ReportSourceTag reportSourceTag = out.get(0); - assertEquals("Action name didn't match.", ReportSourceTagIngesterFormatter.ACTION_SAVE, - reportSourceTag.getAction()); - assertEquals("Source did not match.", "aSource", reportSourceTag.getSource()); - assertTrue("SourceTag1 did not match.", reportSourceTag.getAnnotations().contains - ("sourceTag1")); - - // Test with one sourceTag - out.clear(); - decoder.decode(String.format("@%s action = %s source = \"A Source\" sourceTag3", - SOURCE_TAG, ReportSourceTagIngesterFormatter.ACTION_SAVE), out); - reportSourceTag = out.get(0); - assertEquals("Action name didn't match.", ReportSourceTagIngesterFormatter.ACTION_SAVE, - reportSourceTag.getAction()); - assertEquals("Source did not match.", "A Source", reportSourceTag.getSource()); - assertTrue("SourceTag3 did not match.", reportSourceTag.getAnnotations() - .contains("sourceTag3")); - - // Test with a multi-word source tag - out.clear(); - decoder.decode(String.format("@%s action = %s source=aSource \"A source tag\" " + - "\"Another tag\"", SOURCE_TAG, ReportSourceTagIngesterFormatter.ACTION_SAVE), out); - reportSourceTag = out.get(0); - assertEquals("Action name didn't match.", ReportSourceTagIngesterFormatter.ACTION_SAVE, - reportSourceTag.getAction()); - assertEquals("Source did not match.", "aSource", reportSourceTag.getSource()); - assertTrue("'A source tag' did not match.", reportSourceTag.getAnnotations() - .contains("A source tag")); - assertTrue("'Another tag' did not match", reportSourceTag.getAnnotations() - .contains("Another tag")); - - // Test sourceTag without any action -- this should result in an exception - String msg = String.format("@%s source=aSource sourceTag4 sourceTag5", SOURCE_TAG); - out.clear(); - boolean isException = false; - try { - decoder.decode(msg, out); - } catch (Exception ex) { - isException = true; - logger.info(ex.getMessage()); - } - assertTrue("Did not see an exception for SourceTag message without an action for input : " + - msg, isException); - - // Test sourceTag without any source -- this should result in an exception - msg = String.format("@%s action=save description=desc sourceTag5", SOURCE_TAG); - out.clear(); - isException = false; - try { - decoder.decode(msg, out); - } catch (Exception ex) { - isException = true; - logger.info(ex.getMessage()); - } - assertTrue("Did not see an exception for SourceTag message without a source for input : " + - msg, isException); - - // Test sourceTag with action, source, and description -- this should result in an exception - out.clear(); - msg = String.format("@%s action = save source = aSource description = desc sourceTag5", - SOURCE_TAG); - isException = false; - try { - decoder.decode(msg, out); - } catch (Exception ex) { - isException = true; - logger.info(ex.getMessage()); - } - assertTrue("Did not see an exception when description was present in SourceTag message for " + - "input : " + msg, isException); - - // Test sourceTag message without the source field -- should throw an exception - out.clear(); - isException = false; - msg = "@SourceTag action=remove sourceTag3"; - try { - decoder.decode(msg, out); - } catch (Exception ex) { - isException = true; - logger.info(ex.getMessage()); - } - assertTrue("Did not see an exception when source field was absent for input: " + msg, - isException); - - // Test a message where action is not save or delete -- should throw an exception - out.clear(); - isException = false; - msg = String.format("@%s action = anAction source=aSource sourceTag2 sourceTag3", SOURCE_TAG); - try { - decoder.decode(msg, out); - } catch (Exception ex) { - isException = true; - logger.info(ex.getMessage()); - } - assertTrue("Did not see an exception when action field was invalid for input : " + msg, - isException); - } - - @Test - public void testSimpleSourceDescriptions() throws Exception { - ReportSourceTagDecoder decoder = new ReportSourceTagDecoder(); - List out = new ArrayList<>(); - // Testwith source description - decoder.decode(String.format("@%s %s=%s %s= aSource description=desc", - SOURCE_DESCRIPTION, - ReportSourceTagIngesterFormatter.ACTION, ReportSourceTagIngesterFormatter.ACTION_SAVE, - ReportSourceTagIngesterFormatter.SOURCE), out); - ReportSourceTag reportSourceTag = out.get(0); - assertEquals("Action name didn't match.", ReportSourceTagIngesterFormatter.ACTION_SAVE, - reportSourceTag.getAction()); - assertEquals("Source did not match.", "aSource", reportSourceTag.getSource()); - assertEquals("Description did not match.", "desc", reportSourceTag.getDescription()); - - // Test delete action where description field is not necessary - out.clear(); - String format = String.format("@%s %s=%s %s=aSource", SOURCE_DESCRIPTION, - ReportSourceTagIngesterFormatter.ACTION, ReportSourceTagIngesterFormatter.ACTION_DELETE, - ReportSourceTagIngesterFormatter.SOURCE); - decoder.decode(format, out); - reportSourceTag = out.get(0); - assertEquals("Action name did not match for input : " + format, ReportSourceTagIngesterFormatter - .ACTION_DELETE, - reportSourceTag.getAction()); - assertEquals("Source did not match for input : " + format, "aSource", reportSourceTag - .getSource()); - - // Add a source tag to the SourceDescription message -- this should cause an exception - out.clear(); - String msg = String.format("@%s action = save source = aSource description = desc " + - "sourceTag4", SOURCE_DESCRIPTION); - boolean isException = false; - try { - decoder.decode(msg, out); - } catch (Exception ex) { - isException = true; - logger.info(ex.getMessage()); - } - assertTrue("Expected an exception, since source tag was supplied in SourceDescription " + - "message for input : " + msg, isException); - - } -} diff --git a/java-lib/src/test/java/com/wavefront/ingester/SpanDecoderTest.java b/java-lib/src/test/java/com/wavefront/ingester/SpanDecoderTest.java deleted file mode 100644 index 72cbed084..000000000 --- a/java-lib/src/test/java/com/wavefront/ingester/SpanDecoderTest.java +++ /dev/null @@ -1,240 +0,0 @@ -package com.wavefront.ingester; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import wavefront.report.Span; - -import static org.junit.Assert.assertEquals; - -/** - * Tests for SpanDecoder - * - * @author vasily@wavefront.com - */ -public class SpanDecoderTest { - - // Decoder is supposed to convert all timestamps to this unit. Should we decide to change the timestamp precision - // in the Span object, this is the only change required to keep unit tests valid. - private static TimeUnit expectedTimeUnit = TimeUnit.MILLISECONDS; - - private static long startTs = 1532012145123L; - private static long duration = 1111; - - private SpanDecoder decoder = new SpanDecoder("unitTest"); - - @Test(expected = RuntimeException.class) - public void testSpanWithoutDurationThrows() { - List out = new ArrayList<>(); - decoder.decode("testSpanName source=spanSource spanId=spanId traceId=traceId tagkey1=tagvalue1 1532012145123456 ", - out); - Assert.fail("Missing duration didn't raise an exception"); - } - - @Test(expected = RuntimeException.class) - public void testSpanWithExtraTagsThrows() { - List out = new ArrayList<>(); - decoder.decode("testSpanName source=spanSource spanId=spanId traceId=traceId tagkey1=tagvalue1 1532012145123456 " + - "1532012146234567 extraTag=extraValue", out); - Assert.fail("Extra tags after timestamps didn't raise an exception"); - } - - @Test(expected = RuntimeException.class) - public void testSpanWithoutSpanIdThrows() { - List out = new ArrayList<>(); - decoder.decode("testSpanName source=spanSource traceId=traceId tagkey1=tagvalue1 1532012145123456 " + - "1532012146234567", out); - Assert.fail("Missing spanId didn't raise an exception"); - } - - @Test(expected = RuntimeException.class) - public void testSpanWithoutTraceIdThrows() { - List out = new ArrayList<>(); - decoder.decode("testSpanName source=spanSource spanId=spanId tagkey1=tagvalue1 1532012145123456 " + - "1532012146234567", out); - Assert.fail("Missing traceId didn't raise an exception"); - } - - @Test - public void testSpanUsesDefaultSource() { - List out = new ArrayList<>(); - decoder.decode("testSpanName spanId=4217104a-690d-4927-baff-d9aa779414c2 " + - "traceId=d5355bf7-fc8d-48d1-b761-75b170f396e0 tagkey1=tagvalue1 t2=v2 1532012145123456 1532012146234567 ", - out); - assertEquals(1, out.size()); - assertEquals("unitTest", out.get(0).getSource()); - } - - @Test - public void testSpanWithUnquotedUuids() { - List out = new ArrayList<>(); - decoder.decode("testSpanName source=spanSource spanId=4217104a-690d-4927-baff-d9aa779414c2 " + - "traceId=d5355bf7-fc8d-48d1-b761-75b170f396e0 tagkey1=tagvalue1 t2=v2 1532012145123456 1532012146234567 ", - out); - assertEquals(1, out.size()); - assertEquals("4217104a-690d-4927-baff-d9aa779414c2", out.get(0).getSpanId()); - assertEquals("d5355bf7-fc8d-48d1-b761-75b170f396e0", out.get(0).getTraceId()); - } - - @Test - public void testSpanWithQuotedUuids() { - List out = new ArrayList<>(); - decoder.decode("testSpanName source=spanSource spanId=\"4217104a-690d-4927-baff-d9aa779414c2\" " + - "traceId=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" tagkey1=tagvalue1 t2=v2 1532012145123456 1532012146234567 ", - out); - assertEquals(1, out.size()); - assertEquals("4217104a-690d-4927-baff-d9aa779414c2", out.get(0).getSpanId()); - assertEquals("d5355bf7-fc8d-48d1-b761-75b170f396e0", out.get(0).getTraceId()); - } - - @Test - public void testSpanWithQuotesInTags() { - List out = new ArrayList<>(); - decoder.decode("testSpanName source=spanSource spanId=\"4217104a-690d-4927-baff-d9aa779414c2\" " + - "traceId=\"traceid\" tagkey1=\"tag\\\"value\\\"1\" t2=v2 1532012145123456 1532012146234567 ", - out); - assertEquals(1, out.size()); - assertEquals(2, out.get(0).getAnnotations().size()); - assertEquals("tagkey1", out.get(0).getAnnotations().get(0).getKey()); - assertEquals("tag\"value\"1", out.get(0).getAnnotations().get(0).getValue()); - } - - @Test - public void testSpanWithRepeatableAnnotations() { - List out = new ArrayList<>(); - decoder.decode("testSpanName source=spanSource spanId=spanid traceId=traceid parent=parentid1 tag1=v1 " + - "parent=parentid2 1532012145123456 1532012146234567 ", - out); - assertEquals(1, out.size()); - assertEquals(3, out.get(0).getAnnotations().size()); - assertEquals("parent", out.get(0).getAnnotations().get(0).getKey()); - assertEquals("parentid1", out.get(0).getAnnotations().get(0).getValue()); - assertEquals("parent", out.get(0).getAnnotations().get(2).getKey()); - assertEquals("parentid2", out.get(0).getAnnotations().get(2).getValue()); - } - - @Test - public void testBasicSpanParse() { - List out = new ArrayList<>(); - decoder.decode("testSpanName source=spanSource spanId=4217104a-690d-4927-baff-d9aa779414c2 " + - "traceId=d5355bf7-fc8d-48d1-b761-75b170f396e0 tagkey1=tagvalue1 " + - "t2=v2 1532012145123 1532012146234", out); - assertEquals(1, out.size()); - assertEquals("testSpanName", out.get(0).getName()); - assertEquals("spanSource", out.get(0).getSource()); - assertEquals("4217104a-690d-4927-baff-d9aa779414c2", out.get(0).getSpanId()); - assertEquals("d5355bf7-fc8d-48d1-b761-75b170f396e0", out.get(0).getTraceId()); - assertEquals(2, out.get(0).getAnnotations().size()); - assertEquals("tagkey1", out.get(0).getAnnotations().get(0).getKey()); - assertEquals("tagvalue1", out.get(0).getAnnotations().get(0).getValue()); - assertEquals("t2", out.get(0).getAnnotations().get(1).getKey()); - assertEquals("v2", out.get(0).getAnnotations().get(1).getValue()); - assertEquals(expectedTimeUnit.convert(startTs, TimeUnit.MILLISECONDS), - (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration, TimeUnit.MILLISECONDS), - (long) out.get(0).getDuration()); - - out.clear(); - // test that annotations order doesn't matter - decoder.decode("testSpanName tagkey1=tagvalue1 traceId=d5355bf7-fc8d-48d1-b761-75b170f396e0 " + - "spanId=4217104a-690d-4927-baff-d9aa779414c2 source=spanSource " + - "t2=v2 1532012145123 1532012146234 ", out); - assertEquals(1, out.size()); - assertEquals("testSpanName", out.get(0).getName()); - assertEquals("spanSource", out.get(0).getSource()); - assertEquals("4217104a-690d-4927-baff-d9aa779414c2", out.get(0).getSpanId()); - assertEquals("d5355bf7-fc8d-48d1-b761-75b170f396e0", out.get(0).getTraceId()); - assertEquals(2, out.get(0).getAnnotations().size()); - assertEquals("tagkey1", out.get(0).getAnnotations().get(0).getKey()); - assertEquals("tagvalue1", out.get(0).getAnnotations().get(0).getValue()); - assertEquals("t2", out.get(0).getAnnotations().get(1).getKey()); - assertEquals("v2", out.get(0).getAnnotations().get(1).getValue()); - assertEquals(expectedTimeUnit.convert(startTs, TimeUnit.MILLISECONDS), (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration, TimeUnit.MILLISECONDS), (long) out.get(0).getDuration()); - - } - - @Test - public void testQuotedSpanParse() { - List out = new ArrayList<>(); - decoder.decode("\"testSpanName\" \"source\"=\"spanSource\" \"spanId\"=\"4217104a-690d-4927-baff-d9aa779414c2\" " + - "\"traceId\"=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" \"tagkey1\"=\"tagvalue1\" \"t2\"=\"v2\" " + - "1532012145123 1532012146234", out); - assertEquals(1, out.size()); - assertEquals("testSpanName", out.get(0).getName()); - assertEquals("spanSource", out.get(0).getSource()); - assertEquals("4217104a-690d-4927-baff-d9aa779414c2", out.get(0).getSpanId()); - assertEquals("d5355bf7-fc8d-48d1-b761-75b170f396e0", out.get(0).getTraceId()); - assertEquals(2, out.get(0).getAnnotations().size()); - assertEquals("tagkey1", out.get(0).getAnnotations().get(0).getKey()); - assertEquals("tagvalue1", out.get(0).getAnnotations().get(0).getValue()); - assertEquals("t2", out.get(0).getAnnotations().get(1).getKey()); - assertEquals("v2", out.get(0).getAnnotations().get(1).getValue()); - assertEquals(expectedTimeUnit.convert(startTs, TimeUnit.MILLISECONDS), (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration, TimeUnit.MILLISECONDS), (long) out.get(0).getDuration()); - } - - @Test - public void testSpanTimestampFormats() { - List out = new ArrayList<>(); - // nanoseconds with end_ts - decoder.decode("testSpanName source=spanSource spanId=spanid traceId=traceid 1532012145123456 1532012146234567", - out); - assertEquals(expectedTimeUnit.convert(startTs, TimeUnit.MILLISECONDS), (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration, TimeUnit.MILLISECONDS), (long) out.get(0).getDuration()); - out.clear(); - - // nanoseconds with duration - decoder.decode("testSpanName source=spanSource spanId=spanid traceId=traceid 1532012145123456789 1111111111", - out); - assertEquals(expectedTimeUnit.convert(startTs, TimeUnit.MILLISECONDS), (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration, TimeUnit.MILLISECONDS), (long) out.get(0).getDuration()); - out.clear(); - - // microseconds with end_ts - decoder.decode("testSpanName source=spanSource spanId=spanid traceId=traceid 1532012145123456 1532012146234567 ", - out); - assertEquals(expectedTimeUnit.convert(startTs, TimeUnit.MILLISECONDS), (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration, TimeUnit.MILLISECONDS), (long) out.get(0).getDuration()); - out.clear(); - - // microseconds with duration - decoder.decode("testSpanName source=spanSource spanId=spanid traceId=traceid 1532012145123456 1111111 ", - out); - assertEquals(expectedTimeUnit.convert(startTs, TimeUnit.MILLISECONDS), (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration, TimeUnit.MILLISECONDS), (long) out.get(0).getDuration()); - out.clear(); - - // milliseconds with end_ts - decoder.decode("testSpanName source=spanSource spanId=spanid traceId=traceid 1532012145123 1532012146234", - out); - assertEquals(expectedTimeUnit.convert(startTs, TimeUnit.MILLISECONDS), (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration, TimeUnit.MILLISECONDS), (long) out.get(0).getDuration()); - out.clear(); - - // milliseconds with duration - decoder.decode("testSpanName source=spanSource spanId=spanid traceId=traceid 1532012145123 1111", - out); - assertEquals(expectedTimeUnit.convert(startTs, TimeUnit.MILLISECONDS), (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration, TimeUnit.MILLISECONDS), (long) out.get(0).getDuration()); - out.clear(); - - // seconds with end_ts - decoder.decode("testSpanName source=spanSource spanId=spanid traceId=traceid 1532012145 1532012146", - out); - assertEquals(expectedTimeUnit.convert(startTs / 1000, TimeUnit.SECONDS), (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration/ 1000, TimeUnit.SECONDS), (long) out.get(0).getDuration()); - out.clear(); - - // seconds with duration - decoder.decode("testSpanName source=spanSource spanId=spanid traceId=traceid 1532012145 1", - out); - assertEquals(expectedTimeUnit.convert(startTs / 1000, TimeUnit.SECONDS), (long) out.get(0).getStartMillis()); - assertEquals(expectedTimeUnit.convert(duration/ 1000, TimeUnit.SECONDS), (long) out.get(0).getDuration()); - out.clear(); - } -} diff --git a/java-lib/src/test/java/com/wavefront/ingester/SpanSerializerTest.java b/java-lib/src/test/java/com/wavefront/ingester/SpanSerializerTest.java deleted file mode 100644 index 9c76c63a4..000000000 --- a/java-lib/src/test/java/com/wavefront/ingester/SpanSerializerTest.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.wavefront.ingester; - -import com.google.common.collect.ImmutableList; - -import org.junit.Test; - -import java.util.function.Function; - -import wavefront.report.Annotation; -import wavefront.report.Span; - -import static org.junit.Assert.assertEquals; - -/** - * @author vasily@wavefront.com - */ -public class SpanSerializerTest { - - private Function serializer = new SpanSerializer(); - - @Test - public void testSpanToString() { - Span span = Span.newBuilder() - .setCustomer("dummy") - .setSpanId("4217104a-690d-4927-baff-d9aa779414c2") - .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0") - .setName("testSpanName") - .setSource("spanSource") - .setStartMillis(1532012145123456L) - .setDuration(1111111L) - .setAnnotations(ImmutableList.of(new Annotation("tagk1", "tagv1"), new Annotation("tagk2", "tagv2"))) - .build(); - assertEquals("\"testSpanName\" source=\"spanSource\" spanId=\"4217104a-690d-4927-baff-d9aa779414c2\" " + - "traceId=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" \"tagk1\"=\"tagv1\" \"tagk2\"=\"tagv2\" " + - "1532012145123456 1111111", serializer.apply(span)); - } - - @Test - public void testSpanWithQuotesInTagsToString() { - Span span = Span.newBuilder() - .setCustomer("dummy") - .setSpanId("4217104a-690d-4927-baff-d9aa779414c2") - .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0") - .setName("testSpanName") - .setSource("spanSource") - .setStartMillis(1532012145123456L) - .setDuration(1111111L) - .setAnnotations(ImmutableList.of(new Annotation("tagk1", "tag\"v\"1"), new Annotation("tagk2", "\"tagv2"))) - .build(); - assertEquals("\"testSpanName\" source=\"spanSource\" spanId=\"4217104a-690d-4927-baff-d9aa779414c2\" " + - "traceId=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" \"tagk1\"=\"tag\\\"v\\\"1\" \"tagk2\"=\"\\\"tagv2\" " + - "1532012145123456 1111111", serializer.apply(span)); - } - - @Test - public void testSpanWithNullTagsToString() { - Span span = Span.newBuilder() - .setCustomer("dummy") - .setSpanId("4217104a-690d-4927-baff-d9aa779414c2") - .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0") - .setName("testSpanName") - .setSource("spanSource") - .setStartMillis(1532012145123456L) - .setDuration(1111111L) - .build(); - assertEquals("\"testSpanName\" source=\"spanSource\" spanId=\"4217104a-690d-4927-baff-d9aa779414c2\" " + - "traceId=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" 1532012145123456 1111111", serializer.apply(span)); - } - - @Test - public void testSpanWithEmptyTagsToString() { - Span span = Span.newBuilder() - .setCustomer("dummy") - .setSpanId("4217104a-690d-4927-baff-d9aa779414c2") - .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0") - .setName("testSpanName2") - .setSource("spanSource") - .setStartMillis(1532012145123456L) - .setDuration(1111111L) - .setAnnotations(ImmutableList.of()) - .build(); - assertEquals("\"testSpanName2\" source=\"spanSource\" spanId=\"4217104a-690d-4927-baff-d9aa779414c2\" " + - "traceId=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" 1532012145123456 1111111", serializer.apply(span)); - } -} diff --git a/java-lib/src/test/java/com/wavefront/metrics/JsonMetricsGeneratorTest.java b/java-lib/src/test/java/com/wavefront/metrics/JsonMetricsGeneratorTest.java deleted file mode 100644 index efa09ed67..000000000 --- a/java-lib/src/test/java/com/wavefront/metrics/JsonMetricsGeneratorTest.java +++ /dev/null @@ -1,281 +0,0 @@ -package com.wavefront.metrics; - -import com.google.common.collect.ImmutableList; - -import com.wavefront.common.Pair; -import com.yammer.metrics.core.Counter; -import com.yammer.metrics.core.Histogram; -import com.yammer.metrics.core.MetricName; -import com.yammer.metrics.core.MetricsRegistry; -import com.yammer.metrics.core.WavefrontHistogram; - -import org.codehaus.jackson.map.ObjectMapper; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - -import static com.google.common.truth.Truth.assertThat; - -/** - * Basic unit tests around {@link JsonMetricsGenerator} - * - * @author Tim Schmidt (tim@wavefront.com). - */ -public class JsonMetricsGeneratorTest { - private AtomicLong time = new AtomicLong(0); - private MetricsRegistry testRegistry; - private final ObjectMapper objectMapper = new ObjectMapper(); - - @Before - public void setup() { - testRegistry = new MetricsRegistry(); - time = new AtomicLong(0); - } - - private String generate(boolean includeVMMetrics, - boolean includeBuildMetrics, - boolean clearMetrics, - MetricTranslator metricTranslator) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - JsonMetricsGenerator.generateJsonMetrics(baos, testRegistry, includeVMMetrics, includeBuildMetrics, clearMetrics, - metricTranslator); - return new String(baos.toByteArray()); - } - - /** - * @param map A raw map. - * @param key A key. - * @param clazz See T. - * @param The expected dynamic type of map.get(key) - * @return map.get(key) if it exists and is the right type. Otherwise, fail the calling test. - */ - private T safeGet(Map map, String key, Class clazz) { - assertThat(map.containsKey(key)).isTrue(); - assertThat(map.get(key)).isInstanceOf(clazz); - return clazz.cast(map.get(key)); - } - - @Test - public void testJvmMetrics() throws IOException { - String json = generate(true, false, false, null); - Map top = objectMapper.readValue(json, Map.class); - Map jvm = safeGet(top, "jvm", Map.class); - - Map memory = safeGet(jvm, "memory", Map.class); - safeGet(memory, "totalInit", Double.class); - safeGet(memory, "memory_pool_usages", Map.class); - - Map buffers = safeGet(jvm, "buffers", Map.class); - safeGet(buffers, "direct", Map.class); - safeGet(buffers, "mapped", Map.class); - - safeGet(jvm, "fd_usage", Double.class); - safeGet(jvm, "current_time", Long.class); - - Map threadStates = safeGet(jvm, "thread-states", Map.class); - safeGet(threadStates, "runnable", Double.class); - - Map garbageCollectors = safeGet(jvm, "garbage-collectors", Map.class); - assertThat(threadStates).isNotEmpty(); - // Check that any GC has a "runs" entry. - String key = (String) garbageCollectors.keySet().iterator().next(); // e.g. "PS MarkSweep" - Map gcMap = safeGet(garbageCollectors, key, Map.class); - safeGet(gcMap, "runs", Double.class); - safeGet(gcMap, "time", Double.class); - } - - @Test - public void testTranslator() throws IOException { - Counter counter = testRegistry.newCounter(new MetricName("test", "foo", "bar")); - counter.inc(); - counter.inc(); - String json = generate(false, false, false, metricNameMetricPair -> { - assertThat(metricNameMetricPair._1).isEquivalentAccordingToCompareTo(new MetricName("test", "foo", "bar")); - assertThat(metricNameMetricPair._2).isInstanceOf(Counter.class); - assertThat(((Counter)metricNameMetricPair._2).count()).isEqualTo(2); - return new Pair<>(new MetricName("test", "baz", "qux"), metricNameMetricPair._2); - }); - assertThat(json).isEqualTo("{\"test.qux\":2}"); - json = generate(false, false, false, metricNameMetricPair -> null); - assertThat(json).isEqualTo("{}"); - } - - @Test - public void testYammerHistogram() throws IOException { - Histogram wh = testRegistry.newHistogram(new MetricName("test", "", "metric"), false); - - wh.update(10); - wh.update(100); - wh.update(1000); - - String json = generate(false, false, false, null); - - assertThat(json).isEqualTo("{\"test.metric\":{\"count\":3,\"min\":10.0,\"max\":1000.0,\"mean\":370.0,\"sum\":1110.0,\"stddev\":547.4486277268397,\"median\":100.0,\"p75\":1000.0,\"p95\":1000.0,\"p99\":1000.0,\"p999\":1000.0}}"); - } - - @Test - public void testWavefrontHistogram() throws IOException { - Histogram wh = WavefrontHistogram.get(testRegistry, new MetricName("test", "", "metric"), time::get); - - wh.update(10); - wh.update(100); - wh.update(1000); - - // Simulate the 1 minute has passed and we are ready to flush the histogram - // (i.e. all the values prior to the current minute) over the wire... - time.addAndGet(60001L); - - String json = generate(false, false, false, null); - - assertThat(json).isEqualTo("{\"test.metric\":{\"bins\":[{\"count\":3,\"startMillis\":0,\"durationMillis\":60000,\"means\":[10.0,100.0,1000.0],\"counts\":[1,1,1]}]}}"); - } - - @Test - public void testWavefrontHistogramClear() throws IOException { - Histogram wh = WavefrontHistogram.get(testRegistry, new MetricName("test", "", "metric"), time::get); - wh.update(10); - - // Simulate the 1 minute has passed and we are ready to flush the histogram - // (i.e. all the values prior to the current minute) over the wire... - time.addAndGet(60001L); - - generate(false, false, true, null); - - wh.update(100); - - // Simulate the 1 minute has passed and we are ready to flush the histogram - // (i.e. all the values prior to the current minute) over the wire... - time.addAndGet(60001L); - - String json = generate(false, false, true, null); - - assertThat(json).isEqualTo("{\"test.metric\":{\"bins\":[{\"count\":1,\"startMillis\":60000,\"durationMillis\":60000,\"means\":[100.0],\"counts\":[1]}]}}"); - } - - @Test - public void testWavefrontHistogramNoClear() throws IOException { - Histogram wh = WavefrontHistogram.get(testRegistry, new MetricName("test", "", "metric"), time::get); - - wh.update(10); - generate(false, false, false, null); - wh.update(100); - - // Simulate the 1 minute has passed and we are ready to flush the histogram - // (i.e. all the values prior to the current minute) over the wire... - time.addAndGet(60001L); - - String json = generate(false, false, true, null); - - assertThat(json).isEqualTo("{\"test.metric\":{\"bins\":[{\"count\":2,\"startMillis\":0,\"durationMillis\":60000,\"means\":[10.0,100.0],\"counts\":[1,1]}]}}"); - } - - @Test - public void testWavefrontHistogramSpanMultipleMinutes() throws IOException { - Histogram wh = WavefrontHistogram.get(testRegistry, new MetricName("test", "", "metric"), time::get); - - wh.update(10); - wh.update(100); - - // Simulate the clock advanced by 1 minute - time.set(61 * 1000); - wh.update(1000); - - // Simulate the clock advanced by 1 minute - time.set(61 * 1000 * 2); - String json = generate(false, false, false, null); - - assertThat(json).isEqualTo("{\"test.metric\":{\"bins\":[{\"count\":2,\"startMillis\":0,\"durationMillis\":60000,\"means\":[10.0,100.0],\"counts\":[1,1]},{\"count\":1,\"startMillis\":60000,\"durationMillis\":60000,\"means\":[1000.0],\"counts\":[1]}]}}"); - } - - @Test - public void testWavefrontHistogramPrunesOldBins() throws IOException { - Histogram wh = WavefrontHistogram.get(testRegistry, new MetricName("test", "", "metric"), time::get); - //1 - wh.update(10); - //2 - time.set(61 * 1000); - wh.update(100); - //3 - time.set(121 * 1000); - wh.update(1000); - //4 - time.set(181 * 1000); - wh.update(10000); - //5 - time.set(241 * 1000); - wh.update(100000); - //6 - time.set(301 * 1000); - wh.update(100001); - //7 - time.set(361 * 1000); - wh.update(100011); - //8 - time.set(421 * 1000); - wh.update(100111); - //9 - time.set(481 * 1000); - wh.update(101111); - //10 - time.set(541 * 1000); - wh.update(111111); - //11 - time.set(601 * 1000); - wh.update(111112); - - // Simulate the 1 minute has passed and we are ready to flush the histogram - // (i.e. all the values prior to the current minute) over the wire... - time.addAndGet(60001L); - - String json = generate(false, false, false, null); - assertThat(json).isEqualTo("{\"test.metric\":{\"bins\":[{\"count\":1,\"startMillis\":60000,\"durationMillis\":60000,\"means\":[100.0],\"counts\":[1]},{\"count\":1,\"startMillis\":120000,\"durationMillis\":60000,\"means\":[1000.0],\"counts\":[1]},{\"count\":1,\"startMillis\":180000,\"durationMillis\":60000,\"means\":[10000.0],\"counts\":[1]},{\"count\":1,\"startMillis\":240000,\"durationMillis\":60000,\"means\":[100000.0],\"counts\":[1]},{\"count\":1,\"startMillis\":300000,\"durationMillis\":60000,\"means\":[100001.0],\"counts\":[1]},{\"count\":1,\"startMillis\":360000,\"durationMillis\":60000,\"means\":[100011.0],\"counts\":[1]},{\"count\":1,\"startMillis\":420000,\"durationMillis\":60000,\"means\":[100111.0],\"counts\":[1]},{\"count\":1,\"startMillis\":480000,\"durationMillis\":60000,\"means\":[101111.0],\"counts\":[1]},{\"count\":1,\"startMillis\":540000,\"durationMillis\":60000,\"means\":[111111.0],\"counts\":[1]},{\"count\":1,\"startMillis\":600000,\"durationMillis\":60000,\"means\":[111112.0],\"counts\":[1]}]}}"); - } - - @Test - public void testWavefrontHistogramBulkUpdate() throws IOException { - WavefrontHistogram wh = WavefrontHistogram.get(testRegistry, new MetricName("test", "", "metric"), time::get); - - wh.bulkUpdate(ImmutableList.of(15d, 30d, 45d), ImmutableList.of(1, 5, 1)); - - // Simulate the 1 minute has passed and we are ready to flush the histogram - // (i.e. all the values prior to the current minute) over the wire... - time.addAndGet(60001L); - - String json = generate(false, false, false, null); - - assertThat(json).isEqualTo("{\"test.metric\":{\"bins\":[{\"count\":7,\"startMillis\":0,\"durationMillis\":60000,\"means\":[15.0,30.0,45.0],\"counts\":[1,5,1]}]}}"); - } - - @Test - public void testWavefrontHistogramBulkUpdateHandlesMismatchedLengths() throws IOException { - WavefrontHistogram wh = WavefrontHistogram.get(testRegistry, new MetricName("test", "", "metric"), time::get); - - wh.bulkUpdate(ImmutableList.of(15d, 30d, 45d, 100d), ImmutableList.of(1, 5, 1)); - wh.bulkUpdate(ImmutableList.of(1d, 2d, 3d), ImmutableList.of(1, 1, 1, 9)); - - // Simulate the 1 minute has passed and we are ready to flush the histogram - // (i.e. all the values prior to the current minute) over the wire... - time.addAndGet(60001L); - - String json = generate(false, false, false, null); - - assertThat(json).isEqualTo("{\"test.metric\":{\"bins\":[{\"count\":10,\"startMillis\":0,\"durationMillis\":60000,\"means\":[1.0,2.0,3.0,15.0,30.0,45.0],\"counts\":[1,1,1,1,5,1]}]}}"); - } - - @Test - public void testWavefrontHistogramBulkUpdateHandlesNullParams() throws IOException { - WavefrontHistogram wh = WavefrontHistogram.get(testRegistry, new MetricName("test", "", "metric"), time::get); - - wh.bulkUpdate(null, ImmutableList.of(1, 5, 1)); - wh.bulkUpdate(ImmutableList.of(15d, 30d, 45d, 100d), null); - wh.bulkUpdate(null, null); - - String json = generate(false, false, false, null); - - assertThat(json).isEqualTo("{\"test.metric\":{\"bins\":[]}}"); - } -} diff --git a/java-lib/src/test/java/com/wavefront/metrics/JsonMetricsParserTest.java b/java-lib/src/test/java/com/wavefront/metrics/JsonMetricsParserTest.java deleted file mode 100644 index 674dcdfc6..000000000 --- a/java-lib/src/test/java/com/wavefront/metrics/JsonMetricsParserTest.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.wavefront.metrics; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; - -import wavefront.report.Histogram; -import wavefront.report.ReportPoint; - -import static com.google.common.collect.Lists.newArrayList; -import static com.google.common.truth.Truth.assertThat; -import static com.wavefront.metrics.JsonMetricsParser.report; - -/** - * Unit tests around {@link JsonMetricsParser} - * - * @author Tim Schmidt (tim@wavefront.com). - */ -public class JsonMetricsParserTest { - private final static JsonFactory factory = new JsonFactory(new ObjectMapper()); - private final List points = newArrayList(); - - @Before - public void setUp() throws Exception { - points.clear(); - } - - @Test - public void testStandardHistogram() throws IOException { - JsonNode node = factory.createParser( - "{\"test.metric\":{\"count\":3,\"min\":10.0,\"max\":1000.0,\"mean\":370.0,\"median\":100.0,\"p75\":1000.0,\"p95\":1000.0,\"p99\":1000.0,\"p999\":1000.0}}" - ).readValueAsTree(); - report("customer", "path", node, points, "host", 100L); - - assertThat(points).hasSize(9); - assertThat(points.get(0).getMetric()).isEqualTo("path.test.metric.count"); - assertThat(points.get(0).getValue()).isEqualTo(3.0); - assertThat(points.get(0).getHost()).isEqualTo("host"); - assertThat(points.get(0).getTable()).isEqualTo("customer"); - assertThat(points.get(0).getTimestamp()).isEqualTo(100L); - - assertThat(points.get(1).getMetric()).isEqualTo("path.test.metric.min"); - assertThat(points.get(1).getValue()).isEqualTo(10.0); - assertThat(points.get(1).getHost()).isEqualTo("host"); - assertThat(points.get(1).getTable()).isEqualTo("customer"); - assertThat(points.get(1).getTimestamp()).isEqualTo(100L); - - assertThat(points.get(2).getMetric()).isEqualTo("path.test.metric.max"); - assertThat(points.get(2).getValue()).isEqualTo(1000.0); - assertThat(points.get(2).getHost()).isEqualTo("host"); - assertThat(points.get(2).getTable()).isEqualTo("customer"); - assertThat(points.get(2).getTimestamp()).isEqualTo(100L); - - assertThat(points.get(3).getMetric()).isEqualTo("path.test.metric.mean"); - assertThat(points.get(3).getValue()).isEqualTo(370.0); - assertThat(points.get(3).getHost()).isEqualTo("host"); - assertThat(points.get(3).getTable()).isEqualTo("customer"); - assertThat(points.get(3).getTimestamp()).isEqualTo(100L); - - assertThat(points.get(4).getMetric()).isEqualTo("path.test.metric.median"); - assertThat(points.get(4).getValue()).isEqualTo(100.0); - assertThat(points.get(4).getHost()).isEqualTo("host"); - assertThat(points.get(4).getTable()).isEqualTo("customer"); - assertThat(points.get(4).getTimestamp()).isEqualTo(100L); - - assertThat(points.get(5).getMetric()).isEqualTo("path.test.metric.p75"); - assertThat(points.get(5).getValue()).isEqualTo(1000.0); - assertThat(points.get(5).getHost()).isEqualTo("host"); - assertThat(points.get(5).getTable()).isEqualTo("customer"); - assertThat(points.get(5).getTimestamp()).isEqualTo(100L); - - assertThat(points.get(6).getMetric()).isEqualTo("path.test.metric.p95"); - assertThat(points.get(6).getValue()).isEqualTo(1000.0); - assertThat(points.get(6).getHost()).isEqualTo("host"); - assertThat(points.get(6).getTable()).isEqualTo("customer"); - assertThat(points.get(6).getTimestamp()).isEqualTo(100L); - - assertThat(points.get(7).getMetric()).isEqualTo("path.test.metric.p99"); - assertThat(points.get(7).getValue()).isEqualTo(1000.0); - assertThat(points.get(7).getHost()).isEqualTo("host"); - assertThat(points.get(7).getTable()).isEqualTo("customer"); - assertThat(points.get(7).getTimestamp()).isEqualTo(100L); - - assertThat(points.get(8).getMetric()).isEqualTo("path.test.metric.p999"); - assertThat(points.get(8).getValue()).isEqualTo(1000.0); - assertThat(points.get(8).getHost()).isEqualTo("host"); - assertThat(points.get(8).getTable()).isEqualTo("customer"); - assertThat(points.get(8).getTimestamp()).isEqualTo(100L); - } - - @Test - public void testWavefrontHistogram() throws IOException { - JsonNode node = factory.createParser("{\"test.metric\":{\"bins\":[{\"count\":2,\"startMillis\":0,\"durationMillis\":60000,\"means\":[10.0,100.0],\"counts\":[1,1]},{\"count\":1,\"startMillis\":60000,\"durationMillis\":60000,\"means\":[1000.0],\"counts\":[1]}]}}").readValueAsTree(); - report("customer", "path", node, points, "host", 100L); - - assertThat(points).hasSize(2); - - assertThat(points.get(0).getMetric()).isEqualTo("path.test.metric"); - assertThat(points.get(0).getValue()).isInstanceOf(Histogram.class); - assertThat(((Histogram) points.get(0).getValue()).getDuration()).isEqualTo(60000L); - assertThat(((Histogram) points.get(0).getValue()).getBins()).isEqualTo(newArrayList(10.0, 100.0)); - assertThat(((Histogram) points.get(0).getValue()).getCounts()).isEqualTo(newArrayList(1, 1)); - assertThat(points.get(0).getHost()).isEqualTo("host"); - assertThat(points.get(0).getTable()).isEqualTo("customer"); - assertThat(points.get(0).getTimestamp()).isEqualTo(0L); - - assertThat(points.get(1).getMetric()).isEqualTo("path.test.metric"); - assertThat(points.get(1).getValue()).isInstanceOf(Histogram.class); - assertThat(((Histogram) points.get(1).getValue()).getDuration()).isEqualTo(60000L); - assertThat(((Histogram) points.get(1).getValue()).getBins()).isEqualTo(newArrayList(1000.0)); - assertThat(((Histogram) points.get(1).getValue()).getCounts()).isEqualTo(newArrayList(1)); - assertThat(points.get(1).getHost()).isEqualTo("host"); - assertThat(points.get(1).getTable()).isEqualTo("customer"); - assertThat(points.get(1).getTimestamp()).isEqualTo(60000L); - } -} diff --git a/java-lib/src/test/java/com/wavefront/metrics/ReconnectingSocketTest.java b/java-lib/src/test/java/com/wavefront/metrics/ReconnectingSocketTest.java deleted file mode 100644 index b7a42419a..000000000 --- a/java-lib/src/test/java/com/wavefront/metrics/ReconnectingSocketTest.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.wavefront.metrics; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.ServerSocket; -import java.net.Socket; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author Mori Bellamy (mori@wavefront.com) - */ -public class ReconnectingSocketTest { - protected static final Logger logger = Logger.getLogger(ReconnectingSocketTest.class.getCanonicalName()); - - private Thread testServer; - private ReconnectingSocket toServer; - private int connects = 0; - - @Before - public void initTestServer() throws IOException, InterruptedException { - connects = 0; - final ServerSocket serverSocket = new ServerSocket(0); - - testServer = new Thread(() -> { - while (!Thread.currentThread().isInterrupted()) { - try { - Socket fromClient = serverSocket.accept(); - connects++; - BufferedReader inFromClient = new BufferedReader(new InputStreamReader(fromClient.getInputStream())); - while (true) { - String input = inFromClient.readLine().trim().toLowerCase(); - if (input.equals("give_fin")) { - fromClient.shutdownOutput(); - break; // Go to outer while loop, accept a new socket from serverSocket. - } else if (input.equals("give_rst")) { - fromClient.setSoLinger(true, 0); - fromClient.close(); - break; // Go to outer while loop, accept a new socket from serverSocket. - } - } - } catch (IOException e) { - logger.log(Level.SEVERE, "Unexpected test error.", e); - // OK to go back to the top of the loop. - } - } - }); - - testServer.start(); - toServer = new ReconnectingSocket("localhost", serverSocket.getLocalPort()); - } - - @After - public void teardownTestServer() throws IOException { - testServer.interrupt(); - toServer.close(); - } - - @Test(timeout = 5000L) - public void testReconnect() throws Exception { - toServer.write("ping\n"); - toServer.flush(); - toServer.write("give_fin\n"); - toServer.flush(); - toServer.maybeReconnect(); - toServer.write("ping\n"); - toServer.flush(); - toServer.write("give_rst\n"); - toServer.flush(); - toServer.maybeReconnect(); - toServer.write("ping\n"); - - Assert.assertEquals(2, connects); - } -} \ No newline at end of file diff --git a/macos/log4j2.xml b/macos/log4j2.xml new file mode 100644 index 000000000..063c37de3 --- /dev/null +++ b/macos/log4j2.xml @@ -0,0 +1,73 @@ + + + + + + + + %d %-5level [%c{1}:%M] %m%n + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/macos/wavefront.conf b/macos/wavefront.conf new file mode 100644 index 000000000..a767239ed --- /dev/null +++ b/macos/wavefront.conf @@ -0,0 +1,427 @@ +# +# Wavefront proxy configuration file +# +# Typically in /etc/wavefront/wavefront-proxy/wavefront.conf +# +######################################################################################################################## +# Wavefront API endpoint URL. Usually the same as the URL of your Wavefront instance, with an `api` +# suffix -- or Wavefront provides the URL. +server=WAVEFRONT_SERVER_URL + +# The hostname will be used to identify the internal proxy statistics around point rates, JVM info, etc. +# We strongly recommend setting this to a name that is unique among your entire infrastructure, to make this +# proxy easy to identify. This hostname does not need to correspond to any actual hostname or DNS entry; it's merely +# a string that we pass with the internal stats and ~proxy.* metrics. +hostname=myHost + +# The Token is any valid API token for an account that has *Proxy Management* permissions. To get to the token: +# 1. Click the gear icon at the top right in the Wavefront UI. +# 2. Click your account name (usually your email) +# 3. Click *API access*. +token=TOKEN_HERE + +####################################################### INPUTS ######################################################### +# Comma-separated list of ports to listen on for Wavefront formatted data (Default: 2878) +pushListenerPorts=2878 +## Maximum line length for received points in plaintext format on Wavefront/OpenTSDB/Graphite ports. Default: 32KB +#pushListenerMaxReceivedLength=32768 +## Maximum request size (in bytes) for incoming HTTP requests on Wavefront/OpenTSDB/Graphite ports. Default: 16MB +#pushListenerHttpBufferSize=16777216 + +## Delta counter pre-aggregation settings. +## Comma-separated list of ports to listen on for Wavefront-formatted delta counters. Helps reduce +## outbound point rate by pre-aggregating delta counters at proxy. Default: none +#deltaCountersAggregationListenerPorts=12878 +## Interval between flushing aggregating delta counters to Wavefront. Defaults to 30 seconds. +#deltaCountersAggregationIntervalSeconds=60 + +## Graphite input settings. +## If you enable either `graphitePorts` or `picklePorts`, make sure to uncomment and set `graphiteFormat` as well. +## Comma-separated list of ports to listen on for collectd/Graphite formatted data (Default: none) +#graphitePorts=2003 +## Comma-separated list of ports to listen on for Graphite pickle formatted data (from carbon-relay) (Default: none) +#picklePorts=2004 +## Which fields (1-based) should we extract and concatenate (with dots) as the hostname? +#graphiteFormat=2 +## Which characters should be replaced by dots in the hostname, after extraction? (Default: _) +#graphiteDelimiters=_ +## Comma-separated list of fields (metric segments) to remove (1-based). This is an optional setting. (Default: none) +#graphiteFieldsToRemove=3,4,5 + +## OTLP/OpenTelemetry input settings. +## Comma-separated list of ports to listen on for OTLP formatted data over gRPC (Default: none) +#otlpGrpcListenerPorts=4317 +## Comma-separated list of ports to listen on for OTLP formatted data over HTTP (Default: none) +#otlpHttpListenerPorts=4318 + +## DDI/Relay endpoint: in environments where no direct outbound connections to Wavefront servers are possible, you can +## use another Wavefront proxy that has outbound access to act as a relay and forward all the data received on that +## endpoint (from direct data ingestion clients and/or other proxies) to Wavefront servers. +## This setting is a comma-separated list of ports. (Default: none) +#pushRelayListenerPorts=2978 +## If true, aggregate histogram distributions received on the relay port. +## Please refer to histogram settings section below for more configuration options. Default: false +#pushRelayHistogramAggregator=false + +## Comma-separated list of ports to listen on for OpenTSDB formatted data (Default: none) +#opentsdbPorts=4242 + +## Comma-separated list of ports to listen on for HTTP JSON formatted data (Default: none) +#jsonListenerPorts=3878 + +## Comma-separated list of ports to listen on for HTTP collectd write_http data (Default: none) +#writeHttpJsonListenerPorts=4878 + +################################################# DATA PREPROCESSING ################################################### +## Path to the optional config file with preprocessor rules (advanced regEx replacements and allow/block lists) +#preprocessorConfigFile=/etc/wavefront/wavefront-proxy/preprocessor_rules.yaml + +## When using the Wavefront or TSDB data formats, the proxy will automatically look for a tag named +## source= or host= (preferring source=) and treat that as the source/host within Wavefront. +## customSourceTags is a comma-separated, ordered list of additional tag keys to use if neither +## source= or host= is present +#customSourceTags=fqdn, hostname + +## The prefix should either be left undefined, or can be any prefix you want +## prepended to all data points coming through this proxy (such as `production`). +#prefix=production + +## Regex pattern (Java) that input lines must match to be accepted. Use preprocessor rules for finer control. +#allow=^(production|stage).* +## Regex pattern (Java) that input lines must NOT match to be accepted. Use preprocessor rules for finer control. +#block=^(qa|development|test).* + +## This setting defines the cut-off point for what is considered a valid timestamp for back-dated points. +## Default (and recommended) value is 8760 (1 year), so all the data points from more than 1 year ago will be rejected. +#dataBackfillCutoffHours=8760 +## This setting defines the cut-off point for what is considered a valid timestamp for pre-dated points. +## Default (and recommended) value is 24 (1 day), so all the data points from more than 1 day in future will be rejected. +#dataPrefillCutoffHours=24 + +################################################## ADVANCED SETTINGS ################################################### +## Number of threads that flush data to the server. If not defined in wavefront.conf it defaults to +## the number of available vCPUs (min 4; max 16). Setting this value too large will result in sending +## batches that are too small to the server and wasting connections. This setting is per listening port. +#flushThreads=4 +## Max points per single flush. Default: 40000. +#pushFlushMaxPoints=40000 +## Max histograms per single flush. Default: 10000. +#pushFlushMaxHistograms=10000 +## Max spans per single flush. Default: 5000. +#pushFlushMaxSpans=5000 +## Max span logs per single flush. Default: 1000. +#pushFlushMaxSpanLogs=1000 + +## Milliseconds between flushes to the Wavefront servers. Typically 1000. +#pushFlushInterval=1000 + +## Limit outbound points per second rate at the proxy. Default: do not throttle +#pushRateLimit=20000 +## Limit outbound histograms per second rate at the proxy. Default: do not throttle +#pushRateLimitHistograms=2000 +## Limit outbound spans per second rate at the proxy. Default: do not throttle +#pushRateLimitSpans=1000 +## Limit outbound span logs per second rate at the proxy. Default: do not throttle +#pushRateLimitSpanLogs=1000 + +## Max number of burst seconds to allow when rate limiting to smooth out uneven traffic. +## Set to 1 when doing data backfills. Default: 10 +#pushRateLimitMaxBurstSeconds=10 + +## Location of buffer.* files for saving failed transmissions for retry. Default: /var/spool/wavefront-proxy/buffer +#buffer=/var/spool/wavefront-proxy/buffer +## Buffer file partition size, in MB. Setting this value too low may reduce the efficiency of disk +## space utilization, while setting this value too high will allocate disk space in larger +## increments. Default: 128 +#bufferShardSize=128 +## Use single-file buffer (legacy functionality). Default: false +#disableBufferSharding=false + +## For exponential backoff when retry threads are throttled, the base (a in a^b) in seconds. Default 2.0 +#retryBackoffBaseSeconds=2.0 +## Whether to split the push batch size when the push is rejected by Wavefront due to rate limit. Default false. +#splitPushWhenRateLimited=false + +## The following settings are used to connect to Wavefront servers through a HTTP proxy: +#proxyHost=localhost +#proxyPort=8080 +## Optional: if http proxy requires authentication +#proxyUser=proxy_user +#proxyPassword=proxy_password +## HTTP proxies may implement a security policy to only allow traffic with particular User-Agent header values. +## When set, overrides User-Agent in request headers for outbound HTTP requests. +#httpUserAgent=WavefrontProxy + +## HTTP client settings +## Control whether metrics traffic from the proxy to the Wavefront endpoint is gzip-compressed. Default: true +#gzipCompression=true +## If gzipCompression is enabled, sets compression level (1-9). Higher compression levels slightly reduce +## the volume of traffic between the proxy and Wavefront, but use more CPU. Default: 4 +#gzipCompressionLevel=4 +## Connect timeout (in milliseconds). Default: 5000 (5s) +#httpConnectTimeout=5000 +## Socket timeout (in milliseconds). Default: 10000 (10s) +#httpRequestTimeout=10000 +## Max number of total connections to keep open (Default: 200) +#httpMaxConnTotal=100 +## Max connections per route to keep open (Default: 100) +#httpMaxConnPerRoute=100 +## Number of times to retry http requests before queueing, set to 0 to disable (default: 3) +#httpAutoRetries=3 + +## Close idle inbound connections after specified time in seconds. Default: 300 (5 minutes) +#listenerIdleConnectionTimeout=300 +## When receiving Wavefront-formatted data without source/host specified, use remote IP address as +## source instead of trying to resolve the DNS name. Default false. +#disableRdnsLookup=true +## The following setting enables SO_LINGER on listening ports with the specified linger time in seconds (Default: off) +#soLingerTime=0 + +## Max number of points that can stay in memory buffers before spooling to disk. Defaults to 16 * pushFlushMaxPoints, +## minimum allowed size: pushFlushMaxPoints. Setting this value lower than default reduces memory usage but will force +## the proxy to spool to disk more frequently if you have points arriving at the proxy in short bursts. +#pushMemoryBufferLimit=640000 + +## If there are blocked points, a small sampling of them can be written into the main log file. +## This setting how many lines to print to the log every 10 seconds. Typically 5. +#pushBlockedSamples=5 +## When logging blocked points is enabled (see https://docs.wavefront.com/proxies_configuring.html#blocked-point-log +## for more details), by default all blocked points/histograms/spans etc are logged into the same `RawBlockedPoints` +## logger. Settings below allow finer control over these loggers: +## Logger name for blocked points. Default: RawBlockedPoints. +#blockedPointsLoggerName=RawBlockedPoints +## Logger name for blocked histograms. Default: RawBlockedPoints. +#blockedHistogramsLoggerName=RawBlockedPoints +## Logger name for blocked spans. Default: RawBlockedPoints. +#blockedSpansLoggerName=RawBlockedPoints + +## Discard all received data (debug/performance test mode). If enabled, you will lose received data! Default: false +#useNoopSender=false + +## Settings for incoming HTTP request authentication. Authentication is done by a token, proxy is looking for +## tokens in the querystring ("token=" and "api_key=" parameters) and in request headers ("X-AUTH-TOKEN: ", +## "Authorization: Bearer", "Authorization: " headers). TCP streams are disabled when authentication is turned on. +## Allowed authentication methods: NONE, STATIC_TOKEN, HTTP_GET, OAUTH2. Default: NONE +## - NONE: All requests are considered valid +## - STATIC_TOKEN: Compare incoming token with the value of authStaticToken setting. +## - OAUTH2: Validate all tokens against a RFC7662-compliant token introspection endpoint. +## - HTTP_GET: Validate all tokens against a specific URL. Use {{token}} placeholder in the URL to pass the token +## in question to the endpoint. Use of https is strongly recommended for security reasons. The endpoint +## must return any 2xx status for valid tokens, any other response code is considered a fail. +#authMethod=NONE +## URL for the token introspection endpoint used to validate tokens for incoming HTTP requests. +## Required when authMethod is OAUTH2 or HTTP_GET +#authTokenIntrospectionServiceUrl=https://auth.acme.corp/api/token/{{token}}/validate +## Optional credentials for use with the token introspection endpoint if it requires authentication. +#authTokenIntrospectionAuthorizationHeader=Authorization: Bearer +## Cache TTL (in seconds) for token validation results (re-authenticate when expired). Default: 600 seconds +#authResponseRefreshInterval=600 +## Maximum allowed cache TTL (in seconds) for token validation results when token introspection service is +## unavailable. Default: 86400 seconds (1 day) +#authResponseMaxTtl=86400 +## Static token that is considered valid for all incoming HTTP requests. Required when authMethod = STATIC_TOKEN. +#authStaticToken=token1234abcd + +## Enables intelligent traffic shaping based on received rate over last 5 minutes. Default: disabled +#trafficShaping=false +## Sets the width (in seconds) for the sliding time window which would be used to calculate received +## traffic rate. Default: 600 (10 minutes) +#trafficShapingWindowSeconds=600 +## Sets the headroom multiplier to use for traffic shaping when there's backlog. Default: 1.15 (15% headroom) +#trafficShapingHeadroom=1.15 + +## Enables CORS for specified comma-delimited list of listening ports. Default: none (CORS disabled) +#corsEnabledPorts=2879 +## Allowed origin for CORS requests, or '*' to allow everything. Default: none +#corsOrigin=* +## Allow 'null' origin for CORS requests. Default: false +#corsAllowNullOrigin=false + +## Enables TLS for specified listening ports (comma-separated list). Use '*' to secure all ports. Defaut: none (TLS disabled) +#tlsPorts=4443 +## TLS certificate path to use for securing all the ports. X.509 certificate chain file in PEM format. +#privateCertPath=/etc/wavefront/wavefront-proxy/cert.pem +## TLS private key path to use for securing all the ports. PKCS#8 private key file in PEM format. +#privateKeyPath=/etc/wavefront/wavefront-proxy/private_key.pem + +########################################### MANAGED HEALTHCHECK ENDPOINT ############################################### +## Comma-delimited list of ports to function as standalone healthchecks. May be used independently of +## httpHealthCheckAllPorts parameter. Default: none +#httpHealthCheckPorts=8880 +## When true, all listeners that support HTTP protocol also respond to healthcheck requests. May be +## used independently of httpHealthCheckPorts parameter. Default: false +#httpHealthCheckAllPorts=true +## Healthcheck's path, for example, '/health'. Default: '/' +#httpHealthCheckPath=/status +## Optional Content-Type to use in healthcheck response, for example, 'application/json'. Default: none +#httpHealthCheckResponseContentType=text/plain +## HTTP status code for 'pass' health checks. Default: 200 +#httpHealthCheckPassStatusCode=200 +## Optional response body to return with 'pass' health checks. Default: none +#httpHealthCheckPassResponseBody=good to go! +## HTTP status code for 'fail' health checks. Default: 503 +#httpHealthCheckFailStatusCode=503 +## Optional response body to return with 'fail' health checks. Default: none +#httpHealthCheckFailResponseBody=try again later... +## Enables admin port to control healthcheck status per port. Default: none +#adminApiListenerPort=8888 +## Remote IPs must match this regex to access admin API +#adminApiRemoteIpAllowRegex=^.*$ + +############################################# LOGS TO METRICS SETTINGS ################################################# +## Port on which to listen for FileBeat data (Lumberjack protocol). Default: none +#filebeatPort=5044 +## Port on which to listen for raw logs data (TCP and HTTP). Default: none +#rawLogsPort=5045 +## Maximum line length for received raw logs (Default: 4096) +#rawLogsMaxReceivedLength=4096 +## Maximum allowed request size (in bytes) for incoming HTTP requests with raw logs (Default: 16MB) +#rawLogsHttpBufferSize=16777216 +## Location of the `logsingestion.yaml` configuration file +#logsIngestionConfigFile=/etc/wavefront/wavefront-proxy/logsingestion.yaml + +########################################### DISTRIBUTED TRACING SETTINGS ############################################### +## Comma-separated list of ports to listen on for spans in Wavefront format. Defaults to none. +#traceListenerPorts=30000 +## Maximum line length for received spans and span logs (Default: 1MB) +#traceListenerMaxReceivedLength=1048576 +## Maximum allowed request size (in bytes) for incoming HTTP requests on tracing ports (Default: 16MB) +#traceListenerHttpBufferSize=16777216 + +## Comma-separated list of ports to listen on for spans from SDKs that send raw data. Unlike +## `traceListenerPorts` setting, also derives RED metrics based on received spans. Defaults to none. +#customTracingListenerPorts=30001 +## Custom application name for spans received on customTracingListenerPorts. Defaults to defaultApp. +#customTracingApplicationName=defaultApp +## Custom service name for spans received on customTracingListenerPorts. Defaults to defaultService. +#customTracingServiceName=defaultService + +## Comma-separated list of ports on which to listen on for Jaeger Thrift formatted data over TChannel protocol. Defaults to none. +#traceJaegerListenerPorts=14267 +## Comma-separated list of ports on which to listen on for Jaeger Thrift formatted data over HTTP. Defaults to none. +#traceJaegerHttpListenerPorts=14268 +## Comma-separated list of ports on which to listen on for Jaeger Protobuf formatted data over gRPC. Defaults to none. +#traceJaegerGrpcListenerPorts=14250 +## Custom application name for traces received on Jaeger's traceJaegerListenerPorts. +#traceJaegerApplicationName=Jaeger +## Comma-separated list of ports on which to listen on for Zipkin trace data over HTTP. Defaults to none. +## Recommended value is 9411, which is the port Zipkin's server listens on and is the default configuration in Istio. +#traceZipkinListenerPorts=9411 +## Custom application name for traces received on Zipkin's traceZipkinListenerPorts. +#traceZipkinApplicationName=Zipkin +## Comma-separated list of additional custom tag keys to include along as metric tags for the derived +## RED (Request, Error, Duration) metrics. Applicable to Jaeger and Zipkin integration only. +#traceDerivedCustomTagKeys=tenant, env, location + +## The following settings are used to configure trace data sampling: +## The rate for traces to be sampled. Can be from 0.0 to 1.0. Defaults to 1.0 +#traceSamplingRate=1.0 +## The minimum duration threshold (in milliseconds) for the spans to be sampled. +## Spans above the given duration are reported. Defaults to 0 (include everything). +#traceSamplingDuration=0 + +########################################## HISTOGRAM ACCUMULATION SETTINGS ############################################# +## Histograms can be ingested in Wavefront scalar and distribution format. For scalar samples ports can be specified for +## minute, hour and day granularity. Granularity for the distribution format is encoded inline. Before using any of +## these settings, reach out to Wavefront Support to ensure your account is enabled for native Histogram support and +## to optimize the settings for your specific use case. + +## Accumulation parameters +## Directory for persisting proxy state, must be writable. +#histogramStateDirectory=/var/spool/wavefront-proxy +## Interval to write back accumulation changes to disk in milliseconds (only applicable when memory cache is enabled). +#histogramAccumulatorResolveInterval=5000 +## Interval to check for histograms ready to be sent to Wavefront, in milliseconds. +#histogramAccumulatorFlushInterval=10000 +## Max number of histograms to send to Wavefront in one flush (Default: no limit) +#histogramAccumulatorFlushMaxBatchSize=4000 +## Maximum line length for received histogram data (Default: 65536) +#histogramMaxReceivedLength=65536 +## Maximum allowed request size (in bytes) for incoming HTTP requests on histogram ports (Default: 16MB) +#histogramHttpBufferSize=16777216 + +## Wavefront format, minute aggregation: +## Comma-separated list of ports to listen on. +#histogramMinuteListenerPorts=40001 +## Time-to-live in seconds for a minute granularity accumulation on the proxy (before the intermediary is shipped to WF). +#histogramMinuteFlushSecs=70 +## Bounds the number of centroids per histogram. Must be between 20 and 1000, default: 32 +#histogramMinuteCompression=32 +## Expected upper bound of concurrent accumulations. Should be approximately the number of timeseries * 2 (use a higher +## multiplier if out-of-order points more than 1 minute apart are expected). Setting this value too high will cause +## excessive disk space usage, setting this value too low may cause severe performance issues. +#histogramMinuteAccumulatorSize=1000 +## Enabling memory cache reduces I/O load with fewer time series and higher frequency data (more than 1 point per +## second per time series). Default: false +#histogramMinuteMemoryCache=false +## Whether to persist accumulation state. When true, all histograms are written to disk immediately if memory cache is +## disabled, or every histogramAccumulatorResolveInterval seconds if memory cache is enabled. +## If accumulator is not persisted, up to histogramMinuteFlushSecs seconds worth of histograms may be lost on proxy shutdown. +#histogramMinuteAccumulatorPersisted=false + +## Wavefront format, hour aggregation: +## Comma-separated list of ports to listen on. +#histogramHourListenerPorts=40002 +## Time-to-live in seconds for an hour granularity accumulation on the proxy (before the intermediary is shipped to WF). +#histogramHourFlushSecs=4200 +## Bounds the number of centroids per histogram. Must be between 20 and 1000, default: 32 +#histogramHourCompression=32 +## Expected upper bound of concurrent accumulations. Should be approximately the number of timeseries * 2 (use a higher +## multiplier if out-of-order points more than 1 hour apart are expected). Setting this value too high will cause +## excessive disk space usage, setting this value too low may cause severe performance issues. +#histogramHourAccumulatorSize=100000 +## Enabling memory cache reduces I/O load with fewer time series and higher frequency data (more than 1 point per +## second per time series). Default: false +#histogramHourMemoryCache=true +## Whether to persist accumulation state. When true, all histograms are written to disk immediately if memory cache is +## disabled, or every `histogramAccumulatorResolveInterval` seconds if memory cache is enabled. +## If accumulator is not persisted, up to `histogramHourFlushSecs` seconds worth of histograms may be lost on proxy shutdown. +#histogramHourAccumulatorPersisted=true + +## Wavefront format, day aggregation: +## Comma-separated list of ports to listen on. +#histogramDayListenerPorts=40003 +## Time-to-live in seconds for a day granularity accumulation on the proxy (before the intermediary is shipped to WF). +#histogramDayFlushSecs=18000 +## Bounds the number of centroids per histogram. Must be between 20 and 1000, default: 32 +#histogramDayCompression=32 +## Expected upper bound of concurrent accumulations. Should be approximately the number of timeseries * 2 (use a higher +## multiplier if out-of-order points more than 1 day apart are expected). Setting this value too high will cause +## excessive disk space usage, setting this value too low may cause severe performance issues. +#histogramDayAccumulatorSize=100000 +## Enabling memory cache reduces I/O load with fewer time series and higher frequency data (more than 1 point per +## second per time series). Default: false +#histogramDayMemoryCache=false +## Whether to persist accumulation state. When true, all histograms are written to disk immediately if memory cache is +## disabled, or every `histogramAccumulatorResolveInterval` seconds if memory cache is enabled. +## If accumulator is not persisted, up to `histogramDayFlushSecs` seconds worth of histograms may be lost on proxy shutdown. +#histogramDayAccumulatorPersisted=true + +## Distribution format: +## Comma-separated list of ports to listen on. +#histogramDistListenerPorts=40000 +## Time-to-live in seconds for a distribution accumulation on the proxy (before the intermediary is shipped to WF). +#histogramDistFlushSecs=70 +## Bounds the number of centroids per histogram. Must be between 20 and 1000, default: 32 +#histogramDistCompression=32 +## Expected upper bound of concurrent accumulations. Should be approximately the number of timeseries * 2 (use a higher +## multiplier if out-of-order points more than 1 bin apart are expected). Setting this value too high will cause +## excessive disk space usage, setting this value too low may cause severe performance issues. +#histogramDistAccumulatorSize=100000 +## Enabling memory cache reduces I/O load with fewer time series and higher frequency data (more than 1 point per +## second per time series). Default: false +#histogramDistMemoryCache=false +## Whether to persist accumulation state. When true, all histograms are written to disk immediately if memory cache is +## disabled, or every histogramAccumulatorResolveInterval seconds if memory cache is enabled. +## If accumulator is not persisted, up to histogramDistFlushSecs seconds worth of histograms may be lost on proxy shutdown. +#histogramDistAccumulatorPersisted=true + +## Histogram accumulation for relay ports (only applicable if pushRelayHistogramAggregator is true): +## Time-to-live in seconds for the accumulator (before the intermediary is shipped to WF). Default: 70 +#pushRelayHistogramAggregatorFlushSecs=70 +## Bounds the number of centroids per histogram. Must be between 20 and 1000, default: 32 +#pushRelayHistogramAggregatorCompression=32 +## Expected upper bound of concurrent accumulations. Should be approximately the number of timeseries * 2 (use a higher +## multiplier if out-of-order points more than 1 bin apart are expected). Setting this value too high will cause +## excessive disk space usage, setting this value too low may cause severe performance issues. +#pushRelayHistogramAggregatorAccumulatorSize=1000000 diff --git a/macos/wfproxy b/macos/wfproxy new file mode 100644 index 000000000..6fc2a0744 --- /dev/null +++ b/macos/wfproxy @@ -0,0 +1,46 @@ +#!/bin/bash -x + +path=$(dirname "$0") +path=${path%opt*} + +eval $($path/bin/brew shellenv) +CMDDIR=$(dirname $0) +NAME=$(basename $0) +JAVA_HOME=$(/usr/libexec/java_home -v 11) +PROXY=$(brew --prefix $NAME) + +BUFFER_DIR=${HOMEBREW_PREFIX}/var/spool/wavefront-proxy +LOG_DIR=${HOMEBREW_PREFIX}/var/log/wavefront-proxy + +if [ -x ${JAVA_HOME}/bin/java ]; then + echo "Using JAVA_HOME=${JAVA_HOME}" +else + echo "JAVA_HOME not found" +fi + +mkdir -p ${BUFFER_DIR} +mkdir -p ${LOG_DIR} + +if [ ! -w ${BUFFER_DIR} ]; then + echo "Error!!! can't write to '${BUFFER_DIR}', please review directory permissions" + exit -1 +fi + +if [ ! -w ${LOG_DIR} ]; then + echo "Error!!! can't write to '${LOG_DIR}', please review directory permissions" + exit -1 +fi + +ENV_FILE=${HOMEBREW_PREFIX}/etc/wfproxy/prox_env.conf +if [ -f "${ENV_FILE}" ] ; then + . ${ENV_FILE} +fi + +${JAVA_HOME}/bin/java \ + $JAVA_ARGS \ + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \ + -Dlog4j.configurationFile=${HOMEBREW_PREFIX}/etc/wavefront/wavefront-proxy/log4j2.xml \ + -jar ${PROXY}/lib/wavefront-proxy.jar \ + -f ${HOMEBREW_PREFIX}/etc/wavefront/wavefront-proxy/wavefront.conf \ + --idFile ${BUFFER_DIR}/.id \ + --buffer ${BUFFER_DIR}/buffer \ No newline at end of file diff --git a/macos_proxy_notarization/create_credentials.sh b/macos_proxy_notarization/create_credentials.sh new file mode 100644 index 000000000..d697f7075 --- /dev/null +++ b/macos_proxy_notarization/create_credentials.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +mkdir -p ~/.aws + +cat > ~/.aws/credentials << EOL +[default] +aws_access_key_id=${AWS_ACCESS_KEY_ID} +aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} + +EOL \ No newline at end of file diff --git a/macos_proxy_notarization/github_workflow_wrapper_for_notarization.sh b/macos_proxy_notarization/github_workflow_wrapper_for_notarization.sh new file mode 100755 index 000000000..71f3f9a4a --- /dev/null +++ b/macos_proxy_notarization/github_workflow_wrapper_for_notarization.sh @@ -0,0 +1,217 @@ +usage () { + set +x + echo "command line parameters" + echo "1 | proxy version | required=True | default='' | example: '11.1.0'" + echo "3 | github API token | required=True | default='' | example: 'ghp_xxxxx'" + echo "3 | release type | required=False | default='proxy-test' | example: 'proxy-snapshot' / 'proxy-GA'" + echo "4 | github org | required=False | default='wavefrontHQ' | example: 'wavefrontHQ' / 'sbhakta-vmware' (for forked repos)" + echo "5 | debug | required=False | default='' | example: 'debug'" + + + echo "Example command:" + echo "$0 11.1.0 ghp_xxxx proxy-snapshot sbhakta-vmware debug" + echo "$0 11.1.0 ghp_xxxx proxy-snapshot debug # uses wavefrontHQ org" + +} + +trigger_workflow() { + # trigger workflow and make sure it is running + # trigger the Github workflow and sleep for 5- + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${github_token}" "https://api.github.com/repos/${github_org}/${github_repo}/actions/workflows/${github_notarization_workflow_yml}/dispatches" -d '{"ref":"'$github_branch'","inputs":{"proxy_version":"'$proxy_version'","release_type":"'$release_type'"}' + sleep 5 + +} + + +check_jobs_completion() { + jobs_url=$1 + + # add some safeguard in place for infinite while loop + # if allowed_loops=100, sleep_bwtween_runs=15, total allowed time for loop to run= 15*100=1500sec(25min) (normal run takes 15min or so) + allowed_loops=100 + current_loop=0 + sleep_between_runs=15 + + total_allowed_loop_time=`expr $allowed_loops \* $sleep_between_runs` + + # start checking status of our workflow run until it succeeds/fails/times out + while true; + do + # increment current loop count + ((current_loop++)) + + # infinite loop safeguard + if [[ $current_loop -ge $allowed_loops ]]; then + echo "Total allowed time exceeded: $total_allowed_loop_time sec! Workflow taking too long to finish... Quitting!!" + exit 1 + fi + + + echo "Checking status and conclusion of the running job...." + status=`curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${github_token}" $jobs_url | jq '.jobs[0].status' | tr -d '"'`; + conclusion=`curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${github_token}" $jobs_url | jq '.jobs[0].conclusion' | tr -d '"'`; + echo "### status=$status & conclusion=$conclusion" + if [[ ( "$status" == "completed" ) && ( "$conclusion" == "success" ) ]]; then + echo "Job completed successfully" + break + elif [[ ("$status" == "in_progress") || ("$status" == "queued") ]]; then + echo "Still in progress or queued. Sleep for $sleep_between_runs sec and try again..." + echo "loop time so far / total allowed loop time = `expr $current_loop \* $sleep_between_runs` / $total_allowed_loop_time" + sleep $sleep_between_runs + else # everything else + echo "Job did not complete successfully" + exit 1 + fi + + done + +} + + + +####################################### +############# MAIN #################### +####################################### + + +if [[ "$1" == "--help" || "$1" == "-h" ]]; then + usage + exit 0 +fi + +# command line args +proxy_version=$1 +github_token=$2 +release_type=$3 +github_org=$4 +debug=$5 + +# constants +github_repo='wavefront-proxy' +github_notarization_workflow_yml='mac_tarball_notarization.yml' +github_branch='master' + +if [[ -z $proxy_version ]]; then + echo "proxy version is required as 1st cmd line argument. example: '11.1.0-proxy-snapshot'. Exiting!" + usage + exit 1 +fi + +if [[ -z $release_type ]]; then + release_type='proxy-test' +fi + +if [[ -z $github_token ]]; then + echo "github token is required as 3rd cmd line argument. Exiting!" + usage + exit 1 +fi + +if [[ -z $github_org ]]; then + github_org='wavefrontHQ' +fi + +if [[ ! -z $debug ]]; then + set -x +fi + +# print all variables for reference +echo "proxy_version=$proxy_version" +echo "release_type=$release_type" +echo "github_org=$github_org" +echo "github_repo=$github_repo" +echo "github_branch=$github_branch" +echo "github_notarization_workflow_yml=$github_notarization_workflow_yml" + + +# get current date/time that github workflow API understands +# we'll us this in our REST API call to get latest runs later than current time. +format_date=`date +'%Y-%d-%m'` +format_current_time=`date +'%H-%M-%S'` +date_str=$format_date'T'$format_current_time +echo "date_str=$date_str" + + +# trigger the workflow +trigger_workflow + + +# get count of currently running jobs for our workflow, later than "date_str" +# retry 4 times, our triggered workflow may need some time to get started. +max_retries=4 +sleep_between_retries=15 +for retry in $(seq 1 $max_retries); do + + current_running_jobs=`curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${github_token}" "https://api.github.com/repos/${github_org}/${github_repo}/actions/workflows/${github_notarization_workflow_yml}/runs?status=in_progress&created>=${date_str}" | jq '.total_count'` + echo "### total runs right now=$current_running_jobs" + + if [[ $current_running_jobs == 0 ]]; then + echo "No currently running jobs found. sleep for $sleep_between_retries sec and retry! ${retry}/$max_retries" + sleep $sleep_between_retries + else # current runs are > 0 + break + fi +done + +# if no current running jobs, exit +if [[ $current_running_jobs == 0 ]]; then + echo "No currently running jobs found. retry=${retry}/$max_retries.. Exiting" + exit 1 +fi + +# we get the triggered run's jobs_url for checking status + +# there may be multiple workflows running, we need t uniquely identify which is our workflow +# Steps to identify our workflow uniquely - +# 1. loop through all runs in progress +# 2. Get the "jobs_url" for each +# 3. Run a GET API call on "jobs_url", and look at the step names of the workflow. +# - sometimes the steps may take time to load, retry if there are no step names so far +# 4. the workflow is set in such a way that the steps have a unique name depending on the version passed +# 5. Identlfy the jobs_url with the unique step name, and store the jobs_url to see if the workflow is successful or not + +## set variables +jobs_url='' +found_jobs_url=False + +for i in $(seq 0 $((current_running_jobs-1))); do + jobs_url=`curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${github_token}" "https://api.github.com/repos/${github_org}/${github_repo}/actions/runs?status=in_progress&created>=$date_str" | jq '.workflow_runs['"$i"'].jobs_url' | tr -d '"'`; + echo "### jobs_url=$jobs_url" + if [[ jobs_url != '' ]]; then + for retry_step_name in $(seq 1 3); do + # assuming only 1 run inside a job, get the 2nd step name, which has the unique version associated with it. + step_name=`curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${github_token}" "$jobs_url" | jq '.jobs[0].steps[1].name'` + echo "### step_name=$step_name" + # if step_name is null, retry again + if [[ -z $step_name ]]; then + echo "Step_name is null, sleep and rety again!!!" + sleep 10 + continue + # verify the step name has the version passed as cmd line to this script + elif [[ (! -z $step_name) && ($step_name =~ .*$proxy_version.*) ]]; then + echo "We've found our running job for proxy_version:$proxy_version. Final jobs_url below..." + found_jobs_url=True + break; + # this may not be the correct job_url we're looking for + else + echo "Reset jobs_url" + jobs_url='' + fi + done + fi + + if [[ $found_jobs_url == True ]]; then + break + fi +done + +# check if we found the correct jobs_url for our running job +if [[ $jobs_url == '' ]]; then + echo "no jobs_url found for proxy_version:$proxy_version.. quitting" + exit 1 +fi +echo "Confirmed jobs_url=$jobs_url" + + +check_jobs_completion $jobs_url +set +x diff --git a/macos_proxy_notarization/proxy_notarization.sh b/macos_proxy_notarization/proxy_notarization.sh new file mode 100644 index 000000000..519e79347 --- /dev/null +++ b/macos_proxy_notarization/proxy_notarization.sh @@ -0,0 +1,134 @@ +set -ev + +WFPROXY_TARBALL=$1 +echo "This is the tarball that was just uplaoded: $1" +PARSED_TARBALL="`echo $WFPROXY_TARBALL | sed 's/.tar.gz//'`" +echo $PARSED_TARBALL + +echo "List of proxy that are already notarized:" +LIST_ALREADY_NOTARIZED="`aws s3 ls s3://wavefront-cdn/brew/ | sort -r | grep wavefront-proxy | awk '{print $4}'`" +echo $LIST_ALREADY_NOTARIZED + +# Checking against this list that is already notarized +check_notarized_list() { + if [[ "$LIST_ALREADY_NOTARIZED" == *"$PARSED_TARBALL"* ]]; then + echo "$PARSED_TARBALL is in the bucket" + exit 0 + else + echo "It's not in the directory, we need to do the whole notarization process and move it into brew folder." + fi +} +# Create Apple Developer certs on travisci env +create_dev_certs() { + echo "Adding OSX Certificates" + KEY_CHAIN=build.keychain + CERTIFICATE_P12=certificate.p12 + ESO_TEAM_P12=eso_certificate.p12 + + echo "Recreate the certificate from the secure environment variable" + echo $WAVEFRONT_TEAM_CERT_P12 | base64 -D -o $ESO_TEAM_P12; + echo $CERTIFICATE_OSX_P12 | base64 -D -o $CERTIFICATE_P12; + + echo "Create a keychain" + security create-keychain -p travis $KEY_CHAIN + + echo "Make the keychain the default so identities are found" + security default-keychain -s $KEY_CHAIN + + echo "Unlock the keychain 1" + security unlock-keychain -p travis $KEY_CHAIN + + echo "Unlock the keychain 2" + ls + echo $CERTIFICATE_P12 + echo $ESO_TEAM_P12 + security import ./eso_certificate.p12 -x -t agg -k $KEY_CHAIN -P $WAVEFRONT_TEAM_CERT_PASSWORD -T /usr/bin/codesign; + security import ./certificate.p12 -x -t agg -k $KEY_CHAIN -P $CERTIFICATE_PASSWORD -T /usr/bin/codesign; + + echo "Finding identity" + security find-identity -v + + echo "Unlock the keychain 3" + security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k travis $KEY_CHAIN + + echo "Delete certs" + rm -fr *.p12 +} + +# Parse the proxy version our of the +parse_proxy_version_tarball() { + echo "Get the version" + TO_BE_NOTARIZED=$(aws s3 ls s3://eso-wfproxy-testing/to_be_notarized/$WFPROXY_TARBALL | awk '{print $4}') + RE=[0-9]+\.[0-9]+\.[0-9]+ + if [[ $TO_BE_NOTARIZED =~ $RE ]]; then + echo ${BASH_REMATCH[0]}; + VERSION=${BASH_REMATCH[0]} + fi + echo $VERSION +} + +# CP non-notarized proxy, cp signed jdk with it, package as .zip +repackage_proxy() { + COPY_FORM_TO_BE_NOTARIZED="aws s3 cp s3://eso-wfproxy-testing/to_be_notarized/wfproxy-$VERSION.tar.gz ." + $COPY_FORM_TO_BE_NOTARIZED + TARBALL="wfproxy-$VERSION.tar.gz" + tar xvzf $TARBALL + zip -r wavefront-proxy-$VERSION.zip bin/ etc/ lib/ +} + +# Notarized the .zip and upload to Apply +notarized_newly_package_proxy() { + echo "Codesigning the wavefront-proxy package" + codesign -f -s "$ESO_DEV_ACCOUNT" wavefront-proxy-$VERSION.zip --deep --options runtime + + echo "Verifying the codesign" + codesign -vvv --deep --strict wavefront-proxy-$VERSION.zip + + echo "Uploading the package for Notarization" + response="$(xcrun altool --notarize-app --primary-bundle-id "com.wavefront" --username "$USERNAME" --password "$APP_SPECIFIC_PW" --file "wavefront-proxy-$VERSION.zip" | sed -n '2 p')" + echo $response + + echo "Grabbing Request UUID" + requestuuid=${response#*= } + echo $requestuuid + + echo "Executing this command to see the status of notarization" + xcrun altool --notarization-info "$requestuuid" -u "$USERNAME" -p "$APP_SPECIFIC_PW" +} + +# Pass or fail based on notarization status +wait_for_notarization() { + status="$(xcrun altool --notarization-info "$requestuuid" -u "$USERNAME" -p "$APP_SPECIFIC_PW")" + in_progress='Status: in progress' + success='Status Message: Package Approved' + invalid='Status: invalid' + + while true; + do + echo $status + if [[ "$status" == *"$success"* ]]; then + echo "Successful notarization" + aws s3 cp wavefront-proxy-$VERSION.zip s3://wavefront-cdn/brew/ + exit 0 + elif [[ "$status" == *"$in_progress"* ]]; then + status="$(xcrun altool --notarization-info "$requestuuid" -u "$USERNAME" -p "$APP_SPECIFIC_PW")" + sleep 60 + elif [[ "$status" == *"$invalid"* ]]; then + echo "Failed notarization" + exit 1 + fi + done +} + +main() { + check_notarized_list + create_dev_certs + parse_proxy_version_tarball + echo $VERSION + repackage_proxy + notarized_newly_package_proxy + sleep 20 + wait_for_notarization +} + +main \ No newline at end of file diff --git a/macos_proxy_notarization/proxy_notarization_sbhakta_test.sh b/macos_proxy_notarization/proxy_notarization_sbhakta_test.sh new file mode 100644 index 000000000..1595826cb --- /dev/null +++ b/macos_proxy_notarization/proxy_notarization_sbhakta_test.sh @@ -0,0 +1,110 @@ +set -ev + +WFPROXY_ZIP_TO_BE_NOTARIZED=$1 +echo "This is the zip that was just uplaoded: $1" + +# Create Apple Developer certs on travisci env +create_dev_certs() { + echo "Adding OSX Certificates" + KEY_CHAIN=build.keychain + CERTIFICATE_P12=certificate.p12 + ESO_TEAM_P12=eso_certificate.p12 + + echo "Recreate the certificate from the secure environment variable" + echo $WAVEFRONT_TEAM_CERT_P12 | base64 -D -o $ESO_TEAM_P12; + echo $CERTIFICATE_OSX_P12 | base64 -D -o $CERTIFICATE_P12; + + echo "Create a keychain" + security create-keychain -p travis $KEY_CHAIN + + echo "Make the keychain the default so identities are found" + security default-keychain -s $KEY_CHAIN + + echo "Unlock the keychain 1" + security unlock-keychain -p travis $KEY_CHAIN + + echo "Unlock the keychain 2" + ls + echo $CERTIFICATE_P12 + echo $ESO_TEAM_P12 + security import ./eso_certificate.p12 -x -t agg -k $KEY_CHAIN -P $WAVEFRONT_TEAM_CERT_PASSWORD -T /usr/bin/codesign; + security import ./certificate.p12 -x -t agg -k $KEY_CHAIN -P $CERTIFICATE_PASSWORD -T /usr/bin/codesign; + + echo "Finding identity" + security find-identity -v + + echo "Unlock the keychain 3" + security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k travis $KEY_CHAIN + + echo "Delete certs" + rm -fr *.p12 +} + +# Parse the proxy version our of the +parse_proxy_version_from_zip() { + echo "Get the version" + TO_BE_NOTARIZED=$(aws s3 ls s3://eso-wfproxy-testing/to_be_notarized/$WFPROX_ZIP_TO_BE_NOTARIZED | awk '{print $4}') + RE=[0-9]+\.[0-9]+.[0-9-_]+ + if [[ $TO_BE_NOTARIZED =~ $RE ]]; then + echo ${BASH_REMATCH[0]}; + VERSION=${BASH_REMATCH[0]} + fi + echo $VERSION +} + +# Notarized the .zip and upload to Apply +notarized_newly_package_proxy() { + echo "Downloading the ZIP to be notarized" + aws s3 cp s3://eso-wfproxy-testing/to_be_notarized/$WFPROXY_ZIP_TO_BE_NOTARIZED . + echo "Codesigning the wavefront-proxy package" + codesign -f -s "$ESO_DEV_ACCOUNT" $WFPROXY_ZIP_TO_BE_NOTARIZED --deep --options runtime + + echo "Verifying the codesign" + codesign -vvv --deep --strict $WFPROXY_ZIP_TO_BE_NOTARIZED + + echo "Uploading the package for Notarization" + response="$(xcrun altool --notarize-app --primary-bundle-id "com.wavefront" --username "$USERNAME" --password "$APP_SPECIFIC_PW" --file "$WFPROXY_ZIP_TO_BE_NOTARIZED" | sed -n '2 p')" + echo $response + + echo "Grabbing Request UUID" + requestuuid=${response#*= } + echo $requestuuid + + echo "Executing this command to see the status of notarization" + xcrun altool --notarization-info "$requestuuid" -u "$USERNAME" -p "$APP_SPECIFIC_PW" +} + +# Pass or fail based on notarization status +wait_for_notarization() { + status="$(xcrun altool --notarization-info "$requestuuid" -u "$USERNAME" -p "$APP_SPECIFIC_PW")" + in_progress='Status: in progress' + success='Status Message: Package Approved' + invalid='Status: invalid' + + while true; + do + echo $status + if [[ "$status" == *"$success"* ]]; then + echo "Successful notarization" + aws s3 cp $WFPROXY_ZIP_TO_BE_NOTARIZED s3://eso-wfproxy-testing/notarized_test/wavefront-proxy-notarized-$VERSION.zip + exit 0 + elif [[ "$status" == *"$in_progress"* ]]; then + status="$(xcrun altool --notarization-info "$requestuuid" -u "$USERNAME" -p "$APP_SPECIFIC_PW")" + sleep 60 + elif [[ "$status" == *"$invalid"* ]]; then + echo "Failed notarization" + exit 1 + fi + done +} + +main() { + create_dev_certs + parse_proxy_version_from_zip + echo $VERSION + notarized_newly_package_proxy + sleep 20 + wait_for_notarization +} + +main \ No newline at end of file diff --git a/macos_proxy_notarization/wfproxy.entitlements b/macos_proxy_notarization/wfproxy.entitlements new file mode 100644 index 000000000..244ba034b --- /dev/null +++ b/macos_proxy_notarization/wfproxy.entitlements @@ -0,0 +1,12 @@ + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-executable-page-protection + + com.apple.security.cs.allow-dyld-environment-variables + + + diff --git a/open_source_licenses.txt b/open_source_licenses.txt index 4a5631d35..efcfc3817 100644 --- a/open_source_licenses.txt +++ b/open_source_licenses.txt @@ -1,6660 +1,6118 @@ -open_source_licenses.txt - -Wavefront by VMware 4.34 GA - -====================================================================== - -The following copyright statements and licenses apply to various open -source software packages (or portions thereof) that are included in -this VMware service. - -The VMware service may also include other VMware components, which may -contain additional open source software packages. One or more such -open_source_licenses.txt files may therefore accompany this VMware -service. - -The VMware service that includes this file does not necessarily use all the open -source software packages referred to below and may also only use portions of a -given package. - -=============== TABLE OF CONTENTS ============================= - -The following is a listing of the open source components detailed in -this document. This list is provided for your convenience; please read -further if you wish to review the copyright notice(s) and the full text -of the license associated with each component. - - -SECTION 1: Apache License, V2.0 - - >>> com.beust:jcommander-1.72 - >>> com.fasterxml.jackson.core:jackson-annotations-2.9.6 - >>> com.fasterxml.jackson.core:jackson-core-2.9.6 - >>> com.fasterxml.jackson.core:jackson-databind-2.9.6 - >>> com.fasterxml.jackson.dataformat:jackson-dataformat-yaml-2.9.6 - >>> com.fasterxml.jackson.datatype:jackson-datatype-guava-2.9.6 - >>> com.fasterxml.jackson.datatype:jackson-datatype-jdk8-2.9.6 - >>> com.fasterxml.jackson.datatype:jackson-datatype-joda-2.9.6 - >>> com.fasterxml.jackson.datatype:jackson-datatype-jsr310-2.9.6 - >>> com.fasterxml.jackson.jaxrs:jackson-jaxrs-base-2.8.6 - >>> com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider-2.8.6 - >>> com.fasterxml.jackson.module:jackson-module-afterburner-2.9.6 - >>> com.fasterxml.jackson.module:jackson-module-jaxb-annotations-2.8.6 - >>> com.fasterxml.jackson.module:jackson-module-parameter-names-2.9.6 - >>> com.fasterxml:classmate-1.3.4 - >>> com.github.ben-manes.caffeine:caffeine-2.6.2 - >>> com.github.tony19:named-regexp-0.2.3 - >>> com.google.code.findbugs:jsr305-2.0.1 - >>> com.google.code.findbugs:jsr305-3.0.0 - >>> com.google.code.gson:gson-2.2.2 - >>> com.google.errorprone:error_prone_annotations-2.1.3 - >>> com.google.guava:guava-26.0-jre - >>> com.google.j2objc:j2objc-annotations-1.1 - >>> com.intellij:annotations-12.0 - >>> com.lmax:disruptor-3.3.7 - >>> com.squareup.okhttp3:okhttp-3.8.1 - >>> com.squareup.okio:okio-1.13.0 - >>> com.squareup:javapoet-1.5.1 - >>> com.squareup:tape-1.2.3 - >>> com.tdunning:t-digest-3.1 - >>> com.tdunning:t-digest-3.2 - >>> commons-codec:commons-codec-1.9 - >>> commons-collections:commons-collections-3.2.2 - >>> commons-daemon-1.0.15 - >>> commons-io:commons-io-2.5 - >>> commons-lang:commons-lang-2.6 - >>> commons-logging:commons-logging-1.2 - >>> io.dropwizard.metrics5:metrics-core-5.0.0-rc2 - >>> io.dropwizard.metrics5:metrics-jvm-5.0.0-rc2 - >>> io.dropwizard.metrics:metrics-core-4.0.2 - >>> io.dropwizard.metrics:metrics-jvm-4.0.2 - >>> io.dropwizard:dropwizard-jackson-1.3.5 - >>> io.dropwizard:dropwizard-lifecycle-1.3.5 - >>> io.dropwizard:dropwizard-metrics-1.3.5 - >>> io.dropwizard:dropwizard-util-1.3.5 - >>> io.dropwizard:dropwizard-validation-1.3.5 - >>> io.jaegertracing:jaeger-core-0.31.0 - >>> io.jaegertracing:jaeger-thrift-0.31.0 - >>> io.netty:netty-buffer-4.1.25.final - >>> io.netty:netty-codec-4.1.17.final - >>> io.netty:netty-codec-4.1.25.final - >>> io.netty:netty-codec-http-4.1.25.final - >>> io.netty:netty-common-4.1.25.final - >>> io.netty:netty-handler-4.1.25.final - >>> io.netty:netty-resolver-4.1.25.final - >>> io.netty:netty-transport-4.1.25.final - >>> io.netty:netty-transport-native-epoll-4.1.17.final - >>> io.netty:netty-transport-native-epoll-4.1.25.final - >>> io.netty:netty-transport-native-unix-common-4.1.25.final - >>> io.opentracing:opentracing-api-0.31.0 - >>> io.opentracing:opentracing-noop-0.31.0 - >>> io.opentracing:opentracing-util-0.31.0 - >>> io.thekraken:grok-0.1.4 - >>> io.zipkin.zipkin2:zipkin-2.11.12 - >>> javax.validation:validation-api-1.1.0.final - >>> jna-platform-4.2.1 - >>> joda-time:joda-time-2.6 - >>> net.jafama:jafama-2.1.0 - >>> net.jpountz.lz4:lz4-1.3.0 - >>> net.openhft:affinity-3.1.7 - >>> net.openhft:chronicle-algorithms-1.15.0 - >>> net.openhft:chronicle-bytes-1.16.21 - >>> net.openhft:chronicle-core-1.16.16 - >>> net.openhft:chronicle-map-3.16.0 - >>> net.openhft:chronicle-threads-1.16.3 - >>> net.openhft:chronicle-wire-1.16.15 - >>> net.openhft:compiler-2.3.1 - >>> org.apache.avro:avro-1.8.2 - >>> org.apache.commons:commons-compress-1.8.1 - >>> org.apache.commons:commons-lang3-3.1 - >>> org.apache.httpcomponents:httpclient-4.5.3 - >>> org.apache.httpcomponents:httpcore-4.4.5 - >>> org.apache.logging.log4j:log4j-1.2-api-2.11.1 - >>> org.apache.logging.log4j:log4j-api-2.11.1 - >>> org.apache.logging.log4j:log4j-core-2.11.1 - >>> org.apache.logging.log4j:log4j-jul-2.11.1 - >>> org.apache.logging.log4j:log4j-slf4j-impl-2.11.1 - >>> org.apache.logging.log4j:log4j-web-2.11.1 - >>> org.apache.thrift:libthrift-0.11.0 - >>> org.codehaus.jackson:jackson-core-asl-1.9.13 - >>> org.codehaus.jackson:jackson-mapper-asl-1.9.13 - >>> org.codehaus.jettison:jettison-1.3.8 - >>> org.eclipse.jetty:jetty-http-9.4.14.v20181114 - >>> org.eclipse.jetty:jetty-io-9.4.14.v20181114 - >>> org.eclipse.jetty:jetty-security-9.4.14.v20181114 - >>> org.eclipse.jetty:jetty-server-9.4.14.v20181114 - >>> org.eclipse.jetty:jetty-servlet-9.4.14.v20181114 - >>> org.eclipse.jetty:jetty-util-9.4.14.v20181114 - >>> org.hibernate:hibernate-validator-5.4.2.final - >>> org.javassist:javassist-3.22.0-ga - >>> org.jboss.logging:jboss-logging-3.3.0.final - >>> org.jboss.resteasy:resteasy-client-3.0.21.final - >>> org.jboss.resteasy:resteasy-jackson2-provider-3.0.21.final - >>> org.jboss.resteasy:resteasy-jaxrs-3.0.21.final - >>> org.ops4j.pax.url:pax-url-aether-2.5.2 - >>> org.ops4j.pax.url:pax-url-aether-support-2.5.2 - >>> org.slf4j:jcl-over-slf4j-1.7.25 - >>> org.xerial.snappy:snappy-java-1.1.1.3 - >>> org.yaml:snakeyaml-1.17 - - -SECTION 2: BSD-STYLE, MIT-STYLE, OR SIMILAR STYLE LICENSES - - >>> com.thoughtworks.paranamer:paranamer-2.7 - >>> com.thoughtworks.xstream:xstream-1.4.10 - >>> com.uber.tchannel:tchannel-core-0.8.5 - >>> com.yammer.metrics:metrics-core-2.2.0 - >>> io.dropwizard.metrics:metrics-core-3.1.2 - >>> net.razorvine:pyrolite-4.10 - >>> net.razorvine:serpent-1.12 - >>> org.antlr:antlr4-runtime-4.7.1 - >>> org.checkerframework:checker-qual-2.5.2 - >>> org.codehaus.mojo:animal-sniffer-annotations-1.14 - >>> org.hamcrest:hamcrest-all-1.3 - >>> org.json:json-20160212 - >>> org.slf4j:slf4j-api-1.7.25 - >>> org.tukaani:xz-1.5 - >>> xmlpull:xmlpull-1.1.3.1 - >>> xpp3:xpp3_min-1.1.4c - - -SECTION 3: Common Development and Distribution License, V1.0 - - >>> org.jboss.spec.javax.annotation:jboss-annotations-api_1.2_spec-1.0.0.final - - -SECTION 4: Common Development and Distribution License, V1.1 - - >>> javax.activation:activation-1.1.1 - >>> javax.annotation:javax.annotation-api-1.2 - >>> javax.servlet:javax.servlet-api-3.1.0 - >>> javax.ws.rs:javax.ws.rs-api-2.0.1 - >>> org.glassfish:javax.el-3.0.0 - >>> org.jboss.spec.javax.ws.rs:jboss-jaxrs-api_2.0_spec-1.0.1.beta1 - - -SECTION 5: Creative Commons Attribution 2.5 - - >>> net.jcip:jcip-annotations-1.0 - - -SECTION 6: Eclipse Public License, V1.0 - - >>> org.eclipse.aether:aether-api-1.1.0 - >>> org.eclipse.aether:aether-impl-1.1.0 - >>> org.eclipse.aether:aether-spi-1.1.0 - >>> org.eclipse.aether:aether-util-1.1.0 - - -SECTION 7: GNU Lesser General Public License, V2.1 - - >>> jna-4.2.1 - - -SECTION 8: GNU Lesser General Public License, V3.0 - - >>> net.openhft:chronicle-values-1.16.0 - - -APPENDIX. Standard License Files - - >>> Apache License, V2.0 - - >>> Creative Commons Attribution 2.5 - - >>> Common Development and Distribution License, V1.0 - - >>> Common Development and Distribution License, V1.1 - - >>> Eclipse Public License, V1.0 - - >>> GNU Lesser General Public License, V2.1 - - >>> GNU Lesser General Public License, V3.0 - - >>> Artistic License, V1.0 - - >>> Mozilla Public License, V2.0 - - >>> GNU General Public License, V2.0 - - >>> Eclipse Public License, V2.0 - - - ---------------- SECTION 1: Apache License, V2.0 ---------- - -Apache License, V2.0 is applicable to the following component(s). - - ->>> com.beust:jcommander-1.72 - -Copyright (C) 2010 the original author or authors. -See the notice.md file distributed with this work for additional -information regarding copyright ownership. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - ->>> com.fasterxml.jackson.core:jackson-annotations-2.9.6 - -This copy of Jackson JSON processor annotations is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - ->>> com.fasterxml.jackson.core:jackson-core-2.9.6 - -Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -Licensing - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -This copy of Jackson JSON processor streaming parser/generator is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - ->>> com.fasterxml.jackson.core:jackson-databind-2.9.6 - -Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -Licensing - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -This copy of Jackson JSON processor databind module is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - ->>> com.fasterxml.jackson.dataformat:jackson-dataformat-yaml-2.9.6 - -Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -Licensing - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -This copy of Jackson JSON processor YAML module is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - ->>> com.fasterxml.jackson.datatype:jackson-datatype-guava-2.9.6 - -License : The Apache Software License, Version 2.0 - ->>> com.fasterxml.jackson.datatype:jackson-datatype-jdk8-2.9.6 - -License: Apache2.0 - ->>> com.fasterxml.jackson.datatype:jackson-datatype-joda-2.9.6 - -This copy of Jackson JSON processor streaming parser/generator is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - ->>> com.fasterxml.jackson.datatype:jackson-datatype-jsr310-2.9.6 - -This copy of Jackson JSON processor streaming parser/generator is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - ->>> com.fasterxml.jackson.jaxrs:jackson-jaxrs-base-2.8.6 - -This copy of Jackson JSON processor databind module is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - ->>> com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider-2.8.6 - -This copy of Jackson JSON processor databind module is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - ->>> com.fasterxml.jackson.module:jackson-module-afterburner-2.9.6 - -License : The Apache Software License, Version 2.0 - ->>> com.fasterxml.jackson.module:jackson-module-jaxb-annotations-2.8.6 - -This copy of Jackson JSON processor databind module is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - ->>> com.fasterxml.jackson.module:jackson-module-parameter-names-2.9.6 - -License: Apache 2.0 - ->>> com.fasterxml:classmate-1.3.4 - -Java ClassMate library was originally written by Tatu Saloranta (tatu.saloranta@iki.fi) - -Other developers who have contributed code are: - -* Brian Langel - -This copy of Java ClassMate library is licensed under Apache (Software) License, -version 2.0 ("the License"). -See the License for details about distribution rights, and the specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 - ->>> com.github.ben-manes.caffeine:caffeine-2.6.2 - -Copyright 2018 Ben Manes. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http:www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - ->>> com.github.tony19:named-regexp-0.2.3 - -Copyright (C) 2012-2013 The named-regexp Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - ->>> com.google.code.findbugs:jsr305-2.0.1 - -License: Apache 2.0 - ->>> com.google.code.findbugs:jsr305-3.0.0 - -License : Apache 2.0 - ->>> com.google.code.gson:gson-2.2.2 - -Copyright (C) 2008 Google Inc. + +Wavefront by VMware 14.0 GA +====================================================================== + +The following copyright statements and licenses apply to open source +software ("OSS") distributed with the Broadcom product (the "Licensed +Product"). The term "Broadcom" refers solely to the Broadcom Inc. +corporate affiliate that distributes the Licensed Product. The +Licensed Product does not necessarily use all the OSS referred to +below and may also only use portions of a given OSS component. + +To the extent required under an applicable open source license, +Broadcom will make source code available for applicable OSS upon +request. Please send an inquiry to opensource@broadcom.com including +your name, address, the product name and version, operating system, +and the place of purchase. + +To the extent the Licensed Product includes OSS, the OSS is typically +not owned by Broadcom. THE OSS IS PROVIDED AS IS WITHOUT WARRANTY OR +CONDITION OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT +LIMITATION, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE. To the full extent permitted under applicable +law, Broadcom and its corporate affiliates disclaim all warranties +and liability arising from or related to any use of the OSS. + +To the extent the Licensed Product includes OSS licensed under the +GNU General Public License ("GPL") or the GNU Lesser General Public +License (“LGPL”), the use, copying, distribution and modification of +the GPL OSS or LGPL OSS is governed, respectively, by the GPL or LGPL. +A copy of the GPL or LGPL license may be found with the applicable OSS. +Additionally, a copy of the GPL License or LGPL License can be found +at https://www.gnu.org/licenses or obtained by writing to the Free +Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +MA 02111-1307 USA. + +==================== TABLE OF CONTENTS ==================== + +The following is a listing of the open source components detailed in +this document. This list is provided for your convenience; please read +further if you wish to review the copyright notice(s) and the full text +of the license associated with each component. + +PART 1. APPLICATION LAYER + +SECTION 1: Apache License, V2.0 + + >>> org.apache.commons:commons-lang3-3.1 + >>> commons-lang:commons-lang-2.6 + >>> com.github.fge:msg-simple-1.1 + >>> com.github.fge:btf-1.2 + >>> commons-logging:commons-logging-1.2 + >>> com.github.fge:json-patch-1.9 + >>> com.github.fge:jackson-coreutils-1.6 + >>> com.github.stephenc.jcip:jcip-annotations-1.0-1 + >>> commons-collections:commons-collections-3.2.2 + >>> net.jpountz.lz4:lz4-1.3.0 + >>> software.amazon.ion:ion-java-1.0.2 + >>> javax.validation:validation-api-1.1.0.final + >>> org.jetbrains:annotations-13.0 + >>> io.thekraken:grok-0.1.5 + >>> com.tdunning:t-digest-3.2 + >>> io.dropwizard.metrics5:metrics-core-5.0.0-rc2 + >>> com.google.guava:listenablefuture-9999.0-empty-to-avoid-conflict-with-guava + >>> com.google.guava:failureaccess-1.0.1 + >>> com.google.android:annotations-4.1.1.4 + >>> com.google.j2objc:j2objc-annotations-1.3 + >>> io.opentracing:opentracing-noop-0.33.0 + >>> org.apache.commons:commons-collections4-4.4 + >>> io.opentracing:opentracing-api-0.33.0 + >>> jakarta.validation:jakarta.validation-api-2.0.2 + >>> org.jboss.logging:jboss-logging-3.4.1.final + >>> io.opentracing:opentracing-util-0.33.0 + >>> net.java.dev.jna:jna-5.5.0 + >>> org.apache.httpcomponents:httpcore-4.4.13 + >>> net.java.dev.jna:jna-platform-5.5.0 + >>> com.squareup.tape2:tape-2.0.0-beta1 + >>> com.beust:jcommander-1.78 + >>> com.github.java-json-tools:jackson-coreutils-2.0 + >>> com.github.java-json-tools:msg-simple-1.2 + >>> com.github.java-json-tools:json-patch-1.13 + >>> commons-codec:commons-codec-1.15 + >>> com.github.java-json-tools:btf-1.3 + >>> org.apache.httpcomponents:httpclient-4.5.13 + >>> org.elasticsearch:jna-5.5.0 + >>> com.squareup.okio:okio-2.8.0 + >>> com.ibm.async:asyncutil-0.1.0 + >>> com.google.api.grpc:proto-google-common-protos-2.0.1 + >>> org.objenesis:objenesis-3.2 + >>> net.openhft:compiler-2.21ea1 + >>> com.lmax:disruptor-3.4.4 + >>> commons-io:commons-io-2.11.0 + >>> org.apache.commons:commons-compress-1.21 + >>> com.wavefront:wavefront-sdk-java-3.0.0 + >>> org.apache.avro:avro-1.11.0 + >>> com.github.ben-manes.caffeine:caffeine-2.9.3 + >>> com.wavefront:wavefront-internal-reporter-java-1.7.10 + >>> org.jboss.logging:jboss-logging-3.4.3.final + >>> org.yaml:snakeyaml-1.30 + >>> com.squareup.okhttp3:okhttp-4.9.3 + >>> com.google.code.gson:gson-2.9.0 + >>> io.jaegertracing:jaeger-thrift-1.8.0 + >>> io.jaegertracing:jaeger-core-1.8.0 + >>> org.apache.logging.log4j:log4j-jul-2.17.2 + >>> org.apache.logging.log4j:log4j-core-2.17.2 + >>> org.apache.logging.log4j:log4j-api-2.17.2 + >>> org.apache.logging.log4j:log4j-slf4j-impl-2.17.2 + >>> org.jboss.resteasy:resteasy-client-3.15.3.Final + >>> org.jboss.resteasy:resteasy-jaxrs-3.15.3.Final + >>> org.jboss.resteasy:resteasy-jackson2-provider-3.15.3.Final + >>> joda-time:joda-time-2.10.14 + >>> com.beust:jcommander-1.82 + >>> io.netty:netty-transport-classes-epoll-4.1.77.final + >>> io.netty:netty-transport-native-unix-common-4.1.77.final + >>> com.fasterxml.jackson.core:jackson-core-2.13.3 + >>> com.fasterxml.jackson.core:jackson-annotations-2.13.3 + >>> com.fasterxml.jackson.core:jackson-databind-2.13.3 + >>> com.fasterxml.jackson.dataformat:jackson-dataformat-yaml-2.13.3 + >>> com.fasterxml.jackson.dataformat:jackson-dataformat-cbor-2.13.3 + >>> com.fasterxml.jackson.jaxrs:jackson-jaxrs-base-2.13.3 + >>> com.fasterxml.jackson.module:jackson-module-jaxb-annotations-2.13.3 + >>> com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider-2.13.3 + >>> org.jetbrains.kotlin:kotlin-stdlib-common-1.6.21 + >>> commons-daemon:commons-daemon-1.3.1 + >>> com.google.errorprone:error_prone_annotations-2.14.0 + >>> com.fasterxml.jackson.module:jackson-module-afterburner-2.13.3 + >>> net.openhft:chronicle-analytics-2.21ea0 + >>> io.grpc:grpc-context-1.46.0 + >>> io.opentelemetry.proto:opentelemetry-proto-0.17.0-alpha + >>> io.grpc:grpc-api-1.46.0 + >>> net.openhft:chronicle-bytes-2.21.89 + >>> net.openhft:chronicle-map-3.21.86 + >>> io.zipkin.zipkin2:zipkin-2.23.16 + >>> io.grpc:grpc-core-1.46.0 + >>> net.openhft:chronicle-threads-2.21.85 + >>> net.openhft:chronicle-wire-2.21.89 + >>> io.grpc:grpc-stub-1.46.0 + >>> net.openhft:chronicle-core-2.21.91 + >>> io.perfmark:perfmark-api-0.25.0 + >>> org.apache.thrift:libthrift-0.16.0 + >>> com.amazonaws:aws-java-sdk-sqs-1.12.229 + >>> net.openhft:chronicle-values-2.21.82 + >>> com.amazonaws:aws-java-sdk-core-1.12.229 + >>> org.jetbrains.kotlin:kotlin-stdlib-1.7.0-RC + >>> net.openhft:chronicle-algorithms-2.21.82 + >>> io.opentelemetry:opentelemetry-exporter-jaeger-proto-1.14.0 + >>> io.grpc:grpc-netty-1.46.0 + >>> com.squareup:javapoet-1.13.0 + >>> net.jafama:jafama-2.3.2 + >>> net.openhft:affinity-3.21ea5 + >>> io.grpc:grpc-protobuf-lite-1.46.0 + >>> io.grpc:grpc-protobuf-1.46.0 + >>> org.yaml:snakeyaml-1.33 + >>> io.zipkin.zipkin2:zipkin-2.23.14 + >>> com.fasterxml.jackson.dataformat:jackson-dataformat-cbor-2.14.0 + >>> com.fasterxml.jackson.core:jackson-core-2.14.0 + >>> com.fasterxml.jackson.core:jackson-annotations-2.14.0 + >>> com.fasterxml.jackson.core:jackson-databind-2.14.0 + >>> com.fasterxml.jackson.dataformat:jackson-dataformat-yaml-2.14.0 + >>> com.wavefront:java-lib-2022-10.1 + >>> no.ssb.jpms:jsr305-and-javax.annotation-api-1.0 + >>> com.fasterxml.jackson.module:jackson-module-jaxb-annotations-2.14.0 + >>> com.wavefront:proxy-12.0 + >>> com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider-2.14.0 + >>> com.fasterxml.jackson.jaxrs:jackson-jaxrs-base-2.14.0 + >>> com.fasterxml.jackson.module:jackson-module-afterburner-2.14.0 + >>> me.him188:kotlin-jvm-blocking-bridge-compiler-2.1.0-170.1 + >>> com.wavefront:wavefront-internal-reporter-java-1.7.13 + >>> com.wavefront:java-lib-2022-11.1 + >>> com.google.code.gson:gson-2.10.1 + >>> Jetbrains annotations-13.0 + >>> com.wavefront:wavefront-internal-reporter-java-1.7.16 + >>> io.jaegertracing:jaeger-core-1.8.1 + >>> io.jaegertracing:jaeger-thrift-1.8.1 + >>> com.wavefront:proxy-12.1 + >>> org.springframework.boot:spring-boot-loader-tools-2.7.0 + >>> io.perfmark:perfmark-api-0.26.0 + >>> com.turbospaces.boot:bootstrap-protobuf-core-1.0.113 + >>> org.springframework.boot:spring-boot-starter-log4j2-2.7.12 + >>> com.google.j2objc:j2objc-annotations-2.8 + >>> com.fasterxml.jackson.core:jackson-core-2.15.2 + >>> com.fasterxml.jackson.core:jackson-annotations-2.15.2 + >>> com.fasterxml.jackson.dataformat:jackson-dataformat-cbor-2.15.2 + >>> com.fasterxml.jackson.datatype:jackson-datatype-jsr310-2.15.2 + >>> com.fasterxml.jackson.core:jackson-databind-2.15.2 + >>> com.fasterxml.jackson.dataformat:jackson-dataformat-yaml-2.15.2 + >>> org.jetbrains.kotlin:kotlin-stdlib-jdk7-1.8.21 + >>> org.jetbrains.kotlin:kotlin-stdlib-jdk8-1.8.21 + >>> org.springframework.boot:spring-boot-loader-2.7.12 + >>> org.springframework.boot:spring-boot-jarmode-layertools-2.7.12 + >>> com.fasterxml.jackson.module:jackson-module-jaxb-annotations-2.15.2 + >>> com.fasterxml.jackson.jaxrs:jackson-jaxrs-base-2.15.2 + >>> com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider-2.15.2 + >>> commons-daemon:commons-daemon-1.3.4 + >>> ua.parser.wso2:ua-parser-1.5.4.wso2v1 + >>> beanvalidation-api-1.1.0 + >>> org.yaml:snakeyaml-2.2 + >>> com.google.guava:guava-v30.1.1 + >>> com.wavefront:proxy-12.2 + >>> com.wavefront:java-lib-2023-08.2 + >>> com.squareup.okio:okio-3.6.0 + >>> com.squareup.okio:okio-jvm-3.6.0 + >>> org.apache.avro:avro-1.11.4 + >>> io.netty:netty-buffer-4.1.77 + >>> io.netty:netty-codec-http2-4.1.77 + >>> io.netty:netty-codec-http-4.1.77 + >>> io.netty:netty-codec-socks-4.1.77 + >>> io.netty:netty-codec-4.1.77 + >>> io.netty:netty-common-4.1.77 + >>> io.netty:netty-handler-proxy-4.1.77 + >>> io.netty:netty-handler-4.1.77 + >>> io.netty:netty-resolver-4.1.77 + >>> io.netty:netty-transport-native-epoll-4.1.77 + >>> io.netty:netty-transport-4.1.77 + >>> com.wavefront:wavefront-sdk-java-3.4.3 + >>> org.apache.commons:commons-lang3-3.14.0 + >>> commons-io:commons-io-2.15.1 + >>> com.wavefront:proxy-13.4 + >>> com.squareup.okhttp3:okhttp-4.12.0 + >>> commons-logging:commons-logging-1.3.0 + >>> com.google.guava:failureaccess-1.0.2 + >>> com.google.guava:guava-33.0.0-jre + >>> com.toasttab.protokt:protokt-core-1.0.0-beta.0 + >>> commons-codec:commons-codec-1.16.1 + >>> org.apache.commons:commons-compress-1.26.0 + >>> org.apache.logging.log4j:log4j-api-2.23.0 + >>> org.apache.logging.log4j:log4j-core-2.23.0 + >>> com.amazonaws:aws-java-sdk-core-1.12.667 + >>> com.amazonaws:jmespath-java-1.12.667 + >>> com.amazonaws:aws-java-sdk-sqs-1.12.667 + >>> org.apache.logging.log4j:log4j-slf4j-impl-2.23.0 + >>> org.jboss.resteasy:resteasy-jackson2-provider-5.0.9.Final + >>> org.jboss.resteasy:resteasy-client-api-5.0.9.Final + >>> io.grpc:grpc-protobuf-1.62.2 + >>> io.grpc:grpc-core-1.62.2 + >>> io.grpc:grpc-protobuf-lite-1.62.2 + >>> org.apache.logging.log4j:log4j-jul-2.23.0 + >>> org.jboss.resteasy:resteasy-core-5.0.9.Final + >>> io.opentelemetry:opentelemetry-exporter-jaeger-proto-1.17.0 + >>> io.grpc:grpc-netty-1.62.2 + >>> com.google.errorprone:error_prone_annotations-2.25.0 + >>> io.grpc:grpc-api-1.62.2 + >>> io.grpc:grpc-context-1.62.2 + >>> org.jboss.resteasy:resteasy-core-spi-5.0.9.Final + >>> io.grpc:grpc-stub-1.62.2 + >>> io.grpc:grpc-util-1.62.2 + >>> org.jboss.resteasy:resteasy-client-5.0.9.Final + >>> org.apache.tomcat:tomcat-annotations-api-9.0.87 + >>> io.netty:netty-transport-native-unix-common-4.1.108.Final + >>> io.netty:netty-transport-classes-epoll-4.1.108.Final + >>> io.netty:netty-buffer-4.1.108 + >>> io.netty:netty-codec-http2-4.1.108 + >>> io.netty:netty-codec-http-4.1.108 + >>> io.netty:netty-codec-socks-4.1.108 + >>> io.netty:netty-codec-4.1.108 + >>> io.netty:netty-common-4.1.108 + >>> io.netty:netty-handler-proxy-4.1.108 + >>> io.netty:netty-handler-4.1.108 + >>> io.netty:netty-resolver-4.1.108 + >>> io.netty:netty-transport-native-epoll-4.1.108 + >>> io.netty:netty-transport-4.1.108 + >>> dev.ikm.jpms:commons-lang3-3.14.0-r1 + >>> GoogleGson-2.9.0 + >>> com.wavefront:java-lib-2022-01.31-QA + >>> me.him188:kotlin-jvm-blocking-bridge-compiler-embeddable-2.1.0-170.1 + +SECTION 2: BSD-STYLE, MIT-STYLE, OR SIMILAR STYLE LICENSES + + >>> com.yammer.metrics:metrics-core-2.2.0 + >>> org.hamcrest:hamcrest-all-1.3 + >>> net.razorvine:pyrolite-4.10 + >>> net.razorvine:serpent-1.12 + >>> backport-util-concurrent:backport-util-concurrent-3.1 + >>> org.antlr:antlr4-runtime-4.7.2 + >>> org.slf4j:slf4j-api-1.8.0-beta4 + >>> com.sun.activation:jakarta.activation-1.2.1 + >>> jakarta.activation:jakarta.activation-api-1.2.1 + >>> org.reactivestreams:reactive-streams-1.0.3 + >>> jakarta.activation:jakarta.activation-api-1.2.2 + >>> com.sun.activation:jakarta.activation-1.2.2 + >>> jakarta.xml.bind:jakarta.xml.bind-api-2.3.3 + >>> com.rubiconproject.oss:jchronic-0.2.8 + >>> org.codehaus.mojo:animal-sniffer-annotations-1.19 + >>> org.jboss.spec.javax.xml.bind:jboss-jaxb-api_2.3_spec-2.0.1.Final + >>> org.slf4j:jul-to-slf4j-1.7.36 + >>> com.uber.tchannel:tchannel-core-0.8.30 + >>> com.google.re2j:re2j-1.6 + >>> org.checkerframework:checker-qual-3.22.0 + >>> com.google.protobuf:protobuf-java-3.21.0 + >>> com.google.protobuf:protobuf-java-util-3.21.0 + >>> org.reactivestreams:reactive-streams-1.0.4 + >>> dk.brics:automaton-1.12-4 + >>> Go programming language-20220411-snapshot + >>> com.bybutter.sisyphus:sisyphus-grpc-coroutine-1.4.0 + >>> com.bybutter.sisyphus:sisyphus-grpc-coroutine-1.2.10 + >>> com.google.protobuf:protobuf-java-3.21.12 + >>> com.google.protobuf:protobuf-java-util-3.21.12 + >>> org.codehaus.mojo:animal-sniffer-annotations-1.23 + >>> com.google.protobuf:protobuf-java-3.25.3 + >>> com.google.protobuf:protobuf-java-util-3.25.3 + +SECTION 3: Common Development and Distribution License, V1.1 + + >>> javax.annotation:javax.annotation-api-1.3.2 + +SECTION 4: Creative Commons Attribution 2.5 + + >>> com.google.code.findbugs:jsr305-3.0.2 + +SECTION 5: Eclipse Public License, V1.0 + + >>> org.jacoco:jacoco-0.7.6 + +SECTION 6: Eclipse Public License, V2.0 + + >>> javax.ws.rs:javax.ws.rs-api-2.1.1 + >>> org.jboss.spec.javax.ws.rs:jboss-jaxrs-api_2.1_spec-2.0.1.final + >>> org.jboss.spec.javax.annotation:jboss-annotations-api_1.3_spec-2.0.1.final + >>> jakarta.annotation:jakarta.annotation-api-1.3.5 + >>> org.jboss.spec.javax.ws.rs:jboss-jaxrs-api_2.1_spec-2.0.2.Final + +SECTION 7: GNU Lesser General Public License v3 or later + + >>> com.github.fge:json-patch-1.13 + + +APPENDIX. Standard License Files + + >>> Apache License, V2.0 + >>> Creative Commons Attribution 2.5 + >>> Common Development and Distribution License, V1.1 + >>> Eclipse Public License, V2.0 + >>> Eclipse Public License, V1.0 + >>> GNU Lesser General Public License v3 or later + >>> GNU Lesser General Public License, V3.0 + +==================== PART 1. APPLICATION LAYER ==================== + +-------------------- SECTION 1: Apache License, V2.0 -------------------- + + >>> org.apache.commons:commons-lang3-3.1 + + Apache Commons Lang + + Copyright 2001-2011 The Apache Software Foundation + + + + This product includes software developed by + + The Apache Software Foundation (http://www.apache.org/). + + + + This product includes software from the Spring Framework, + + under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + + Licensed to the Apache Software Foundation (ASF) under one or more + + contributor license agreements. See the NOTICE file distributed with + + this work for additional information regarding copyright ownership. + + The ASF licenses this file to You under the Apache License, Version 2.0 + + (the "License"); you may not use this file except in compliance with + + the License. You may obtain a copy of the License at + + + + http://www.apache.org/licenses/LICENSE-2.0 + + + + Unless required by applicable law or agreed to in writing, software + + distributed under the License is distributed on an "AS IS" BASIS, + + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + See the License for the specific language governing permissions and + + limitations under the License. + + + >>> commons-lang:commons-lang-2.6 + + Apache Commons Lang + + Copyright 2001-2011 The Apache Software Foundation + + + + This product includes software developed by + + The Apache Software Foundation (http://www.apache.org/). + + + + Licensed to the Apache Software Foundation (ASF) under one or more + + contributor license agreements. See the NOTICE file distributed with + + this work for additional information regarding copyright ownership. + + The ASF licenses this file to You under the Apache License, Version 2.0 + + (the "License"); you may not use this file except in compliance with + + the License. You may obtain a copy of the License at + + + + http://www.apache.org/licenses/LICENSE-2.0 + + + + Unless required by applicable law or agreed to in writing, software + + distributed under the License is distributed on an "AS IS" BASIS, + + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + See the License for the specific language governing permissions and + + limitations under the License. + + + >>> com.github.fge:msg-simple-1.1 + + [PLEASE NOTE: BROADCOM ELECTS TO USE AND DISTRIBUTE THIS COMPONENT UNDER THE TERMS OF THE APACHE 2.0 LICENSE. PLEASE SEE APPENDIX FOR THE FULL TEXT OF THE APACHE 2.0 LICENSE. THE ORIGINAL LICENSE TERMS ARE REPRODUCED BELOW ONLY AS A REFERENCE.] + + + + This software is dual-licensed under: + + + + - the Lesser General Public License (LGPL) version 3.0 or, at your option, any + + later version; + + - the Apache Software License (ASL) version 2.0. + + + + The text of both licenses is included (under the names LGPL-3.0.txt and + + ASL-2.0.txt respectively). + + + + Direct link to the sources: + + + + - LGPL 3.0: https://www.gnu.org/licenses/lgpl-3.0.txt + + - ASL 2.0: http://www.apache.org/licenses/LICENSE-2.0.txt + + + >>> com.github.fge:btf-1.2 + + [PLEASE NOTE: BROADCOM ELECTS TO USE AND DISTRIBUTE THIS COMPONENT UNDER THE TERMS OF THE APACHE 2.0 LICENSE. PLEASE SEE THE APPENDIX FOR THE FULL TEXT OF THE APACHE 2.0 LICENSE. THE ORIGINAL LICENSE TERMS ARE REPRODUCED BELOW ONLY AS A REFERENCE.] + + + + This software is dual-licensed under: + + + + - the Lesser General Public License (LGPL) version 3.0 or, at your option, any + + later version; + + - the Apache Software License (ASL) version 2.0. + + + + The text of both licenses is included (under the names LGPL-3.0.txt and + + ASL-2.0.txt respectively). + + + + Direct link to the sources: + + + + - LGPL 3.0: https://www.gnu.org/licenses/lgpl-3.0.txt + + - ASL 2.0: http://www.apache.org/licenses/LICENSE-2.0.txt + + + >>> commons-logging:commons-logging-1.2 + + Apache Commons Logging + + Copyright 2003-2014 The Apache Software Foundation + + + + This product includes software developed at + + The Apache Software Foundation (http://www.apache.org/). + + + + Licensed to the Apache Software Foundation (ASF) under one or more + + contributor license agreements. See the NOTICE file distributed with + + this work for additional information regarding copyright ownership. + + The ASF licenses this file to You under the Apache License, Version 2.0 + + (the "License"); you may not use this file except in compliance with + + the License. You may obtain a copy of the License at + + + + http://www.apache.org/licenses/LICENSE-2.0 + + + + Unless required by applicable law or agreed to in writing, software + + distributed under the License is distributed on an "AS IS" BASIS, + + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + See the License for the specific language governing permissions and + + limitations under the License. + + + + This software consists of voluntary contributions made by many + + individuals on behalf of the Apache Software Foundation. For more + + information on the Apache Software Foundation, please see + + . + + + >>> com.github.fge:json-patch-1.9 + + [PLEASE NOTE: BROADCOM ELECTS TO USE AND DISTRIBUTE THIS COMPONENT UNDER THE TERMS OF APACHE 2.0. PLEASE SEE THE APPENDIX TO REVIEW THE FULL TEXT OF THE APACHE 2.0. THE ORIGINAL LICENSE TERMS ARE REPRODUCED BELOW ONLY AS A REFERENCE.] + + Copyright (c) 2014, Francis Galiegue (fgaliegue@gmail.com) + + This software is dual-licensed under: + + - the Lesser General Public License (LGPL) version 3.0 or, at your option, any + later version; + - the Apache Software License (ASL) version 2.0. + + The text of this file and of both licenses is available at the root of this + project or, if you have the jar distribution, in directory META-INF/, under + the names LGPL-3.0.txt and ASL-2.0.txt respectively. + + Direct link to the sources: + + - LGPL 3.0: https://www.gnu.org/licenses/lgpl-3.0.txt + - ASL 2.0: http://www.apache.org/licenses/LICENSE-2.0.txt + + + >>> com.github.fge:jackson-coreutils-1.6 + + [PLEASE NOTE: BROADCOM ELECTS TO USE AND DISTRIBUTE THIS COMPONENT UNDER THE TERMS OF THE APACHE 2.0. PLEASE SEE THE APPENDIX TO REVIEW THE FULL TEXT OF THE APACHE 2.0. THE ORIGINAL LICENSE TERMS ARE REPRODUCED BELOW ONLY AS A REFERENCE.] + + + + Copyright (c) 2014, Francis Galiegue (fgaliegue@gmail.com) + + + + This software is dual-licensed under: + + + + - the Lesser General Public License (LGPL) version 3.0 or, at your option, any + + later version; + + - the Apache Software License (ASL) version 2.0. + + + + The text of this file and of both licenses is available at the root of this + + project or, if you have the jar distribution, in directory META-INF/, under + + the names LGPL-3.0.txt and ASL-2.0.txt respectively. + + + + Direct link to the sources: + + + + - LGPL 3.0: https://www.gnu.org/licenses/lgpl-3.0.txt + + - ASL 2.0: http://www.apache.org/licenses/LICENSE-2.0.txt + + + >>> com.github.stephenc.jcip:jcip-annotations-1.0-1 + + Copyright 2013 Stephen Connolly. + + + + Licensed under the Apache License, Version 2.0 (the "License"); + + you may not use this file except in compliance with the License. + + You may obtain a copy of the License at + + + + http://www.apache.org/licenses/LICENSE-2.0 + + + + Unless required by applicable law or agreed to in writing, software + + distributed under the License is distributed on an "AS IS" BASIS, + + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + See the License for the specific language governing permissions and + + limitations under the License. + + + >>> commons-collections:commons-collections-3.2.2 + + Apache Commons Collections + + Copyright 2001-2015 The Apache Software Foundation + + + + This product includes software developed by + + The Apache Software Foundation (http://www.apache.org/). + + + + Licensed to the Apache Software Foundation (ASF) under one or more + + contributor license agreements. See the NOTICE file distributed with + + this work for additional information regarding copyright ownership. + + The ASF licenses this file to You under the Apache License, Version 2.0 + + (the "License"); you may not use this file except in compliance with + + the License. You may obtain a copy of the License at + + + + http://www.apache.org/licenses/LICENSE-2.0 + + + + Unless required by applicable law or agreed to in writing, software + + distributed under the License is distributed on an "AS IS" BASIS, + + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + See the License for the specific language governing permissions and + + limitations under the License. + + + >>> net.jpountz.lz4:lz4-1.3.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + + you may not use this file except in compliance with the License. + + You may obtain a copy of the License at + + + + http://www.apache.org/licenses/LICENSE-2.0 + + + + Unless required by applicable law or agreed to in writing, software + + distributed under the License is distributed on an "AS IS" BASIS, + + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + See the License for the specific language governing permissions and + + limitations under the License. + + + >>> software.amazon.ion:ion-java-1.0.2 + + Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + A copy of the License is located at: + + http://aws.amazon.com/apache2.0/ + + or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. + + + >>> javax.validation:validation-api-1.1.0.final + + Copyright 2012-2013, Red Hat, Inc. and/or its affiliates, and individual contributors + by the @authors tag. See the copyright.txt in the distribution for a + full listing of individual contributors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> org.jetbrains:annotations-13.0 + + Copyright 2000-2013 JetBrains s.r.o. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> io.thekraken:grok-0.1.5 + + Copyright 2014 Anthony Corbacho and contributors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> com.tdunning:t-digest-3.2 + + Licensed to Ted Dunning under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> io.dropwizard.metrics5:metrics-core-5.0.0-rc2 + + License: Apache 2.0 + + + >>> com.google.guava:listenablefuture-9999.0-empty-to-avoid-conflict-with-guava + + License : Apache 2.0 + + + >>> com.google.guava:failureaccess-1.0.1 + + Copyright (C) 2018 The Guava Authors + + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + + + >>> com.google.android:annotations-4.1.1.4 + + Copyright (C) 2012 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> com.google.j2objc:j2objc-annotations-1.3 + + Copyright 2012 Google Inc. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> io.opentracing:opentracing-noop-0.33.0 + + Copyright 2016-2019 The OpenTracing Authors + + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + + + >>> org.apache.commons:commons-collections4-4.4 + + Apache Commons Collections + Copyright 2001-2019 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> io.opentracing:opentracing-api-0.33.0 + + Copyright 2016-2019 The OpenTracing Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> jakarta.validation:jakarta.validation-api-2.0.2 + + License: Apache License, Version 2.0 + + See the license.txt file in the root directory or . + + + >>> org.jboss.logging:jboss-logging-3.4.1.final + + JBoss, Home of Professional Open Source. + + Copyright 2013 Red Hat, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> io.opentracing:opentracing-util-0.33.0 + + Copyright 2016-2019 The OpenTracing Authors + + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + + + >>> net.java.dev.jna:jna-5.5.0 + + [PLEASE NOTE: BROADCOM ELECTS TO USE AND DISTRIBUTE THIS COMPONENT UNDER THE TERMS OF THE Apache2.0 LICENSE. PLEASE SEE APPENDIX FOR THE FULL TEXT OF THE Apache2.0 LICENSE. THE ORIGINAL LICENSE TERMS ARE REPRODUCED BELOW ONLY AS A REFERENCE.] + + Copyright (c) 2007 Timothy Wall, All Rights Reserved + + The contents of this file is dual-licensed under 2 + alternative Open Source/Free licenses: LGPL 2.1 or later and + Apache License 2.0. (starting with JNA version 4.0.0). + + You can freely decide which license you want to apply to + the project. + + You may obtain a copy of the LGPL License at: + + http://www.gnu.org/licenses/licenses.html + + A copy is also included in the downloadable source code package + containing JNA, in file "LGPL2.1". + + You may obtain a copy of the Apache License at: + + http://www.apache.org/licenses/ + + A copy is also included in the downloadable source code package + containing JNA, in file "AL2.0". + + + >>> org.apache.httpcomponents:httpcore-4.4.13 + + Apache HttpCore + Copyright 2005-2019 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + ==================================================================== + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + ==================================================================== + + This software consists of voluntary contributions made by many + individuals on behalf of the Apache Software Foundation. For more + information on the Apache Software Foundation, please see + . + + + >>> net.java.dev.jna:jna-platform-5.5.0 + + [NOTE; BROADCOM ELECTS TO USE AND DISTRIBUTE THIS PACKAGE UNDER THE TERMS OF THE APACHE 2.0 LICENSE, THE TERMS OF WHICH ARE SET FORTH BELOW. THE ORIGINAL LICENSE TERMS ARE REPRODUCED BELOW ONLY AS A REFERENCE.] + + Copyright (c) 2015 Andreas "PAX" L\u00FCck, All Rights Reserved + + The contents of this file is dual-licensed under 2 + alternative Open Source/Free licenses: LGPL 2.1 or later and + Apache License 2.0. (starting with JNA version 4.0.0). + + You can freely decide which license you want to apply to + the project. + + You may obtain a copy of the LGPL License at: + + http://www.gnu.org/licenses/licenses.html + + A copy is also included in the downloadable source code package + containing JNA, in file "LGPL2.1". + + You may obtain a copy of the Apache License at: + + http://www.apache.org/licenses/ + + A copy is also included in the downloadable source code package + containing JNA, in file "AL2.0". + + + >>> com.squareup.tape2:tape-2.0.0-beta1 + + Copyright (C) 2010 Square, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> com.beust:jcommander-1.78 + + Copyright (C) 2010 the original author or authors. + See the notice.md file distributed with this work for additional + information regarding copyright ownership. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> com.github.java-json-tools:jackson-coreutils-2.0 + + [NOTE: BROADCOM ELECTS TO USE AND DISTRIBUTE THIS SUBPACKAGE UNDER THE TERMS OF THE APACHE 2.0, THE TEXT OF WHICH IS SET FORTH IN THE APPENDIX. THE ORIGINAL LICENSE TERMS ARE REPRODUCED BELOW ONLY AS A REFERENCE.] + + This software is dual-licensed under: + + - the Lesser General Public License (LGPL) version 3.0 or, at your option, any + later version; + - the Apache Software License (ASL) version 2.0. + + The text of both licenses is included (under the names LGPL-3.0.txt and + ASL-2.0.txt respectively). + + Direct link to the sources: + + - LGPL 3.0: https://www.gnu.org/licenses/lgpl-3.0.txt + - ASL 2.0: http://www.apache.org/licenses/LICENSE-2.0.txt + + + >>> com.github.java-json-tools:msg-simple-1.2 + + [PLEASE NOTE: BROADCOM ELECTS TO USE AND DISTRIBUTE THIS COMPONENT UNDER THE TERMS OF THE APACHE 2.0 LICENSE. PLEASE SEE APPENDIX FOR THE FULL TEXT OF THE APACHE 2.0 LICENSE. THE ORIGINAL LICENSE TERMS ARE REPRODUCED BELOW ONLY AS A REFERENCE.] + + This software is dual-licensed under: + + - the Lesser General Public License (LGPL) version 3.0 or, at your option, any + later version; + - the Apache Software License (ASL) version 2.0. + + The text of both licenses is included (under the names LGPL-3.0.txt and + ASL-2.0.txt respectively). + + Direct link to the sources: + + - LGPL 3.0: https://www.gnu.org/licenses/lgpl-3.0.txt + - ASL 2.0: http://www.apache.org/licenses/LICENSE-2.0.txt + + + >>> com.github.java-json-tools:json-patch-1.13 + + [PLEASE NOTE: BROADCOM ELECTS TO USE AND DISTRIBUTE THIS COMPONENT UNDER THE TERMS OF THE APACHE 2.0 LICENSE. PLEASE SEE APPENDIX FOR THE FULL TEXT OF THE APACHE 2.0 LICENSE. THE ORIGINAL LICENSE TERMS ARE REPRODUCED BELOW ONLY AS A REFERENCE.] + + This software is dual-licensed under: + + - the Lesser General Public License (LGPL) version 3.0 or, at your option, any + later version; + - the Apache Software License (ASL) version 2.0. + + The text of both licenses is included (under the names LGPL-3.0.txt and + ASL-2.0.txt respectively). + + Direct link to the sources: + + - LGPL 3.0: https://www.gnu.org/licenses/lgpl-3.0.txt + - ASL 2.0: http://www.apache.org/licenses/LICENSE-2.0.txt + + + >>> commons-codec:commons-codec-1.15 + + Found in: org/apache/commons/codec/digest/PureJavaCrc32C.java + + Copyright (c) 2004-2006 Intel Corportation + + Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + + + >>> com.github.java-json-tools:btf-1.3 + + [NOTE: BROADCOM ELECTS TO USE AND DISTRIBUTE THIS SUBPACKAGE UNDER THE TERMS OF THE APACHE 2.0, THE TEXT OF WHICH IS SET FORTH IN THE APPENDIX. THE ORIGINAL LICENSE TERMS ARE REPRODUCED BELOW ONLY AS A REFERENCE.] + + This software is dual-licensed under: + + - the Lesser General Public License (LGPL) version 3.0 or, at your option, any + later version; + - the Apache Software License (ASL) version 2.0. + + The text of both licenses is included (under the names LGPL-3.0.txt and + ASL-2.0.txt respectively). + + Direct link to the sources: + + - LGPL 3.0: https://www.gnu.org/licenses/lgpl-3.0.txt + - ASL 2.0: http://www.apache.org/licenses/LICENSE-2.0.txt + + + >>> org.apache.httpcomponents:httpclient-4.5.13 + + + + >>> org.elasticsearch:jna-5.5.0 + + License: Apache 2.0 + + + >>> com.squareup.okio:okio-2.8.0 + + Found in: jvmMain/okio/BufferedSource.kt + + Copyright (c) 2014 Square, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + + + >>> com.ibm.async:asyncutil-0.1.0 + + Found in: com/ibm/asyncutil/iteration/AsyncQueue.java + + Copyright (c) IBM Corporation 2017 + + * This project is licensed under the Apache License 2.0, see LICENSE. + + + >>> com.google.api.grpc:proto-google-common-protos-2.0.1 + + + + >>> org.objenesis:objenesis-3.2 + + Found in: META-INF/NOTICE + + Copyright 2006-2021 Joe Walnes, Henri Tremblay, Leonardo Mesquita + + // NOTICE file corresponding to the section 4d of The Apache License, + // Version 2.0, in this case for Objenesis + + + >>> net.openhft:compiler-2.21ea1 + + + + >>> com.lmax:disruptor-3.4.4 + + * Copyright 2011 LMAX Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + + + >>> commons-io:commons-io-2.11.0 + + Found in: META-INF/NOTICE.txt + + Copyright 2002-2021 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + + + >>> org.apache.commons:commons-compress-1.21 + + Found in: META-INF/NOTICE.txt + + Copyright 2002-2021 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + + + >>> com.wavefront:wavefront-sdk-java-3.0.0 + + License-Apache 2.0 + + + >>> org.apache.avro:avro-1.11.0 + + + + >>> com.github.ben-manes.caffeine:caffeine-2.9.3 + + + * Copyright 2015 Ben Manes. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + >>> com.wavefront:wavefront-internal-reporter-java-1.7.10 + + + + >>> org.jboss.logging:jboss-logging-3.4.3.final + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> org.yaml:snakeyaml-1.30 + + License : Apache 2.0 + + + >>> com.squareup.okhttp3:okhttp-4.9.3 + + + + >>> com.google.code.gson:gson-2.9.0 + + + + >>> io.jaegertracing:jaeger-thrift-1.8.0 + + Copyright (c) 2016, Uber Technologies, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + + + + >>> io.jaegertracing:jaeger-core-1.8.0 + + + + >>> org.apache.logging.log4j:log4j-jul-2.17.2 + + Found in: META-INF/NOTICE + + Copyright 1999-2022 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + + >>> org.apache.logging.log4j:log4j-core-2.17.2 + + Found in: META-INF/LICENSE + + Copyright 1999-2005 The Apache Software Foundation + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 1999-2005 The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + >>> org.apache.logging.log4j:log4j-api-2.17.2 + + Copyright 1999-2022 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache license, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the license for the specific language governing permissions and + limitations under the license. + + + + >>> org.apache.logging.log4j:log4j-slf4j-impl-2.17.2 + + Copyright 1999-2022 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache license, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the license for the specific language governing permissions and + limitations under the license. + + + + >>> org.jboss.resteasy:resteasy-client-3.15.3.Final + + + + >>> org.jboss.resteasy:resteasy-jaxrs-3.15.3.Final + + Found in: org/jboss/resteasy/client/exception/WebApplicationExceptionWrapper.java + + Copyright 2020 Red Hat, Inc., and individual contributors + + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + + + >>> org.jboss.resteasy:resteasy-jackson2-provider-3.15.3.Final + + + + >>> joda-time:joda-time-2.10.14 + + + + = NOTICE file corresponding to section 4d of the Apache License Version 2.0 = + + This product includes software developed by + Joda.org (https://www.joda.org/). + + Copyright 2001-2005 Stephen Colebourne + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + / + + + + >>> com.beust:jcommander-1.82 + + Found in: com/beust/jcommander/converters/BaseConverter.java + + Copyright (c) 2010 the original author or authors + + * See the notice.md file distributed with this work for additional + + * information regarding copyright ownership. + + * + + * Licensed under the Apache License, Version 2.0 (the "License"); + + * you may not use this file except in compliance with the License. + + * You may obtain a copy of the License at + + * + + * http://www.apache.org/licenses/LICENSE-2.0 + + * + + * Unless required by applicable law or agreed to in writing, software + + * distributed under the License is distributed on an "AS IS" BASIS, + + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + * See the License for the specific language governing permissions and + + * limitations under the License. + + + >>> io.netty:netty-transport-classes-epoll-4.1.77.final + + + + + - - - - + + + + + + + + diff --git a/pkg/etc/wavefront/wavefront-proxy/log4j2.xml.default b/pkg/etc/wavefront/wavefront-proxy/log4j2.xml.default index 76072e7e7..b00b0978c 100644 --- a/pkg/etc/wavefront/wavefront-proxy/log4j2.xml.default +++ b/pkg/etc/wavefront/wavefront-proxy/log4j2.xml.default @@ -1,5 +1,5 @@ - + /var/log/wavefront @@ -13,7 +13,7 @@ + filePattern="${log-path}/wavefront-%d{yyyy-MM-dd}-%i.log"> - - + + - + @@ -46,7 +46,7 @@ - + @@ -65,7 +65,7 @@ - + @@ -84,18 +84,37 @@ - + --> + + + logs points filtered out by allow/block rules as well --> + + + + + + + + - - + diff --git a/pkg/etc/wavefront/wavefront-proxy/preprocessor_rules.yaml.default b/pkg/etc/wavefront/wavefront-proxy/preprocessor_rules.yaml.default index 5082b554b..85cbef05d 100644 --- a/pkg/etc/wavefront/wavefront-proxy/preprocessor_rules.yaml.default +++ b/pkg/etc/wavefront/wavefront-proxy/preprocessor_rules.yaml.default @@ -11,8 +11,8 @@ ## ## available action types: ## - replaceRegex: perform search/replace substitutions (substring match) -## - whitelistRegex: reject all points not matching a regex pattern (full match) -## - blacklistRegex: reject all points matching a regex pattern (full match) +## - allow: reject all points not matching a regex pattern (full match) +## - block: reject all points matching a regex pattern (full match) ## - forceLowercase: convert a point's component (metric name, source, point tag value) to lowercase ## - dropTag: remove a point tag. optional: remove tag only when its value matches a regex pattern (full match)) ## - addTag: add a new point tag. if such point tag already exists, an existing value will be replaced @@ -22,8 +22,12 @@ ## - renameTag: rename a point tag, preserving its value. optional: rename it only when the point tag value matches ## a regex pattern (full match). this functionality allows separating a point tag with mixed data into ## separate tags. +## - limitLength: enforce custom string length limits for various data point's components (metric name, source, +## point tag value). Available action sub-types: truncate, truncateWithEllipsis, drop +## - count: count the number of points matching an optional "if" condition. When no "if" condition +## is specified, the rule is effectively a no-op. ## -## "Scope" parameter for replaceRegex/whitelistRegex/blacklistRegex: +## "Scope" parameter for replaceRegex/allow/block: ## - pointLine: applies to the whole point string before it's parsed, which makes it possible to correct ## specific issues in the data stream that would normally make the data point unparseable. ## @@ -34,6 +38,7 @@ ## ## Notes: ## - backslashes in regex patterns should be double-escaped +## - numeric values should be wrapped in quotes ## - "match" patterns must be a full match, i.e. a regex to block the point line that contains "stage" substring ## will look like ".*stage.*". replaceRegex "search" patterns are a substring match, so if the pattern is "A" ## and replace is "B", it will simply replace all A's with B's. @@ -41,21 +46,35 @@ ## unique within the same port. for every rule the proxy reports a counter metric, which represents the number of ## times a rule has been successfully applied, and a rule ID becomes part of the metric name ## (ex: ~agent.preprocessor.replace-badchars.count) -## - multiple whitelistRegex rules are allowed, with a caveat: a point must satisfy all of these rules +## - multiple allow rules are allowed, with a caveat: a point must satisfy all of these rules ## to go through, so if it doesn't match at least one of the patterns, a point is blocked ## ############################################################################## -# rules for port 2878 -'2878': +## rules that apply only to data received on port 2879 +#'2879': - # replace bad characters ("&", "$", "!") with underscores in the entire point line string - ################################################################ - - rule : example-replace-badchars - action : replaceRegex - scope : pointLine - search : "[&\\$!]" - replace : "_" + # Example no-op rule + ################################################################# + # - rule : example-rule-do-nothing + # action : count + +## rules that apply only to data received on port 2878 +#'2878': + + ## Example rule that count points where metric value is 0 and metric name starts with 'a' + ################################################################## + #- rule : example-count-points-zero-value + # action : count + # if : "({{metricName}} startsWith 'a') and ($value = 0)" + + ## replace bad characters ("&", "$", "*") with underscores in the entire point line string + ################################################################# + #- rule : example-replace-badchars + # action : replaceRegex + # scope : pointLine + # search : "[&\\$\\*]" + # replace : "_" ## extract Graphite 1.1+ tags from the metric name, i.e. convert ## "metric.name;source=src;foo=bar;boo=baz 1.0 1507933795" to @@ -90,28 +109,28 @@ ## only allow points that contain "prod" substring anywhere in the point line ################################################################ #- rule : example-allow-only-prod - # action : whitelistRegex + # action : allow # scope : pointLine # match : ".*prod.*" ## block all points with sourceName that starts with qa-statsd ################################################################ #- rule : example-block-qa-statsd - # action : blacklistRegex + # action : block # scope : sourceName # match : "qa-statsd.*" ## block all AWS metrics (all metric names that start with "aws.") ################################################################ #- rule : example-block-aws - # action : blacklistRegex + # action : block # scope : metricName # match : "aws\\..*" ## block all points where "datacenter" point tag starts with "west" ################################################################ #- rule : example-block-west - # action : blacklistRegex + # action : block # scope : datacenter # match : "west.*" @@ -186,7 +205,7 @@ ## block all points that have a point tag "blocker" (any value) ################################################################ #- rule : example-block-by-tag - # action : blacklistRegex + # action : block # scope : blocker # match : ".*" @@ -204,17 +223,59 @@ # replace : "$2" # replaceInput : "$1$3" # optional, omit if you plan on just extracting the tag leaving the metric name intact + ## Advanced predicates for finer control. + ## Example: The below rule uses nested predicates to allow all "prod" metrics i.e. + ## all -> returns true in case both "any" AND "none" is true. + ## any -> returns true in case either nested "all" OR "startsWith" OR "equals" is true. + ## none -> returns true in case nested comparison is false. + ################################################################################### + #- rule: test-allow + # action: allow + # if: + # all: + # - any: + # - all: + # - contains: + # scope: sourceName + # value: "prod" + # - startsWith + # scope: metricName + # value: "foometric." + # - startsWith: + # scope: metricName + # value: "foometric.prod." + # - equals: + # scope: env + # value: "prod" + # - none: + # - equals: + # scope: debug + # value: ["true"] + + ## Advanced condition expression: block QA metrics (with source name containing 'qa' + # or env=qa point tag), that are older than 10 minutes + ################################################################################### + #- rule: test-block-old-qa-metrics + # action: block + # if: '({{sourceName}} contains "qa" or {{env}} = "qa") and not {{debug}} = "true" and $timestamp < time("10 minutes ago")' + + ## Advanced condition expression: do deterministic sampling at 25% based on containerId + # (only allow metrics for 1 out of 4 containers) for staging k8s metrics. + ################################################################################### + #- rule: test-sample-staging-k8s-metrics + # action: block + # if: '{{sourceName}} contains "staging" and {{metricName}} startsWith "k8s." and {{containerId}}.hashCode() % 4 != 0' -# rules for port 4242 -'4242': +## rules for port 4242 +#'4242': - # replace bad characters ("&", "$", "!") with underscores in the metric name + ## replace bad characters ("&", "$", "!") with underscores in the metric name ################################################################ - - rule : example-replace-badchars - action : replaceRegex - scope : metricName - search : "[&\\$!]" - replace : "_" + #- rule : example-replace-badchars + # action : replaceRegex + # scope : metricName + # search : "[&\\$!]" + # replace : "_" ## remove "tsdb." from the metric name ################################################################ @@ -224,3 +285,32 @@ # search : "tsdb\\." # replace : "" +## rules for port 30001 +#'30001': + + ## truncate 'db.statement' annotation value at 128 characters, + ## replace last 3 characters with '...'. + ################################################################ + #- rule : example-limit-db-statement + # action : spanLimitLength + # scope : "db.statement" + # actionSubtype : truncateWithEllipsis + # maxLength : "128" + +## Multiport preprocessor rules +## The following rules will apply to ports 2878, 2980 and 4343 +#'2878, 2980, 4343': + + ## Add k8s cluster name point tag for all points across multiple ports. + #- rule : example-rule-delete-merenametag-k8s-cluster + # action : addTag + # tag : k8scluster + # value : eks-dev + +# rules that apply to all ports explicitly specified above. Global rules must be at the end of the file +# 'global': + + # Example no-op rule + ################################################################# + # - rule : example-rule-do-nothing + # action : count diff --git a/pkg/etc/wavefront/wavefront-proxy/wavefront.conf.default b/pkg/etc/wavefront/wavefront-proxy/wavefront.conf.default index a088a880f..647c1823a 100644 --- a/pkg/etc/wavefront/wavefront-proxy/wavefront.conf.default +++ b/pkg/etc/wavefront/wavefront-proxy/wavefront.conf.default @@ -3,121 +3,167 @@ # # Typically in /etc/wavefront/wavefront-proxy/wavefront.conf # -############################################################################## -# The server should be either the primary Wavefront cloud server, or your custom VPC address. -# This will be provided to you by Wavefront. -# -server=https://try.wavefront.com/api/ - -# The hostname will be used to identify the internal proxy statistics around point rates, JVM info, etc. -# We strongly recommend setting this to a name that is unique among your entire infrastructure, -# possibly including the datacenter information, etc. This hostname does not need to correspond to -# any actual hostname or DNS entry; it's merely a string that we pass with the internal stats. -# -#hostname=my.proxy.host.com - -# The Token is any valid API Token for your account, which can be generated from the gear icon -# at the top right of the Wavefront site, under 'Settings'. Paste that hexadecimal token -# after the '=' below, and the proxy will automatically generate a machine-specific UUID and -# self-register. -# -#token=XXX - -## Set to true when running proxy inside containers or when the proxy is frequently restarted on a new infrastructure. -## When true, terminated proxies will be automatically removed from the UI after 24 hours of inactivity. -#ephemeral=false - -# Comma separated list of ports to listen on for Wavefront formatted data +######################################################################################################################## +# Wavefront API endpoint URL. Usually the same as the URL of your Wavefront instance, with an `api` +# suffix -- or Wavefront provides the URL. +server=SERVER_URL_HERE + +# The proxyname will be used to identify the internal proxy statistics around point rates, JVM info, etc. +# We strongly recommend setting this to a name that is unique among your entire infrastructure, to make this +# proxy easy to identify. This proxyname does not need to correspond to any actual proxyname or DNS entry; it's merely +# a string that we pass with the internal stats and ~proxy.* metrics. +#proxyname=my.proxy.name.com + +# The Token is any valid API token for an account that has *Proxy Management* permissions. To get to the token: +# 1. Click the gear icon at the top right in the Wavefront UI. +# 2. Click your account name (usually your email) +# 3. Click *API access*. +#token=WF_TOKEN_HERE + +# To add a proxy, you need to use an existing API token with AOA service proxy role. If you have no API token yet, you +# can create one under your account page in VMWare Cloud Service. +#cspAPIToken=CSP_API_TOKEN_HERE + +# To add a proxy, you need to use an existing App ID, App Secret for server to serve type of app with AOA service proxy role. +# If you have no App ID and App Secret yet, you can create one for server to serve type of app under Organization/OAuth +# Apps menu item in VMWare Cloud Service. Note: Proxy, based on OAuth apps, has no expiration time. +#cspAppId=CSP_APP_ID_HERE + +# To add a proxy, you need to use an existing App ID, App Secret for server to serve type of app with AOA service proxy role. +# If you have no App ID and App Secret yet, you can create one for server to serve type of app under Organization/OAuth +# Apps menu item in VMWare Cloud Service. Note: Proxy, based on OAuth apps, has no expiration time. +#cspAppSecret=CSP_APP_SECRET_HERE + +# The CSP organisation ID. +#cspOrgId=CSP_ORG_ID_HERE + +# CSP console URL. This will be used in many places like getting token. +#cspBaseUrl=https://console.cloud.vmware.com + +####################################################### INPUTS ######################################################### +# Comma-separated list of ports to listen on for Wavefront formatted data (Default: 2878) pushListenerPorts=2878 -## Comma separated list of ports to listen on for OpenTSDB formatted data +## Maximum line length for received points in plaintext format on Wavefront/OpenTSDB/Graphite ports. Default: 32KB +#pushListenerMaxReceivedLength=32768 +## Maximum request size (in bytes) for incoming HTTP requests on Wavefront/OpenTSDB/Graphite ports. Default: 16MB +#pushListenerHttpBufferSize=16777216 + +## Delta counter pre-aggregation settings. +## Comma-separated list of ports to listen on for Wavefront-formatted delta counters. Helps reduce +## outbound point rate by pre-aggregating delta counters at proxy. Default: none +#deltaCountersAggregationListenerPorts=12878 +## Interval between flushing aggregating delta counters to Wavefront. Defaults to 30 seconds. +#deltaCountersAggregationIntervalSeconds=60 + +## Graphite input settings. +## If you enable either `graphitePorts` or `picklePorts`, make sure to uncomment and set `graphiteFormat` as well. +## Comma-separated list of ports to listen on for collectd/Graphite formatted data (Default: none) +#graphitePorts=2003 +## Comma-separated list of ports to listen on for Graphite pickle formatted data (from carbon-relay) (Default: none) +#picklePorts=2004 +## Which fields (1-based) should we extract and concatenate (with dots) as the proxyname? +#graphiteFormat=2 +## Which characters should be replaced by dots in the proxyname, after extraction? (Default: _) +#graphiteDelimiters=_ +## Comma-separated list of fields (metric segments) to remove (1-based). This is an optional setting. (Default: none) +#graphiteFieldsToRemove=3,4,5 + +## OTLP/OpenTelemetry input settings. +## Comma-separated list of ports to listen on for OTLP formatted data over gRPC (Default: none) +#otlpGrpcListenerPorts=4317 +## Comma-separated list of ports to listen on for OTLP formatted data over HTTP (Default: none) +#otlpHttpListenerPorts=4318 +## If true, includes OTLP resource attributes on metrics (Default: false) +#otlpResourceAttrsOnMetricsIncluded=false + + +## DDI/Relay endpoint: in environments where no direct outbound connections to Wavefront servers are possible, you can +## use another Wavefront proxy that has outbound access to act as a relay and forward all the data received on that +## endpoint (from direct data ingestion clients and/or other proxies) to Wavefront servers. +## This setting is a comma-separated list of ports. (Default: none) +#pushRelayListenerPorts=2978 +## If true, aggregate histogram distributions received on the relay port. +## Please refer to histogram settings section below for more configuration options. Default: false +#pushRelayHistogramAggregator=false + +## Comma-separated list of ports to listen on for OpenTSDB formatted data (Default: none) #opentsdbPorts=4242 -## Comma separated list of ports to listen on for HTTP JSON formatted data + +## Comma-separated list of ports to listen on for HTTP JSON formatted data (Default: none) #jsonListenerPorts=3878 -## Comma separated list of ports to listen on for HTTP collectd write_http data + +## Comma-separated list of ports to listen on for HTTP collectd write_http data (Default: none) #writeHttpJsonListenerPorts=4878 -## Comma separated list of ports to listen on for Graphite pickle formatted data (from carbon-relay) -#picklePorts=5878 -## Which ports should listen for collectd/graphite-formatted data? -## If you uncomment graphitePorts, make sure to uncomment and set 'graphiteFormat' and 'graphiteDelimiters' as well. -#graphitePorts=2003 -## Which fields (1-based) should we extract and concatenate (with dots) as the hostname? -#graphiteFormat=2 -## Which characters should be replaced by dots in the hostname, after extraction? -#graphiteDelimiters=_ +################################################# DATA PREPROCESSING ################################################### +## Path to the optional config file with preprocessor rules (advanced regEx replacements and allow/block lists) +#preprocessorConfigFile=/etc/wavefront/wavefront-proxy/preprocessor_rules.yaml -## Number of threads that flush data to the server. If not defined in wavefront.conf it defaults to the -## number of processors (min 4). Setting this value too large will result in sending batches that are -## too small to the server and wasting connections. This setting is per listening port. -#flushThreads=4 +## When using the Wavefront or TSDB data formats, the proxy will automatically look for a tag named +## source= or host= (preferring source=) and treat that as the source/host within Wavefront. +## customSourceTags is a comma-separated, ordered list of additional tag keys to use if neither +## source= or host= is present +#customSourceTags=fqdn, proxyname + +## The prefix should either be left undefined, or can be any prefix you want +## prepended to all data points coming through this proxy (such as `production`). +#prefix=production -## Max points per flush. Typically 40000. +## Regex pattern (Java) that input lines must match to be accepted. Use preprocessor rules for finer control. +#allow=^(production|stage).* +## Regex pattern (Java) that input lines must NOT match to be accepted. Use preprocessor rules for finer control. +#block=^(qa|development|test).* + +## This setting defines the cut-off point for what is considered a valid timestamp for back-dated points. +## Default (and recommended) value is 8760 (1 year), so all the data points from more than 1 year ago will be rejected. +#dataBackfillCutoffHours=8760 +## This setting defines the cut-off point for what is considered a valid timestamp for pre-dated points. +## Default (and recommended) value is 24 (1 day), so all the data points from more than 1 day in future will be rejected. +#dataPrefillCutoffHours=24 + +################################################## ADVANCED SETTINGS ################################################### +## Number of threads that flush data to the server. If not defined in wavefront.conf it defaults to +## the number of available vCPUs (min 4; max 16). Setting this value too large will result in sending +## batches that are too small to the server and wasting connections. This setting is per listening port. +#flushThreads=4 +## Max points per single flush. Default: 40000. #pushFlushMaxPoints=40000 +## Max histograms per single flush. Default: 10000. +#pushFlushMaxHistograms=10000 +## Max spans per single flush. Default: 5000. +#pushFlushMaxSpans=5000 +## Max span logs per single flush. Default: 1000. +#pushFlushMaxSpanLogs=1000 ## Milliseconds between flushes to the Wavefront servers. Typically 1000. #pushFlushInterval=1000 -## Limit pps rate at the proxy. Default: do not throttle +## Limit outbound points per second rate at the proxy. Default: do not throttle #pushRateLimit=20000 +## Limit outbound histograms per second rate at the proxy. Default: do not throttle +#pushRateLimitHistograms=2000 +## Limit outbound spans per second rate at the proxy. Default: do not throttle +#pushRateLimitSpans=1000 +## Limit outbound span logs per second rate at the proxy. Default: do not throttle +#pushRateLimitSpanLogs=1000 ## Max number of burst seconds to allow when rate limiting to smooth out uneven traffic. ## Set to 1 when doing data backfills. Default: 10 #pushRateLimitMaxBurstSeconds=10 -## Max number of points that can stay in memory buffers before spooling to disk. Defaults to 16 * pushFlushMaxPoints, -## minimum allowed size: pushFlushMaxPoints. Setting this value lower than default reduces memory usage but will force -## the proxy to spool to disk more frequently if you have points arriving at the proxy in short bursts. -#pushMemoryBufferLimit=640000 - -## If there are blocked points, how many lines to print to the log every 10 flushes. Typically 5. -#pushBlockedSamples=5 - -# The push log level determines how much information will be printed to the log. -# Options: NONE, SUMMARY, DETAILED. Typically SUMMARY. -pushLogLevel=SUMMARY - -## The validation level keeps certain data from being sent to Wavefront. -## We strongly recommend keeping this at NUMERIC_ONLY -## Options: NUMERIC_ONLY, NO_VALIDATION. -#pushValidationLevel=NUMERIC_ONLY - -# When using the Wavefront or TSDB data formats the Proxy will automatically look for a tag named -# source= or host= (preferring source=) and treat that as the source/host within Wavefront. -# customSourceTags is a comma separated, ordered list of additional tag keys to use if neither -# source= or host= is present -customSourceTags=fqdn, hostname - -## The prefix should either be left undefined, or can be any prefix you want -## prepended to all data points coming through this proxy (such as 'prod'). -#prefix=production - -## ID file for the proxy. Not used if ephemeral=true -idFile=/etc/wavefront/wavefront-proxy/.wavefront_id - -## Default location of buffer.* files for saving failed transmission for retry. -buffer=/var/spool/wavefront-proxy/buffer - -## Number of threads retrying failed transmissions. Defaults to the number of processors (min. 4) -## Buffer files are maxed out at 2G each so increasing the number of retry threads effectively governs -## the maximum amount of space the proxy will use to buffer points locally -#retryThreads=4 - -## Regex pattern (java.util.regex) that input lines must match to be accepted. -## Input lines are checked against the pattern before the prefix is prepended. -#whitelistRegex=^(production|stage).* - -## Regex pattern (java.util.regex) that input lines must NOT match to be accepted. -## Input lines are checked against the pattern before the prefix is prepended. -#blacklistRegex=^(qa|development|test).* - -## Whether to split the push batch size when the push is rejected by Wavefront due to rate limit. Default false. -#splitPushWhenRateLimited=false +## Location of buffer.* files for saving failed transmissions for retry. Default: /var/spool/wavefront-proxy/buffer +#buffer=/var/spool/wavefront-proxy/buffer +## Buffer file partition size, in MB. Setting this value too low may reduce the efficiency of disk +## space utilization, while setting this value too high will allocate disk space in larger +## increments. Default: 128 +#bufferShardSize=128 +## Use single-file buffer (legacy functionality). Default: false +#disableBufferSharding=false ## For exponential backoff when retry threads are throttled, the base (a in a^b) in seconds. Default 2.0 #retryBackoffBaseSeconds=2.0 - -## Control whether metrics traffic from the proxy to the Wavefront endpoint is gzip-compressed. Default: true -#gzipCompression=false +## Whether to split the push batch size when the push is rejected by Wavefront due to rate limit. Default false. +#splitPushWhenRateLimited=false ## The following settings are used to connect to Wavefront servers through a HTTP proxy: #proxyHost=localhost @@ -125,123 +171,280 @@ buffer=/var/spool/wavefront-proxy/buffer ## Optional: if http proxy requires authentication #proxyUser=proxy_user #proxyPassword=proxy_password -# -## The following setting enables SO_LINGER with the specified linger time in seconds (SO_LINGER disabled by default) -#soLingerTime=0 -## HTTP connect timeout (in milliseconds). Default: 5s (5000) +## HTTP proxies may implement a security policy to only allow traffic with particular User-Agent header values. +## When set, overrides User-Agent in request headers for outbound HTTP requests. +#httpUserAgent=WavefrontProxy + +## HTTP client settings +## Control whether metrics traffic from the proxy to the Wavefront endpoint is gzip-compressed. Default: true +#gzipCompression=true +## If gzipCompression is enabled, sets compression level (1-9). Higher compression levels slightly reduce +## the volume of traffic between the proxy and Wavefront, but use more CPU. Default: 4 +#gzipCompressionLevel=4 +## Connect timeout (in milliseconds). Default: 5000 (5s) #httpConnectTimeout=5000 -## HTTP request timeout (in milliseconds). Default: 10s (10000) +## Socket timeout (in milliseconds). Default: 10000 (10s) #httpRequestTimeout=10000 +## Max number of total connections to keep open (Default: 200) +#httpMaxConnTotal=100 +## Max connections per route to keep open (Default: 100) +#httpMaxConnPerRoute=100 +## Number of times to retry http requests before queueing, set to 0 to disable (default: 3) +#httpAutoRetries=3 + +## Close idle inbound connections after specified time in seconds. Default: 300 (5 minutes) +#listenerIdleConnectionTimeout=300 +## When receiving Wavefront-formatted data without source/host specified, use remote IP address as +## source instead of trying to resolve the DNS name. Default false. +#disableRdnsLookup=true +## The following setting enables SO_LINGER on listening ports with the specified linger time in seconds (Default: off) +#soLingerTime=0 -## Path to the optional config file with preprocessor rules (advanced regEx replacements and whitelist/blacklists) -#preprocessorConfigFile=/etc/wavefront/wavefront-proxy/preprocessor_rules.yaml - -## This setting defines the cut-off point for what is considered a valid timestamp for back-dated points. -## Default (and recommended) value is 8760 (1 year), so all the data points from more than 1 year ago will be rejected. -#dataBackfillCutoffHours=8760 -## This setting defines the cut-off point for what is considered a valid timestamp for pre-dated points. -## Default (and recommended) value is 24 (1 day), so all the data points from more than 1 day in future will be rejected. -#dataPrefillCutoffHours=24 +## Max number of points that can stay in memory buffers before spooling to disk. Defaults to 16 * pushFlushMaxPoints, +## minimum allowed size: pushFlushMaxPoints. Setting this value lower than default reduces memory usage but will force +## the proxy to spool to disk more frequently if you have points arriving at the proxy in short bursts. +#pushMemoryBufferLimit=640000 -## The following settings are used to configure distributed tracing span ingestion: -## Comma-separated list of ports to listen on for Wavefront trace data. Defaults to none. +## If there are blocked points, a small sampling of them can be written into the main log file. +## This setting how many lines to print to the log every 10 seconds. Typically 5. +#pushBlockedSamples=5 +## When logging blocked points is enabled (see https://docs.wavefront.com/proxies_configuring.html#blocked-point-log +## for more details), by default all blocked points/histograms/spans etc are logged into the same `RawBlockedPoints` +## logger. Settings below allow finer control over these loggers: +## Logger name for blocked points. Default: RawBlockedPoints. +#blockedPointsLoggerName=RawBlockedPoints +## Logger name for blocked histograms. Default: RawBlockedPoints. +#blockedHistogramsLoggerName=RawBlockedPoints +## Logger name for blocked spans. Default: RawBlockedPoints. +#blockedSpansLoggerName=RawBlockedPoints + +## Discard all received data (debug/performance test mode). If enabled, you will lose received data! Default: false +#useNoopSender=false + +## Settings for incoming HTTP request authentication. Authentication is done by a token, proxy is looking for +## tokens in the querystring ("token=" and "api_key=" parameters) and in request headers ("X-AUTH-TOKEN: ", +## "Authorization: Bearer", "Authorization: " headers). TCP streams are disabled when authentication is turned on. +## Allowed authentication methods: NONE, STATIC_TOKEN, HTTP_GET, OAUTH2. Default: NONE +## - NONE: All requests are considered valid +## - STATIC_TOKEN: Compare incoming token with the value of authStaticToken setting. +## - OAUTH2: Validate all tokens against a RFC7662-compliant token introspection endpoint. +## - HTTP_GET: Validate all tokens against a specific URL. Use {{token}} placeholder in the URL to pass the token +## in question to the endpoint. Use of https is strongly recommended for security reasons. The endpoint +## must return any 2xx status for valid tokens, any other response code is considered a fail. +#authMethod=NONE +## URL for the token introspection endpoint used to validate tokens for incoming HTTP requests. +## Required when authMethod is OAUTH2 or HTTP_GET +#authTokenIntrospectionServiceUrl=https://auth.acme.corp/api/token/{{token}}/validate +## Optional credentials for use with the token introspection endpoint if it requires authentication. +#authTokenIntrospectionAuthorizationHeader=Authorization: Bearer +## Cache TTL (in seconds) for token validation results (re-authenticate when expired). Default: 600 seconds +#authResponseRefreshInterval=600 +## Maximum allowed cache TTL (in seconds) for token validation results when token introspection service is +## unavailable. Default: 86400 seconds (1 day) +#authResponseMaxTtl=86400 +## Static token that is considered valid for all incoming HTTP requests. Required when authMethod = STATIC_TOKEN. +#authStaticToken=token1234abcd + +## Enables intelligent traffic shaping based on received rate over last 5 minutes. Default: disabled +#trafficShaping=false +## Sets the width (in seconds) for the sliding time window which would be used to calculate received +## traffic rate. Default: 600 (10 minutes) +#trafficShapingWindowSeconds=600 +## Sets the headroom multiplier to use for traffic shaping when there's backlog. Default: 1.15 (15% headroom) +#trafficShapingHeadroom=1.15 + +## Enables CORS for specified comma-delimited list of listening ports. Default: none (CORS disabled) +#corsEnabledPorts=2879 +## Allowed origin for CORS requests, or '*' to allow everything. Default: none +#corsOrigin=* +## Allow 'null' origin for CORS requests. Default: false +#corsAllowNullOrigin=false + +## Enables TLS for specified listening ports (comma-separated list). Use '*' to secure all ports. Defaut: none (TLS disabled) +#tlsPorts=4443 +## TLS certificate path to use for securing all the ports. X.509 certificate chain file in PEM format. +#privateCertPath=/etc/wavefront/wavefront-proxy/cert.pem +## TLS private key path to use for securing all the ports. PKCS#8 private key file in PEM format. +#privateKeyPath=/etc/wavefront/wavefront-proxy/private_key.pem + +########################################### MANAGED HEALTHCHECK ENDPOINT ############################################### +## Comma-delimited list of ports to function as standalone healthchecks. May be used independently of +## httpHealthCheckAllPorts parameter. Default: none +#httpHealthCheckPorts=8880 +## When true, all listeners that support HTTP protocol also respond to healthcheck requests. May be +## used independently of httpHealthCheckPorts parameter. Default: false +#httpHealthCheckAllPorts=true +## Healthcheck's path, for example, '/health'. Default: '/' +#httpHealthCheckPath=/status +## Optional Content-Type to use in healthcheck response, for example, 'application/json'. Default: none +#httpHealthCheckResponseContentType=text/plain +## HTTP status code for 'pass' health checks. Default: 200 +#httpHealthCheckPassStatusCode=200 +## Optional response body to return with 'pass' health checks. Default: none +#httpHealthCheckPassResponseBody=good to go! +## HTTP status code for 'fail' health checks. Default: 503 +#httpHealthCheckFailStatusCode=503 +## Optional response body to return with 'fail' health checks. Default: none +#httpHealthCheckFailResponseBody=try again later... +## Enables admin port to control healthcheck status per port. Default: none +#adminApiListenerPort=8888 +## Remote IPs must match this regex to access admin API +#adminApiRemoteIpAllowRegex=^.*$ + +############################################# LOGS TO METRICS SETTINGS ################################################# +## Port on which to listen for FileBeat data (Lumberjack protocol). Default: none +#filebeatPort=5044 +## Port on which to listen for raw logs data (TCP and HTTP). Default: none +#rawLogsPort=5045 +## Maximum line length for received raw logs (Default: 4096) +#rawLogsMaxReceivedLength=4096 +## Maximum allowed request size (in bytes) for incoming HTTP requests with raw logs (Default: 16MB) +#rawLogsHttpBufferSize=16777216 +## Location of the `logsingestion.yaml` configuration file +#logsIngestionConfigFile=/etc/wavefront/wavefront-proxy/logsingestion.yaml + +########################################### DISTRIBUTED TRACING SETTINGS ############################################### +## Comma-separated list of ports to listen on for spans in Wavefront format. Defaults to none. #traceListenerPorts=30000 -## Comma-separated list of ports on which to listen on for Jaeger Thrift formatted data. Defaults to none. -#traceJaegerListenerPorts=30001 +## Maximum line length for received spans and span logs (Default: 1MB) +#traceListenerMaxReceivedLength=1048576 +## Maximum allowed request size (in bytes) for incoming HTTP requests on tracing ports (Default: 16MB) +#traceListenerHttpBufferSize=16777216 + +## Comma-separated list of ports to listen on for spans from SDKs that send raw data. Unlike +## `traceListenerPorts` setting, also derives RED metrics based on received spans. Defaults to none. +#customTracingListenerPorts=30001 +## Custom application name for spans received on customTracingListenerPorts. Defaults to defaultApp. +#customTracingApplicationName=defaultApp +## Custom service name for spans received on customTracingListenerPorts. Defaults to defaultService. +#customTracingServiceName=defaultService + +## Comma-separated list of ports on which to listen on for Jaeger Thrift formatted data over TChannel protocol. Defaults to none. +#traceJaegerListenerPorts=14267 +## Comma-separated list of ports on which to listen on for Jaeger Thrift formatted data over HTTP. Defaults to none. +#traceJaegerHttpListenerPorts=14268 +## Comma-separated list of ports on which to listen on for Jaeger Protobuf formatted data over gRPC. Defaults to none. +#traceJaegerGrpcListenerPorts=14250 +## Custom application name for traces received on Jaeger's traceJaegerListenerPorts. +#traceJaegerApplicationName=Jaeger +## Comma-separated list of ports on which to listen on for Zipkin trace data over HTTP. Defaults to none. +## Recommended value is 9411, which is the port Zipkin's server listens on and is the default configuration in Istio. +#traceZipkinListenerPorts=9411 +## Custom application name for traces received on Zipkin's traceZipkinListenerPorts. +#traceZipkinApplicationName=Zipkin +## Comma-separated list of additional custom tag keys to include along as metric tags for the derived +## RED (Request, Error, Duration) metrics. Applicable to Jaeger and Zipkin integration only. +#traceDerivedCustomTagKeys=tenant, env, location + +## The following settings are used to configure trace data sampling: +## The rate for traces to be sampled. Can be from 0.0 to 1.0. Defaults to 1.0 +#traceSamplingRate=1.0 +## The minimum duration threshold (in milliseconds) for the spans to be sampled. +## Spans above the given duration are reported. Defaults to 0 (include everything). +#traceSamplingDuration=0 + +########################################## HISTOGRAM ACCUMULATION SETTINGS ############################################# +## Histograms can be ingested in Wavefront scalar and distribution format. For scalar samples ports can be specified for +## minute, hour and day granularity. Granularity for the distribution format is encoded inline. Before using any of +## these settings, reach out to Wavefront Support to ensure your account is enabled for native Histogram support and +## to optimize the settings for your specific use case. -## The following settings are used to configure histogram ingestion: -## Histograms can be ingested in wavefront scalar and distribution format. For scalar samples ports can be specified for -## minute, hour and day granularity. Granularity for the distribution format is encoded inline. -## Before using any of these settings, reach out to Wavefront Support to ensure your account is enabled for native Histogram -## support and to optimize the settings for your specific use case. +## Accumulation parameters +## Directory for persisting proxy state, must be writable. +#histogramStateDirectory=/var/spool/wavefront-proxy +## Interval to write back accumulation changes to disk in milliseconds (only applicable when memory cache is enabled). +#histogramAccumulatorResolveInterval=5000 +## Interval to check for histograms ready to be sent to Wavefront, in milliseconds. +#histogramAccumulatorFlushInterval=10000 +## Max number of histograms to send to Wavefront in one flush (Default: no limit) +#histogramAccumulatorFlushMaxBatchSize=4000 +## Maximum line length for received histogram data (Default: 65536) +#histogramMaxReceivedLength=65536 +## Maximum allowed request size (in bytes) for incoming HTTP requests on histogram ports (Default: 16MB) +#histogramHttpBufferSize=16777216 ## Wavefront format, minute aggregation: ## Comma-separated list of ports to listen on. #histogramMinuteListenerPorts=40001 -## Number of accumulators per minute port -#histogramMinuteAccumulators=2 ## Time-to-live in seconds for a minute granularity accumulation on the proxy (before the intermediary is shipped to WF). #histogramMinuteFlushSecs=70 -## Bounds the number of centroids per histogram. Must be in [20;1000], default: 100 -#histogramMinuteCompression=20 -## Average number of bytes in a [UTF-8] encoded histogram key. ~metric, source and tags concatenation. -#histogramMinuteAvgKeyBytes=150 -## Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel reporting bins. Setting this value too -## high will may cause excessive disk space usage, setting this value too low may cause severe performance issues. +## Bounds the number of centroids per histogram. Must be between 20 and 1000, default: 32 +#histogramMinuteCompression=32 +## Expected upper bound of concurrent accumulations. Should be approximately the number of timeseries * 2 (use a higher +## multiplier if out-of-order points more than 1 minute apart are expected). Setting this value too high will cause +## excessive disk space usage, setting this value too low may cause severe performance issues. #histogramMinuteAccumulatorSize=1000 ## Enabling memory cache reduces I/O load with fewer time series and higher frequency data (more than 1 point per ## second per time series). Default: false -#histogramMinuteMemoryCache=true +#histogramMinuteMemoryCache=false +## Whether to persist accumulation state. When true, all histograms are written to disk immediately if memory cache is +## disabled, or every histogramAccumulatorResolveInterval seconds if memory cache is enabled. +## If accumulator is not persisted, up to histogramMinuteFlushSecs seconds worth of histograms may be lost on proxy shutdown. +#histogramMinuteAccumulatorPersisted=false ## Wavefront format, hour aggregation: ## Comma-separated list of ports to listen on. #histogramHourListenerPorts=40002 -## Number of accumulators per hour port -#histogramHourAccumulators=2 ## Time-to-live in seconds for an hour granularity accumulation on the proxy (before the intermediary is shipped to WF). #histogramHourFlushSecs=4200 -## Bounds the number of centroids per histogram. Must be in [20;1000], default: 100 -#histogramHourCompression=100 -## Average number of bytes in a [UTF-8] encoded histogram key. ~metric, source and tags concatenation. -#histogramHourAvgKeyBytes=150 -## Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel reporting bins. Setting this value too -## high will may cause excessive disk space usage, setting this value too low may cause severe performance issues. +## Bounds the number of centroids per histogram. Must be between 20 and 1000, default: 32 +#histogramHourCompression=32 +## Expected upper bound of concurrent accumulations. Should be approximately the number of timeseries * 2 (use a higher +## multiplier if out-of-order points more than 1 hour apart are expected). Setting this value too high will cause +## excessive disk space usage, setting this value too low may cause severe performance issues. #histogramHourAccumulatorSize=100000 ## Enabling memory cache reduces I/O load with fewer time series and higher frequency data (more than 1 point per ## second per time series). Default: false -#histogramHourMemoryCache=false +#histogramHourMemoryCache=true +## Whether to persist accumulation state. When true, all histograms are written to disk immediately if memory cache is +## disabled, or every `histogramAccumulatorResolveInterval` seconds if memory cache is enabled. +## If accumulator is not persisted, up to `histogramHourFlushSecs` seconds worth of histograms may be lost on proxy shutdown. +#histogramHourAccumulatorPersisted=true ## Wavefront format, day aggregation: ## Comma-separated list of ports to listen on. #histogramDayListenerPorts=40003 -## Number of accumulators per day port -#histogramDayAccumulators=2 ## Time-to-live in seconds for a day granularity accumulation on the proxy (before the intermediary is shipped to WF). #histogramDayFlushSecs=18000 -## Bounds the number of centroids per histogram. Must be in [20;1000], default: 100 -#histogramDayCompression=200 -## Average number of bytes in a [UTF-8] encoded histogram key. ~metric, source and tags concatenation. -#histogramDayAvgKeyBytes=150 -## Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel reporting bins. Setting this value too -## high will may cause excessive disk space usage, setting this value too low may cause severe performance issues. +## Bounds the number of centroids per histogram. Must be between 20 and 1000, default: 32 +#histogramDayCompression=32 +## Expected upper bound of concurrent accumulations. Should be approximately the number of timeseries * 2 (use a higher +## multiplier if out-of-order points more than 1 day apart are expected). Setting this value too high will cause +## excessive disk space usage, setting this value too low may cause severe performance issues. #histogramDayAccumulatorSize=100000 ## Enabling memory cache reduces I/O load with fewer time series and higher frequency data (more than 1 point per ## second per time series). Default: false #histogramDayMemoryCache=false +## Whether to persist accumulation state. When true, all histograms are written to disk immediately if memory cache is +## disabled, or every `histogramAccumulatorResolveInterval` seconds if memory cache is enabled. +## If accumulator is not persisted, up to `histogramDayFlushSecs` seconds worth of histograms may be lost on proxy shutdown. +#histogramDayAccumulatorPersisted=true ## Distribution format: ## Comma-separated list of ports to listen on. #histogramDistListenerPorts=40000 -## Number of accumulators per day port -#histogramDistAccumulators=2 ## Time-to-live in seconds for a distribution accumulation on the proxy (before the intermediary is shipped to WF). #histogramDistFlushSecs=70 -## Bounds the number of centroids per histogram. Must be in [20;1000], default: 100 -#histogramDistCompression=200 -## Average number of bytes in a [UTF-8] encoded histogram key. ~metric, source and tags concatenation. -#histogramDistAvgKeyBytes=150 -## Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel reporting bins. Setting this value too -## high will may cause excessive disk space usage, setting this value too low may cause severe performance issues. +## Bounds the number of centroids per histogram. Must be between 20 and 1000, default: 32 +#histogramDistCompression=32 +## Expected upper bound of concurrent accumulations. Should be approximately the number of timeseries * 2 (use a higher +## multiplier if out-of-order points more than 1 bin apart are expected). Setting this value too high will cause +## excessive disk space usage, setting this value too low may cause severe performance issues. #histogramDistAccumulatorSize=100000 ## Enabling memory cache reduces I/O load with fewer time series and higher frequency data (more than 1 point per ## second per time series). Default: false #histogramDistMemoryCache=false - -## Accumulation parameters -## Directory for persistent proxy state, must be writable. -histogramStateDirectory=/var/spool/wavefront-proxy -## Interval to write-back accumulation changes to disk in millis (only applicable when memory cache is enabled) -#histogramAccumulatorResolveInterval=500 -## Interval to check for histograms ready to be sent to Wavefront, in millis -#histogramAccumulatorFlushInterval=1000 -## Max number of histograms to send to Wavefront in one flush (Default: no limit) -#histogramAccumulatorFlushMaxBatchSize=4000 -## Interval to send received points to the processing queue in millis (Default: 100) -#histogramReceiveBufferFlushInterval=100 -## Processing queue scan interval in millis (Default: 20) -#histogramProcessingQueueScanInterval=20 -## Whether to persist received histogram messages to disk. WARNING only disable this, if loss of unprocessed sample data -## on proxy shutdown is acceptable. -#persistMessages=true -## Whether to persist accumulation state. WARNING any unflushed histograms will be lost on proxy shutdown if disabled -#persistAccumulator=true +## Whether to persist accumulation state. When true, all histograms are written to disk immediately if memory cache is +## disabled, or every histogramAccumulatorResolveInterval seconds if memory cache is enabled. +## If accumulator is not persisted, up to histogramDistFlushSecs seconds worth of histograms may be lost on proxy shutdown. +#histogramDistAccumulatorPersisted=true + +## Histogram accumulation for relay ports (only applicable if pushRelayHistogramAggregator is true): +## Time-to-live in seconds for the accumulator (before the intermediary is shipped to WF). Default: 70 +#pushRelayHistogramAggregatorFlushSecs=70 +## Bounds the number of centroids per histogram. Must be between 20 and 1000, default: 32 +#pushRelayHistogramAggregatorCompression=32 +## Expected upper bound of concurrent accumulations. Should be approximately the number of timeseries * 2 (use a higher +## multiplier if out-of-order points more than 1 bin apart are expected). Setting this value too high will cause +## excessive disk space usage, setting this value too low may cause severe performance issues. +#pushRelayHistogramAggregatorAccumulatorSize=1000000 diff --git a/pkg/open_source_licenses.txt b/pkg/open_source_licenses.txt deleted file mode 120000 index 024da3e57..000000000 --- a/pkg/open_source_licenses.txt +++ /dev/null @@ -1 +0,0 @@ -../open_source_licenses.txt \ No newline at end of file diff --git a/pkg/stage.sh b/pkg/stage.sh deleted file mode 100755 index de329e5df..000000000 --- a/pkg/stage.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -ex - -function die { - echo $@ - exit 1 -} - -PROG_DIR=`dirname $0` -cd $PROG_DIR -PROG_DIR=`pwd` -echo "Cleaning prior build run..." - -rm -rf build -if ls *.deb &> /dev/null; then - rm *.deb -fi -if ls *.rpm &> /dev/null; then - rm *.rpm -fi - -if [[ $# -lt 3 ]]; then - die "Usage: $0 " -fi - -git pull - -JRE=$1 -JRE=${JRE%/} -[[ -d $JRE ]] || die "Bad JRE given." - -COMMONS_DAEMON=$2 -[[ -d $COMMONS_DAEMON ]] || die "Bad agent commons-daemon given." - -PUSH_AGENT_JAR=$3 -[[ -f $PUSH_AGENT_JAR ]] || die "Bad agent jarfile given." - -shift 3 - -WF_DIR=`pwd`/build/opt/wavefront -PROXY_DIR=$WF_DIR/wavefront-proxy - -echo "Create build dirs..." -mkdir build -cp -r opt build/opt -cp -r etc build/etc -cp -r usr build/usr - -COMMONS_DAEMON_COMMIT="7747df1f0bc21175040afb3b9adcccb3f80a8701" -echo "Make jsvc at $COMMONS_DAEMON_COMMIT..." -cp -r $COMMONS_DAEMON $PROXY_DIR -cd $PROXY_DIR/commons-daemon -git pull -git reset --hard $COMMONS_DAEMON_COMMIT -JSVC_BUILD_DIR="$PROXY_DIR/commons-daemon/src/native/unix" -cd $JSVC_BUILD_DIR -support/buildconf.sh -./configure --with-java=$JRE -make -cd $PROXY_DIR/bin -ln -s ../commons-daemon/src/native/unix/jsvc jsvc - -echo "Stage the agent jar..." -cd $PROG_DIR -cp $PUSH_AGENT_JAR $PROXY_DIR/bin/wavefront-push-agent.jar diff --git a/pkg/upload_to_packagecloud.sh b/pkg/upload_to_packagecloud.sh index 80b5400a2..c920a7eeb 100755 --- a/pkg/upload_to_packagecloud.sh +++ b/pkg/upload_to_packagecloud.sh @@ -1,26 +1,45 @@ -if [[ $# -ne 2 ]]; then - echo "Usage: $0 " +#!/bin/bash -ex + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 " exit 1 fi -package_cloud push --config=$2 $1/el/7 *.rpm & -package_cloud push --config=$2 $1/el/6 *.rpm & -package_cloud push --config=$2 $1/ol/7 *.rpm & -package_cloud push --config=$2 $1/ol/6 *.rpm & -package_cloud push --config=$2 $1/sles/12.0 *.rpm & -package_cloud push --config=$2 $1/sles/12.1 *.rpm & -package_cloud push --config=$2 $1/sles/12.2 *.rpm & -package_cloud push --config=$2 $1/fedora/27 *.rpm & -package_cloud push --config=$2 $1/opensuse/42.3 *.rpm & -package_cloud push --config=$2 $1/debian/buster *.deb & -package_cloud push --config=$2 $1/debian/stretch *.deb & -package_cloud push --config=$2 $1/debian/wheezy *.deb & -package_cloud push --config=$2 $1/debian/jessie *.deb & -package_cloud push --config=$2 $1/ubuntu/xenial *.deb & -package_cloud push --config=$2 $1/ubuntu/trusty *.deb & -package_cloud push --config=$2 $1/ubuntu/zesty *.deb & -package_cloud push --config=$2 $1/ubuntu/artful *.deb & -package_cloud push --config=$2 $1/ubuntu/bionic *.deb & -package_cloud push --config=$2 $1/ubuntu/cosmic *.deb & +echo "=============" +echo "ls -l ${3}" +ls -l ${3} +echo "=============" + +package_cloud push ${1}/el/7 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/el/8 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/el/9 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/el/6 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/ol/8 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/ol/7 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/ol/6 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/sles/12.0 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/sles/12.1 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/sles/12.2 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/fedora/27 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/opensuse/42.3 ${3}/*.rpm --config=${2} & +package_cloud push ${1}/debian/buster ${3}/*.deb --config=${2} & +package_cloud push ${1}/debian/stretch ${3}/*.deb --config=${2} & +package_cloud push ${1}/debian/wheezy ${3}/*.deb --config=${2} & +package_cloud push ${1}/debian/jessie ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/focal ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/eoan ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/disco ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/cosmic ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/bionic ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/artful ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/zesty ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/xenial ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/trusty ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/hirsute ${3}/*.deb --config=${2} & +package_cloud push ${1}/ubuntu/jammy ${3}/*.deb --config=${2} & wait + +package_cloud push ${1}/any/any ${3}/*.deb --config=${2} +package_cloud push ${1}/rpm_any/rpm_any ${3}/*.rpm --config=${2} + diff --git a/pom.xml b/pom.xml deleted file mode 100644 index da113c94c..000000000 --- a/pom.xml +++ /dev/null @@ -1,376 +0,0 @@ - - - 4.0.0 - - com.wavefront - wavefront - 4.35-SNAPSHOT - - java-lib - proxy - java-client - dropwizard-metrics/dropwizard-metrics5 - dropwizard-metrics/dropwizard-metrics - dropwizard-metrics/dropwizard-metrics-wavefront - yammer-metrics - - pom - - Wavefront All-in-One - Top-level Wavefront Public Java POM - http://www.wavefront.com - - - The Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - - - - - Clement Pang - clement@wavefront.com - Wavefront - http://www.wavefront.com - - - - scm:git:git@github.com:wavefronthq/java.git - scm:git:git@github.com:wavefronthq/java.git - git@github.com:wavefronthq/java.git - wavefront-3.0 - - - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - - - - UTF-8 - UTF-8 - - - 4.12 - 0.29 - - 1.8 - 2.11.1 - 9.4.14.v20181114 - 2.9.6 - 4.1.25.Final - 4.35-SNAPSHOT - - - - - - maven-compiler-plugin - 3.1 - - ${java.version} - ${java.version} - - - - maven-assembly-plugin - 2.6 - - gnu - - - - org.apache.maven.plugins - maven-release-plugin - 2.5 - - true - false - release - deploy - - - - - - - - - com.wavefront - java-lib - ${public.project.version} - - - com.wavefront - proxy - ${public.project.version} - - - com.wavefront - java-client - ${public.project.version} - - - com.wavefront - dropwizard-metrics - ${public.project.version} - - - com.wavefront - dropwizard-metrics5 - ${public.project.version} - - - com.wavefront - dropwizard-metrics-4.0 - ${public.project.version} - - - com.wavefront - dropwizard-metrics-wavefront - ${public.project.version} - - - com.google.guava - guava - 26.0-jre - - - joda-time - joda-time - 2.6 - - - org.jboss.resteasy - resteasy-bom - 3.0.21.Final - pom - import - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - com.fasterxml.jackson.module - jackson-module-afterburner - ${jackson.version} - - - org.apache.httpcomponents - httpclient - 4.5.3 - - - org.ops4j.pax.url - pax-url-aether - 2.5.2 - - - org.slf4j - slf4j-api - 1.7.25 - - - org.slf4j - jcl-over-slf4j - 1.7.25 - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - - - org.apache.logging.log4j - log4j-api - ${log4j.version} - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - - - org.apache.logging.log4j - log4j-jul - ${log4j.version} - - - org.apache.logging.log4j - log4j-web - ${log4j.version} - - - com.yammer.metrics - metrics-core - 2.2.0 - - - commons-lang - commons-lang - 2.6 - - - commons-collections - commons-collections - 3.2.2 - - - com.thoughtworks.xstream - xstream - 1.4.10 - - - javax.ws.rs - javax.ws.rs-api - 2.0.1 - - - junit - junit - ${junit.version} - test - - - com.google.truth - truth - ${truth.version} - test - - - org.apache.avro - avro - 1.8.2 - - - org.antlr - antlr4-runtime - 4.7.1 - - - io.netty - netty-handler - ${netty.version} - - - io.netty - netty-codec-http - ${netty.version} - - - io.netty - netty-transport-native-epoll - ${netty.version} - linux-x86_64 - - - com.google.code.findbugs - jsr305 - 2.0.1 - - - - - - - disable-java8-doclint - - [1.8,) - - - -Xdoclint:none - - - - release - - true - - - - - org.apache.maven.plugins - maven-source-plugin - 2.2.1 - - - attach-sources - - jar-no-fork - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.9.1 - - - attach-javadocs - - jar - - - -Xdoclint:none - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - sign-artifacts - verify - - true - - - sign - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.3 - true - - ossrh - https://oss.sonatype.org/ - true - - - - - - - diff --git a/proxy/README.md b/proxy/README.md index 39dbbb949..c5ee9f716 100644 --- a/proxy/README.md +++ b/proxy/README.md @@ -5,12 +5,32 @@ The Wavefront proxy is a light-weight Java application that you send your metric Source code under `org.logstash.*` is used from [logstash-input-beats](https://github.com/logstash-plugins/logstash-input-beats) via the Apache 2.0 license. +## Proxy Installation Options + +### Option 1. Use The Wavefront Installer + +The recommended (and by far the easiest) way to install the most recent release of the proxy is to use [the Wavefront installer](https://docs.wavefront.com/proxies_installing.html##proxy-installation). This is a simple, one-line installer that configures the Wavefront proxy and/or `collectd` to send telemetry data to Wavefront in as little as one step. + +### Option 2. Use Linux Packages + +We have pre-build packages for popular Linux distros. +* Packages for released versions are available at https://packagecloud.io/wavefront/proxy. +* Release candidate versions are available at https://packagecloud.io/wavefront/proxy-next. + +### Option 3. Build Your Own Proxy + +To build your own version, run the following commands (you need [Apache Maven](https://maven.apache.org) installed for a successful build). + +``` +git clone https://github.com/wavefrontHQ/wavefront-proxy +cd wavefront-proxy +mvn clean install +``` -## Set Up a Wavefront Proxy +## Setting Up a Wavefront Proxy To set up a Wavefront proxy to listen for metrics, histograms, and trace data: -1. On the host that will run the proxy, use the Wavefront installer to [install the latest proxy version](http://docs.wavefront.com/proxies_installing.html##proxy-installation). +1. On the host that will run the proxy, install using one of the above methods (using [Wavefront installer](http://docs.wavefront.com/proxies_installing.html##proxy-installation) is the easiest). * If you already have an installed proxy, you may need to [upgrade](http://docs.wavefront.com/proxies_installing.html#upgrading-a-proxy) it. You need Version 4.33 or later to listen for trace data. - * See [below](#proxy-installation-options) for other installation options. 2. On the proxy host, open the proxy configuration file `wavefront.conf` for editing. The file location depends on the host: * Linux - `/etc/wavefront/wavefront-proxy/wavefront.conf` * Mac - `/usr/local/etc/wavefront/wavefront-proxy/wavefront.conf` @@ -22,38 +42,18 @@ To set up a Wavefront proxy to listen for metrics, histograms, and trace data: ## wavefront.conf file ... # Listens for metric data. Default: 2878 - pushListenerPort=2878 + pushListenerPorts=2878 ... - # Listens for histogram distributions. Recommended: 40000 - histogramDistListenerPort=40000 + # Listens for histogram distributions. + # Recommended: 2878 (proxy version 4.29 or later) or 40000 (earlier proxy versions) + histogramDistListenerPorts=2878 ... # Listens for trace data. Recommended: 30000 - traceListenerPort=30000 + traceListenerPorts=30000 ``` 4. Save the `wavefront.conf` file. 5. [Start](http://docs.wavefront.com/proxies_installing.html###starting-and-stopping-a-proxy) the proxy. -## Proxy Installation Options - -### Option 1. Use The Wavefront Installer - -The recommended (and by far the easiest) way to install the most recent release of the proxy is to use [the Wavefront installer](https://docs.wavefront.com/proxies_installing.html##proxy-installation). This is a simple, one-line installer that configures the Wavefront proxy and/or `collectd` to send telemetry data to Wavefront in as little as one step. - -### Option 2. Use Linux Packages - -We have pre-build packages for popular Linux distros. -* Packages for released versions are available at https://packagecloud.io/wavefront/proxy. -* Release candidate versions are available at https://packagecloud.io/wavefront/proxy-next. - -### Option 3. Build Your Own Proxy - -To build your own version, run the following commands (you need [Apache Maven](https://maven.apache.org) installed for a successful build). - -``` -git clone https://github.com/wavefrontHQ/java -cd java -mvn install -``` ## Advanced Proxy Configuration diff --git a/proxy/brew/wfproxy b/proxy/brew/wfproxy deleted file mode 100644 index 765ef27d8..000000000 --- a/proxy/brew/wfproxy +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Simple wrapper script for the wavefront proxy homebrew installation. - -# resolve links - $0 may be a softlink -CMD="$0" -while [ -h "$CMD" ]; do - ls=`ls -ld "$CMD"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - CMD="$link" - else - CMD=`dirname "$CMD"`/"$link" - fi -done - -CMDDIR=`dirname "$CMD"` - -java -jar $CMDDIR/../lib/proxy-uber.jar "$@" diff --git a/proxy/docker/Dockerfile b/proxy/docker/Dockerfile deleted file mode 100644 index 4ffa6134a..000000000 --- a/proxy/docker/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM ubuntu:18.04 - -# This script may automatically configure wavefront without prompting, based on -# these variables: -# WAVEFRONT_URL (required) -# WAVEFRONT_TOKEN (required) -# JAVA_HEAP_USAGE (default is 4G) -# WAVEFRONT_HOSTNAME (default is the docker containers hostname) -# WAVEFRONT_PROXY_ARGS (default is none) -# JAVA_ARGS (default is none) - -# Dumb-init -RUN apt-get -y update -RUN apt-get install -y curl -RUN apt-get install -y sudo -RUN apt-get install -y gnupg2 -RUN curl -SLO https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64.deb -RUN dpkg -i dumb-init_*.deb -RUN rm dumb-init_*.deb -ENTRYPOINT ["/usr/bin/dumb-init", "--"] - -# Download wavefront proxy (latest release). Merely extract the debian, don't want to try running startup scripts. -RUN curl -s https://packagecloud.io/install/repositories/wavefront/proxy/script.deb.sh | sudo bash -RUN apt-get -d install wavefront-proxy -RUN dpkg -x $(ls /var/cache/apt/archives/wavefront-proxy* | tail -n1) / - -# Download and install JRE, since it's no longer bundled with the proxy -RUN mkdir /opt/wavefront/wavefront-proxy/jre -RUN curl -s https://s3-us-west-2.amazonaws.com/wavefront-misc/proxy-jre.tgz | tar -xz --strip 1 -C /opt/wavefront/wavefront-proxy/jre - -# Clean up APT when done. -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Configure agent -RUN cp /etc/wavefront/wavefront-proxy/log4j2-stdout.xml.default /etc/wavefront/wavefront-proxy/log4j2.xml - -# Run the agent -EXPOSE 3878 -EXPOSE 2878 -EXPOSE 4242 - -ENV PATH=/opt/wavefront/wavefront-proxy/jre/bin:$PATH -ADD run.sh run.sh -CMD ["/bin/bash", "/run.sh"] diff --git a/proxy/docker/run.sh b/proxy/docker/run.sh deleted file mode 100644 index 3eb93148b..000000000 --- a/proxy/docker/run.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -x - -spool_dir="/var/spool/wavefront-proxy" -mkdir -p $spool_dir - -java_heap_usage=${JAVA_HEAP_USAGE:-4G} -java \ - -Xmx$java_heap_usage -Xms$java_heap_usage $JAVA_ARGS\ - -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \ - -Dlog4j.configurationFile=/etc/wavefront/wavefront-proxy/log4j2.xml \ - -jar /opt/wavefront/wavefront-proxy/bin/wavefront-push-agent.jar \ - -h $WAVEFRONT_URL \ - -t $WAVEFRONT_TOKEN \ - --hostname ${WAVEFRONT_HOSTNAME:-$(hostname)} \ - --ephemeral true \ - --buffer ${spool_dir}/buffer} \ - --flushThreads 6 \ - --retryThreads 6 \ - $WAVEFRONT_PROXY_ARGS diff --git a/proxy/pom.xml b/proxy/pom.xml index 864237f41..277d3a36e 100644 --- a/proxy/pom.xml +++ b/proxy/pom.xml @@ -2,374 +2,783 @@ 4.0.0 - - com.wavefront - wavefront - 4.35-SNAPSHOT - + com.wavefront + proxy + 14.1-SNAPSHOT + + Wavefront Proxy + Service for batching and relaying metric traffic to Wavefront + http://www.wavefront.com + + + The Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + German Laullon + glaullon@wavefront.com + Tanzu Observability by Wavefront + https://tanzu.vmware.com/observability + + + + + ${scm.connection} + ${scm.developer.connection} + ${scm.url} + proxy-13.8 + + + + + 11 + + + UTF-8 + UTF-8 + 1.8 + 1.8 + 1.8 + 2.0.9 + + none - proxy + + + + src/main/resources + + + src/main/resources/build/ + true + + + + + com.cosium.code + git-code-format-maven-plugin + 4.3 + + + format-code + validate + + format-code + + + + + ${skipFormatCode} + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0-M3 + + + enforce-java + + enforce + + + + + [1.8,12) + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + 4 + true + ${jacocoArgLine} -Xmx512m -Duser.country=US + 3 + + + java.util.logging.manager + org.apache.logging.log4j.jul.LogManager + + + log4j.configurationFile + src/test/resources/log4j2-dev.xml + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.3 + + + prepare-agent + + prepare-agent + + + jacocoArgLine + + com.tdunning.math.* + org.logstash.* + condition.parser.* + datadog.* + + + + + check + test + + check + + + + + BUNDLE + + + LINE + COVEREDRATIO + 0.74 + + + + + PACKAGE + + com.wavefront.agent.preprocessor + + + + LINE + COVEREDRATIO + 0.75 + + + + + PACKAGE + + com.wavefront.agent.preprocessor.predicate + + + + LINE + COVEREDRATIO + 0.90 + + + + + PACKAGE + + com.wavefront.agent.histogram + + + + LINE + COVEREDRATIO + 0.89 + + + + + + condition/parser/** + com/tdunning/** + org/logstash/** + datadog/** + + + + + coverage-report + test + + report + + + + condition/parser/** + com/tdunning/** + org/logstash/** + + + + + - Wavefront Proxy - Service for batching and relaying metric traffic to Wavefront + + org.springframework.boot + spring-boot-maven-plugin + 2.7.12 + + + + repackage + + + spring-boot + com.wavefront.agent.PushAgent + + + + + + + + + + + + org.apache.commons + commons-collections4 + 4.4 + + + org.objenesis + objenesis + 3.2 + compile + + + io.jaegertracing + jaeger-core + 1.8.1 + compile + + + jakarta.activation + jakarta.activation-api + 1.2.2 + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + com.google.guava + guava + 33.0.0-jre + + + org.apache.commons + commons-lang3 + 3.14.0 + + + org.jetbrains + annotations + 13.0 + compile + + + org.easymock + easymock + 4.3 + test + + + org.yaml + snakeyaml + 2.2 + + + org.jetbrains.kotlin + kotlin-stdlib-common + 1.6.21 + + + com.fasterxml.jackson + jackson-bom + 2.15.2 + pom + import + + + com.google.protobuf + protobuf-bom + 3.25.3 + pom + import + + + com.google.code.gson + gson + 2.10.1 + + + org.slf4j + slf4j-api + 1.8.0-beta4 + + + io.netty + netty-bom + 4.1.108.Final + pom + import + + + com.google.errorprone + error_prone_annotations + 2.25.0 + + + commons-codec + commons-codec + 1.16.1 + + + commons-logging + commons-logging + 1.3.0 + + + commons-io + commons-io + 2.15.1 + + + io.grpc + grpc-bom + 1.62.2 + pom + import + + + io.opentracing + opentracing-api + 0.33.0 + + + joda-time + joda-time + 2.10.14 + + + junit + junit + 4.13.2 + test + + + org.apache.commons + commons-compress + 1.26.0 + + + org.apache.thrift + libthrift + 0.16.0 + compile + + + org.checkerframework + checker-qual + 3.22.0 + + + org.jboss.resteasy + resteasy-bom + 5.0.9.Final + pom + import + + + org.jetbrains.kotlin + kotlin-stdlib + 1.7.0-RC + + + org.apache.logging.log4j + log4j-bom + 2.23.0 + pom + import + + + org.apache.avro + avro + 1.11.4 + + + io.opentelemetry + opentelemetry-bom + 1.35.0 + pom + import + + + com.squareup.okhttp3 + okhttp + 4.12.0 + + + com.wavefront java-lib + 2026-2.3 - com.wavefront - wavefront-sdk-java - 1.1 + com.fasterxml.jackson.module + jackson-module-afterburner + 2.14.0 - com.beust - jcommander - 1.72 + com.github.ben-manes.caffeine + caffeine + 2.9.3 - com.google.code.gson - gson - 2.2.2 + org.apache.httpcomponents + httpclient - com.squareup - tape - 1.2.3 + com.yammer.metrics + metrics-core + 2.2.0 - io.jaegertracing - jaeger-thrift - 0.31.0 + commons-lang + commons-lang + 2.6 - com.uber.tchannel - tchannel-core - 0.8.5 + org.apache.commons + commons-collections4 - org.apache.thrift - libthrift - 0.11.0 + com.google.truth + truth + 1.4.1 + test - io.zipkin.zipkin2 - zipkin - 2.11.12 + com.rubiconproject.oss + jchronic + 0.2.8 - commons-lang - commons-lang + com.wavefront + wavefront-sdk-java + 3.4.3 + compile - commons-collections - commons-collections + com.wavefront + wavefront-internal-reporter-java + 1.7.16 + compile - com.google.guava - guava + com.amazonaws + aws-java-sdk-sqs + 1.12.667 + compile - org.jboss.resteasy - resteasy-jaxrs + com.beust + jcommander + 1.82 + compile - org.jboss.resteasy - resteasy-client + com.squareup.tape2 + tape + 2.0.0-beta1 + compile - org.jboss.resteasy - resteasy-jackson2-provider + io.jaegertracing + jaeger-thrift + 1.8.1 + compile - com.yammer.metrics - metrics-core + io.opentelemetry + opentelemetry-exporter-jaeger-proto + 1.17.0 - org.eclipse.jetty - jetty-server - ${jetty.version} + io.opentelemetry.proto + opentelemetry-proto + 0.17.0-alpha - org.eclipse.jetty - jetty-servlet - ${jetty.version} + com.uber.tchannel + tchannel-core + 0.8.30 + compile - org.eclipse.jetty - jetty-util - ${jetty.version} + io.zipkin.zipkin2 + zipkin + 2.23.16 + compile + + + org.jboss.resteasy + resteasy-client - javax.annotation - javax.annotation-api - 1.2 + org.jboss.resteasy + resteasy-jackson2-provider com.tdunning t-digest - 3.1 + 3.2 + compile - net.openhft - chronicle-map - 3.16.0 + org.easymock + easymock + test - junit - junit + org.powermock + powermock-module-junit4 + ${powermock.version} test - org.easymock - easymock + org.powermock + powermock-api-easymock + ${powermock.version} test - 4.0.1 - org.hamcrest hamcrest-all 1.3 - - - com.google.truth - truth - test - - - joda-time - joda-time - - - com.google.code.findbugs - jsr305 - - - io.netty - netty-codec-http - ${netty.version} - - - io.netty - netty-handler - ${netty.version} - - - io.netty - netty-buffer - ${netty.version} - - - io.netty - netty-transport - ${netty.version} - - - io.netty - netty-resolver - ${netty.version} - - - io.netty - netty-transport-native-epoll - ${netty.version} - linux-x86_64 + compile commons-daemon commons-daemon - 1.0.15 - - - org.yaml - snakeyaml - 1.17 + 1.3.4 + compile - net.jpountz.lz4 - lz4 - 1.3.0 - - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-1.2-api + com.lmax + disruptor + 3.4.4 + compile - org.apache.logging.log4j - log4j-slf4j-impl + io.thekraken + grok + 0.1.5 + compile - org.apache.logging.log4j - log4j-jul + net.jafama + jafama + 2.3.2 + compile - org.apache.logging.log4j - log4j-web + io.grpc + grpc-netty - - com.lmax - disruptor - 3.3.7 + io.grpc + grpc-protobuf + compile - - com.fasterxml.jackson.module - jackson-module-afterburner + com.google.protobuf + protobuf-java - - com.fasterxml.jackson.core - jackson-databind + com.google.protobuf + protobuf-java-util - com.fasterxml.jackson.core - jackson-annotations + net.openhft + chronicle-map + 3.21.86 - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml + org.springframework.boot + spring-boot-starter-log4j2 + 2.7.12 - com.github.ben-manes.caffeine - caffeine - 2.6.2 + io.grpc + grpc-stub - io.thekraken - grok - 0.1.4 + com.google.api.grpc + proto-google-common-protos + 2.0.1 - net.jafama - jafama - 2.1.0 + org.yaml + snakeyaml - - - - src/main/resources - - - src/main/resources/build/ - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.19 - - 4 - false - -Xmx4G - 3 - - - java.util.logging.manager - org.apache.logging.log4j.jul.LogManager - - - log4j.configurationFile - src/test/resources/log4j2-dev.xml - - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.0.0 - - - package - - shade - - - ${project.artifactId}-${project.version}-uber - - - - com.wavefront.agent.PushAgent - - - - - - - - maven-compiler-plugin - 3.1 - - ${java.version} - ${java.version} - - - - org.codehaus.mojo - appassembler-maven-plugin - 2.0.0 - - - generate-scripts - package - - assemble - - - - - -Xmx1G -Xms1G -server -verbosegc -XX:-UseParallelOldGC -XX:OnOutOfMemoryError="kill -9 %p" - -Djava.util.logging.SimpleFormatter.format="%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %4$-6s %2$s %5$s%6$s%n" - - - unix - - - - com.wavefront.agent.PushAgent - agent - - - - - - maven-assembly-plugin - 3.0.0 - - - package-application - package - - single - + + + release + + true + + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar-no-fork + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + sign-artifacts + verify + + true + + + sign + + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.3 + true - - src/assembly.xml - + ossrh + https://s01.oss.sonatype.org/ + true - - - - - + + + + + + broadcom + + git@github.gwd.broadcom.net:IMS/wf-proxy.git + scm:git:${scm.url} + scm:git:${scm.url} + https://usw1.packages.broadcom.com/artifactory + usw1.packages.broadcom.com_tis-wavefront-maven-dev-local + tis-wavefront-maven-dev-local + usw1.packages.broadcom.com_tis-wavefront-maven-prod-local + tis-wavefront-maven-prod-local + + + + ${snapshot.repo.id} + ${artifactory.url}/${snapshot.repo.name} + + + ${release.repo.id} + ${artifactory.url}/${release.repo.name} + + + + + vmware + + true + + + git@github.com:wavefrontHQ/wavefront-proxy.git + scm:git:${scm.url} + scm:git:${scm.url} + https://repo.wavefront.com/content/repositories + sunnylabs-snapshots + snapshots + sunnylabs-releases + releases + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + + + diff --git a/proxy/src/main/java/com/google/common/util/concurrent/RecyclableRateLimiter.java b/proxy/src/main/java/com/google/common/util/concurrent/RecyclableRateLimiter.java deleted file mode 100644 index 4e454209c..000000000 --- a/proxy/src/main/java/com/google/common/util/concurrent/RecyclableRateLimiter.java +++ /dev/null @@ -1,149 +0,0 @@ -package com.google.common.util.concurrent; - -import com.google.common.math.LongMath; - -import static java.lang.Math.max; -import static java.lang.Math.min; -import static java.util.concurrent.TimeUnit.SECONDS; - -/** - * An alternative RateLimiter implementation that allows to "return" unused permits back to the pool to handle retries - * gracefully and allow precise control over outgoing rate, plus allows accumulating "credits" for unused permits over - * a time window other than 1 second. - * - * Created by: vasily@wavefront.com, with portions from Guava library source code - */ -public class RecyclableRateLimiter extends RateLimiter { - /** - * The currently stored permits. - */ - private double storedPermits; - - /** - * The maximum number of stored permits. - */ - private double maxPermits; - - /** - * The interval between two unit requests, at our stable rate. E.g., a stable rate of 5 permits - * per second has a stable interval of 200ms. - */ - private double stableIntervalMicros; - - /** - * The time when the next request (no matter its size) will be granted. After granting a - * request, this is pushed further in the future. Large requests push this further than small - * requests. - */ - private long nextFreeTicketMicros = 0L; // could be either in the past or future - - /** The work (permits) of how many seconds can be saved up if this RateLimiter is unused? */ - private final double maxBurstSeconds; - - private final SleepingStopwatch stopwatch; - - private final Object mutex; - - public static RecyclableRateLimiter create(double permitsPerSecond, double maxBurstSeconds) { - return new RecyclableRateLimiter( - SleepingStopwatch.createFromSystemTimer(), - permitsPerSecond, - maxBurstSeconds); - } - - private RecyclableRateLimiter(SleepingStopwatch stopwatch, double permitsPerSecond, double maxBurstSeconds) { - super(stopwatch); - this.mutex = new Object(); - this.stopwatch = stopwatch; - this.maxBurstSeconds = maxBurstSeconds; - this.setRate(permitsPerSecond); - } - - /** - * Get the number of accumulated permits - * - * @return number of accumulated permits - */ - public double getAvailablePermits() { - synchronized (mutex) { - resync(stopwatch.readMicros()); - return storedPermits; - } - } - - /** - * Return the specified number of permits back to the pool - * - * @param permits number of permits to return - */ - public void recyclePermits(int permits) { - synchronized (mutex) { - long nowMicros = stopwatch.readMicros(); - resync(nowMicros); - long surplusPermits = permits - (long) ((nextFreeTicketMicros - nowMicros) / stableIntervalMicros); - long waitMicros = -min((long) (surplusPermits * stableIntervalMicros), 0L); - try { - this.nextFreeTicketMicros = LongMath.checkedAdd(nowMicros, waitMicros); - } catch (ArithmeticException e) { - this.nextFreeTicketMicros = Long.MAX_VALUE; - } - storedPermits = min(maxPermits, storedPermits + max(surplusPermits, 0L)); - } - } - - @Override - final void doSetRate(double permitsPerSecond, long nowMicros) { - synchronized (mutex) { - resync(nowMicros); - this.stableIntervalMicros = SECONDS.toMicros(1L) / permitsPerSecond; - double oldMaxPermits = this.maxPermits; - maxPermits = maxBurstSeconds * permitsPerSecond; - storedPermits = (oldMaxPermits == Double.POSITIVE_INFINITY) - ? maxPermits - : (oldMaxPermits == 0.0) - ? 0.0 // initial state - : storedPermits * maxPermits / oldMaxPermits; - } - } - - @Override - final double doGetRate() { - return SECONDS.toMicros(1L) / stableIntervalMicros; - } - - @Override - final long queryEarliestAvailable(long nowMicros) { - synchronized (mutex) { - return nextFreeTicketMicros; - } - } - - @Override - final long reserveEarliestAvailable(int requiredPermits, long nowMicros) { - synchronized (mutex) { - resync(nowMicros); - long returnValue = nextFreeTicketMicros; - double storedPermitsToSpend = min(requiredPermits, this.storedPermits); - double freshPermits = requiredPermits - storedPermitsToSpend; - long waitMicros = (long) (freshPermits * stableIntervalMicros); - - try { - this.nextFreeTicketMicros = LongMath.checkedAdd(nextFreeTicketMicros, waitMicros); - } catch (ArithmeticException e) { - this.nextFreeTicketMicros = Long.MAX_VALUE; - } - this.storedPermits -= storedPermitsToSpend; - return returnValue; - } - } - - private void resync(long nowMicros) { - // if nextFreeTicket is in the past, resync to now - if (nowMicros > nextFreeTicketMicros) { - storedPermits = min(maxPermits, - storedPermits - + (nowMicros - nextFreeTicketMicros) / stableIntervalMicros); - nextFreeTicketMicros = nowMicros; - } - } -} diff --git a/proxy/src/main/java/com/google/protobuf/GoGoProtos.java b/proxy/src/main/java/com/google/protobuf/GoGoProtos.java new file mode 100644 index 000000000..dd8ad2d15 --- /dev/null +++ b/proxy/src/main/java/com/google/protobuf/GoGoProtos.java @@ -0,0 +1,907 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: github.com/gogo/protobuf/gogoproto/gogo.proto + +package com.google.protobuf; + +public final class GoGoProtos { + private GoGoProtos() {} + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) { + registry.add(com.google.protobuf.GoGoProtos.goprotoEnumPrefix); + registry.add(com.google.protobuf.GoGoProtos.goprotoEnumStringer); + registry.add(com.google.protobuf.GoGoProtos.enumStringer); + registry.add(com.google.protobuf.GoGoProtos.enumCustomname); + registry.add(com.google.protobuf.GoGoProtos.enumdecl); + registry.add(com.google.protobuf.GoGoProtos.enumvalueCustomname); + registry.add(com.google.protobuf.GoGoProtos.goprotoGettersAll); + registry.add(com.google.protobuf.GoGoProtos.goprotoEnumPrefixAll); + registry.add(com.google.protobuf.GoGoProtos.goprotoStringerAll); + registry.add(com.google.protobuf.GoGoProtos.verboseEqualAll); + registry.add(com.google.protobuf.GoGoProtos.faceAll); + registry.add(com.google.protobuf.GoGoProtos.gostringAll); + registry.add(com.google.protobuf.GoGoProtos.populateAll); + registry.add(com.google.protobuf.GoGoProtos.stringerAll); + registry.add(com.google.protobuf.GoGoProtos.onlyoneAll); + registry.add(com.google.protobuf.GoGoProtos.equalAll); + registry.add(com.google.protobuf.GoGoProtos.descriptionAll); + registry.add(com.google.protobuf.GoGoProtos.testgenAll); + registry.add(com.google.protobuf.GoGoProtos.benchgenAll); + registry.add(com.google.protobuf.GoGoProtos.marshalerAll); + registry.add(com.google.protobuf.GoGoProtos.unmarshalerAll); + registry.add(com.google.protobuf.GoGoProtos.stableMarshalerAll); + registry.add(com.google.protobuf.GoGoProtos.sizerAll); + registry.add(com.google.protobuf.GoGoProtos.goprotoEnumStringerAll); + registry.add(com.google.protobuf.GoGoProtos.enumStringerAll); + registry.add(com.google.protobuf.GoGoProtos.unsafeMarshalerAll); + registry.add(com.google.protobuf.GoGoProtos.unsafeUnmarshalerAll); + registry.add(com.google.protobuf.GoGoProtos.goprotoExtensionsMapAll); + registry.add(com.google.protobuf.GoGoProtos.goprotoUnrecognizedAll); + registry.add(com.google.protobuf.GoGoProtos.gogoprotoImport); + registry.add(com.google.protobuf.GoGoProtos.protosizerAll); + registry.add(com.google.protobuf.GoGoProtos.compareAll); + registry.add(com.google.protobuf.GoGoProtos.typedeclAll); + registry.add(com.google.protobuf.GoGoProtos.enumdeclAll); + registry.add(com.google.protobuf.GoGoProtos.goprotoRegistration); + registry.add(com.google.protobuf.GoGoProtos.messagenameAll); + registry.add(com.google.protobuf.GoGoProtos.goprotoSizecacheAll); + registry.add(com.google.protobuf.GoGoProtos.goprotoUnkeyedAll); + registry.add(com.google.protobuf.GoGoProtos.goprotoGetters); + registry.add(com.google.protobuf.GoGoProtos.goprotoStringer); + registry.add(com.google.protobuf.GoGoProtos.verboseEqual); + registry.add(com.google.protobuf.GoGoProtos.face); + registry.add(com.google.protobuf.GoGoProtos.gostring); + registry.add(com.google.protobuf.GoGoProtos.populate); + registry.add(com.google.protobuf.GoGoProtos.stringer); + registry.add(com.google.protobuf.GoGoProtos.onlyone); + registry.add(com.google.protobuf.GoGoProtos.equal); + registry.add(com.google.protobuf.GoGoProtos.description); + registry.add(com.google.protobuf.GoGoProtos.testgen); + registry.add(com.google.protobuf.GoGoProtos.benchgen); + registry.add(com.google.protobuf.GoGoProtos.marshaler); + registry.add(com.google.protobuf.GoGoProtos.unmarshaler); + registry.add(com.google.protobuf.GoGoProtos.stableMarshaler); + registry.add(com.google.protobuf.GoGoProtos.sizer); + registry.add(com.google.protobuf.GoGoProtos.unsafeMarshaler); + registry.add(com.google.protobuf.GoGoProtos.unsafeUnmarshaler); + registry.add(com.google.protobuf.GoGoProtos.goprotoExtensionsMap); + registry.add(com.google.protobuf.GoGoProtos.goprotoUnrecognized); + registry.add(com.google.protobuf.GoGoProtos.protosizer); + registry.add(com.google.protobuf.GoGoProtos.compare); + registry.add(com.google.protobuf.GoGoProtos.typedecl); + registry.add(com.google.protobuf.GoGoProtos.messagename); + registry.add(com.google.protobuf.GoGoProtos.goprotoSizecache); + registry.add(com.google.protobuf.GoGoProtos.goprotoUnkeyed); + registry.add(com.google.protobuf.GoGoProtos.nullable); + registry.add(com.google.protobuf.GoGoProtos.embed); + registry.add(com.google.protobuf.GoGoProtos.customtype); + registry.add(com.google.protobuf.GoGoProtos.customname); + registry.add(com.google.protobuf.GoGoProtos.jsontag); + registry.add(com.google.protobuf.GoGoProtos.moretags); + registry.add(com.google.protobuf.GoGoProtos.casttype); + registry.add(com.google.protobuf.GoGoProtos.castkey); + registry.add(com.google.protobuf.GoGoProtos.castvalue); + registry.add(com.google.protobuf.GoGoProtos.stdtime); + registry.add(com.google.protobuf.GoGoProtos.stdduration); + registry.add(com.google.protobuf.GoGoProtos.wktpointer); + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry); + } + + public static final int GOPROTO_ENUM_PREFIX_FIELD_NUMBER = 62001; + /** extend .google.protobuf.EnumOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.EnumOptions, java.lang.Boolean> + goprotoEnumPrefix = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_ENUM_STRINGER_FIELD_NUMBER = 62021; + /** extend .google.protobuf.EnumOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.EnumOptions, java.lang.Boolean> + goprotoEnumStringer = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int ENUM_STRINGER_FIELD_NUMBER = 62022; + /** extend .google.protobuf.EnumOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.EnumOptions, java.lang.Boolean> + enumStringer = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int ENUM_CUSTOMNAME_FIELD_NUMBER = 62023; + /** extend .google.protobuf.EnumOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.EnumOptions, java.lang.String> + enumCustomname = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.String.class, null); + + public static final int ENUMDECL_FIELD_NUMBER = 62024; + /** extend .google.protobuf.EnumOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.EnumOptions, java.lang.Boolean> + enumdecl = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int ENUMVALUE_CUSTOMNAME_FIELD_NUMBER = 66001; + /** extend .google.protobuf.EnumValueOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.EnumValueOptions, java.lang.String> + enumvalueCustomname = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.String.class, null); + + public static final int GOPROTO_GETTERS_ALL_FIELD_NUMBER = 63001; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + goprotoGettersAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_ENUM_PREFIX_ALL_FIELD_NUMBER = 63002; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + goprotoEnumPrefixAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_STRINGER_ALL_FIELD_NUMBER = 63003; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + goprotoStringerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int VERBOSE_EQUAL_ALL_FIELD_NUMBER = 63004; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + verboseEqualAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int FACE_ALL_FIELD_NUMBER = 63005; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + faceAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOSTRING_ALL_FIELD_NUMBER = 63006; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + gostringAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int POPULATE_ALL_FIELD_NUMBER = 63007; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + populateAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int STRINGER_ALL_FIELD_NUMBER = 63008; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + stringerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int ONLYONE_ALL_FIELD_NUMBER = 63009; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + onlyoneAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int EQUAL_ALL_FIELD_NUMBER = 63013; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + equalAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int DESCRIPTION_ALL_FIELD_NUMBER = 63014; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + descriptionAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int TESTGEN_ALL_FIELD_NUMBER = 63015; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + testgenAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int BENCHGEN_ALL_FIELD_NUMBER = 63016; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + benchgenAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int MARSHALER_ALL_FIELD_NUMBER = 63017; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + marshalerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int UNMARSHALER_ALL_FIELD_NUMBER = 63018; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + unmarshalerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int STABLE_MARSHALER_ALL_FIELD_NUMBER = 63019; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + stableMarshalerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int SIZER_ALL_FIELD_NUMBER = 63020; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + sizerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_ENUM_STRINGER_ALL_FIELD_NUMBER = 63021; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + goprotoEnumStringerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int ENUM_STRINGER_ALL_FIELD_NUMBER = 63022; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + enumStringerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int UNSAFE_MARSHALER_ALL_FIELD_NUMBER = 63023; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + unsafeMarshalerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int UNSAFE_UNMARSHALER_ALL_FIELD_NUMBER = 63024; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + unsafeUnmarshalerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_EXTENSIONS_MAP_ALL_FIELD_NUMBER = 63025; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + goprotoExtensionsMapAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_UNRECOGNIZED_ALL_FIELD_NUMBER = 63026; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + goprotoUnrecognizedAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOGOPROTO_IMPORT_FIELD_NUMBER = 63027; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + gogoprotoImport = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int PROTOSIZER_ALL_FIELD_NUMBER = 63028; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + protosizerAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int COMPARE_ALL_FIELD_NUMBER = 63029; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + compareAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int TYPEDECL_ALL_FIELD_NUMBER = 63030; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + typedeclAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int ENUMDECL_ALL_FIELD_NUMBER = 63031; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + enumdeclAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_REGISTRATION_FIELD_NUMBER = 63032; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + goprotoRegistration = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int MESSAGENAME_ALL_FIELD_NUMBER = 63033; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + messagenameAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_SIZECACHE_ALL_FIELD_NUMBER = 63034; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + goprotoSizecacheAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_UNKEYED_ALL_FIELD_NUMBER = 63035; + /** extend .google.protobuf.FileOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FileOptions, java.lang.Boolean> + goprotoUnkeyedAll = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_GETTERS_FIELD_NUMBER = 64001; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + goprotoGetters = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_STRINGER_FIELD_NUMBER = 64003; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + goprotoStringer = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int VERBOSE_EQUAL_FIELD_NUMBER = 64004; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + verboseEqual = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int FACE_FIELD_NUMBER = 64005; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + face = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOSTRING_FIELD_NUMBER = 64006; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + gostring = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int POPULATE_FIELD_NUMBER = 64007; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + populate = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int STRINGER_FIELD_NUMBER = 67008; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + stringer = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int ONLYONE_FIELD_NUMBER = 64009; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + onlyone = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int EQUAL_FIELD_NUMBER = 64013; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + equal = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int DESCRIPTION_FIELD_NUMBER = 64014; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + description = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int TESTGEN_FIELD_NUMBER = 64015; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + testgen = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int BENCHGEN_FIELD_NUMBER = 64016; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + benchgen = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int MARSHALER_FIELD_NUMBER = 64017; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + marshaler = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int UNMARSHALER_FIELD_NUMBER = 64018; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + unmarshaler = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int STABLE_MARSHALER_FIELD_NUMBER = 64019; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + stableMarshaler = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int SIZER_FIELD_NUMBER = 64020; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + sizer = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int UNSAFE_MARSHALER_FIELD_NUMBER = 64023; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + unsafeMarshaler = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int UNSAFE_UNMARSHALER_FIELD_NUMBER = 64024; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + unsafeUnmarshaler = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_EXTENSIONS_MAP_FIELD_NUMBER = 64025; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + goprotoExtensionsMap = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_UNRECOGNIZED_FIELD_NUMBER = 64026; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + goprotoUnrecognized = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int PROTOSIZER_FIELD_NUMBER = 64028; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + protosizer = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int COMPARE_FIELD_NUMBER = 64029; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + compare = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int TYPEDECL_FIELD_NUMBER = 64030; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + typedecl = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int MESSAGENAME_FIELD_NUMBER = 64033; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + messagename = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_SIZECACHE_FIELD_NUMBER = 64034; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + goprotoSizecache = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int GOPROTO_UNKEYED_FIELD_NUMBER = 64035; + /** extend .google.protobuf.MessageOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.MessageOptions, java.lang.Boolean> + goprotoUnkeyed = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int NULLABLE_FIELD_NUMBER = 65001; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.Boolean> + nullable = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int EMBED_FIELD_NUMBER = 65002; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.Boolean> + embed = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int CUSTOMTYPE_FIELD_NUMBER = 65003; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.String> + customtype = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.String.class, null); + + public static final int CUSTOMNAME_FIELD_NUMBER = 65004; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.String> + customname = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.String.class, null); + + public static final int JSONTAG_FIELD_NUMBER = 65005; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.String> + jsontag = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.String.class, null); + + public static final int MORETAGS_FIELD_NUMBER = 65006; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.String> + moretags = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.String.class, null); + + public static final int CASTTYPE_FIELD_NUMBER = 65007; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.String> + casttype = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.String.class, null); + + public static final int CASTKEY_FIELD_NUMBER = 65008; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.String> + castkey = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.String.class, null); + + public static final int CASTVALUE_FIELD_NUMBER = 65009; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.String> + castvalue = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.String.class, null); + + public static final int STDTIME_FIELD_NUMBER = 65010; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.Boolean> + stdtime = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int STDDURATION_FIELD_NUMBER = 65011; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.Boolean> + stdduration = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static final int WKTPOINTER_FIELD_NUMBER = 65012; + /** extend .google.protobuf.FieldOptions { ... } */ + public static final com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, java.lang.Boolean> + wktpointer = + com.google.protobuf.GeneratedMessage.newFileScopedGeneratedExtension( + java.lang.Boolean.class, null); + + public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + return descriptor; + } + + private static com.google.protobuf.Descriptors.FileDescriptor descriptor; + + static { + java.lang.String[] descriptorData = { + "\n-github.com/gogo/protobuf/gogoproto/gog" + + "o.proto\022\tgogoproto\032 google/protobuf/desc" + + "riptor.proto:;\n\023goproto_enum_prefix\022\034.go" + + "ogle.protobuf.EnumOptions\030\261\344\003 \001(\010:=\n\025gop" + + "roto_enum_stringer\022\034.google.protobuf.Enu" + + "mOptions\030\305\344\003 \001(\010:5\n\renum_stringer\022\034.goog" + + "le.protobuf.EnumOptions\030\306\344\003 \001(\010:7\n\017enum_" + + "customname\022\034.google.protobuf.EnumOptions" + + "\030\307\344\003 \001(\t:0\n\010enumdecl\022\034.google.protobuf.E" + + "numOptions\030\310\344\003 \001(\010:A\n\024enumvalue_customna" + + "me\022!.google.protobuf.EnumValueOptions\030\321\203" + + "\004 \001(\t:;\n\023goproto_getters_all\022\034.google.pr" + + "otobuf.FileOptions\030\231\354\003 \001(\010:?\n\027goproto_en" + + "um_prefix_all\022\034.google.protobuf.FileOpti" + + "ons\030\232\354\003 \001(\010:<\n\024goproto_stringer_all\022\034.go" + + "ogle.protobuf.FileOptions\030\233\354\003 \001(\010:9\n\021ver" + + "bose_equal_all\022\034.google.protobuf.FileOpt" + + "ions\030\234\354\003 \001(\010:0\n\010face_all\022\034.google.protob" + + "uf.FileOptions\030\235\354\003 \001(\010:4\n\014gostring_all\022\034" + + ".google.protobuf.FileOptions\030\236\354\003 \001(\010:4\n\014" + + "populate_all\022\034.google.protobuf.FileOptio" + + "ns\030\237\354\003 \001(\010:4\n\014stringer_all\022\034.google.prot" + + "obuf.FileOptions\030\240\354\003 \001(\010:3\n\013onlyone_all\022" + + "\034.google.protobuf.FileOptions\030\241\354\003 \001(\010:1\n" + + "\tequal_all\022\034.google.protobuf.FileOptions" + + "\030\245\354\003 \001(\010:7\n\017description_all\022\034.google.pro" + + "tobuf.FileOptions\030\246\354\003 \001(\010:3\n\013testgen_all" + + "\022\034.google.protobuf.FileOptions\030\247\354\003 \001(\010:4" + + "\n\014benchgen_all\022\034.google.protobuf.FileOpt" + + "ions\030\250\354\003 \001(\010:5\n\rmarshaler_all\022\034.google.p" + + "rotobuf.FileOptions\030\251\354\003 \001(\010:7\n\017unmarshal" + + "er_all\022\034.google.protobuf.FileOptions\030\252\354\003" + + " \001(\010:<\n\024stable_marshaler_all\022\034.google.pr" + + "otobuf.FileOptions\030\253\354\003 \001(\010:1\n\tsizer_all\022" + + "\034.google.protobuf.FileOptions\030\254\354\003 \001(\010:A\n" + + "\031goproto_enum_stringer_all\022\034.google.prot" + + "obuf.FileOptions\030\255\354\003 \001(\010:9\n\021enum_stringe" + + "r_all\022\034.google.protobuf.FileOptions\030\256\354\003 " + + "\001(\010:<\n\024unsafe_marshaler_all\022\034.google.pro" + + "tobuf.FileOptions\030\257\354\003 \001(\010:>\n\026unsafe_unma" + + "rshaler_all\022\034.google.protobuf.FileOption" + + "s\030\260\354\003 \001(\010:B\n\032goproto_extensions_map_all\022" + + "\034.google.protobuf.FileOptions\030\261\354\003 \001(\010:@\n" + + "\030goproto_unrecognized_all\022\034.google.proto" + + "buf.FileOptions\030\262\354\003 \001(\010:8\n\020gogoproto_imp" + + "ort\022\034.google.protobuf.FileOptions\030\263\354\003 \001(" + + "\010:6\n\016protosizer_all\022\034.google.protobuf.Fi" + + "leOptions\030\264\354\003 \001(\010:3\n\013compare_all\022\034.googl" + + "e.protobuf.FileOptions\030\265\354\003 \001(\010:4\n\014typede" + + "cl_all\022\034.google.protobuf.FileOptions\030\266\354\003" + + " \001(\010:4\n\014enumdecl_all\022\034.google.protobuf.F" + + "ileOptions\030\267\354\003 \001(\010:<\n\024goproto_registrati" + + "on\022\034.google.protobuf.FileOptions\030\270\354\003 \001(\010" + + ":7\n\017messagename_all\022\034.google.protobuf.Fi" + + "leOptions\030\271\354\003 \001(\010:=\n\025goproto_sizecache_a" + + "ll\022\034.google.protobuf.FileOptions\030\272\354\003 \001(\010" + + ":;\n\023goproto_unkeyed_all\022\034.google.protobu" + + "f.FileOptions\030\273\354\003 \001(\010::\n\017goproto_getters" + + "\022\037.google.protobuf.MessageOptions\030\201\364\003 \001(" + + "\010:;\n\020goproto_stringer\022\037.google.protobuf." + + "MessageOptions\030\203\364\003 \001(\010:8\n\rverbose_equal\022" + + "\037.google.protobuf.MessageOptions\030\204\364\003 \001(\010" + + ":/\n\004face\022\037.google.protobuf.MessageOption" + + "s\030\205\364\003 \001(\010:3\n\010gostring\022\037.google.protobuf." + + "MessageOptions\030\206\364\003 \001(\010:3\n\010populate\022\037.goo" + + "gle.protobuf.MessageOptions\030\207\364\003 \001(\010:3\n\010s" + + "tringer\022\037.google.protobuf.MessageOptions" + + "\030\300\213\004 \001(\010:2\n\007onlyone\022\037.google.protobuf.Me" + + "ssageOptions\030\211\364\003 \001(\010:0\n\005equal\022\037.google.p" + + "rotobuf.MessageOptions\030\215\364\003 \001(\010:6\n\013descri" + + "ption\022\037.google.protobuf.MessageOptions\030\216" + + "\364\003 \001(\010:2\n\007testgen\022\037.google.protobuf.Mess" + + "ageOptions\030\217\364\003 \001(\010:3\n\010benchgen\022\037.google." + + "protobuf.MessageOptions\030\220\364\003 \001(\010:4\n\tmarsh" + + "aler\022\037.google.protobuf.MessageOptions\030\221\364" + + "\003 \001(\010:6\n\013unmarshaler\022\037.google.protobuf.M" + + "essageOptions\030\222\364\003 \001(\010:;\n\020stable_marshale" + + "r\022\037.google.protobuf.MessageOptions\030\223\364\003 \001" + + "(\010:0\n\005sizer\022\037.google.protobuf.MessageOpt" + + "ions\030\224\364\003 \001(\010:;\n\020unsafe_marshaler\022\037.googl" + + "e.protobuf.MessageOptions\030\227\364\003 \001(\010:=\n\022uns" + + "afe_unmarshaler\022\037.google.protobuf.Messag" + + "eOptions\030\230\364\003 \001(\010:A\n\026goproto_extensions_m" + + "ap\022\037.google.protobuf.MessageOptions\030\231\364\003 " + + "\001(\010:?\n\024goproto_unrecognized\022\037.google.pro" + + "tobuf.MessageOptions\030\232\364\003 \001(\010:5\n\nprotosiz" + + "er\022\037.google.protobuf.MessageOptions\030\234\364\003 " + + "\001(\010:2\n\007compare\022\037.google.protobuf.Message" + + "Options\030\235\364\003 \001(\010:3\n\010typedecl\022\037.google.pro" + + "tobuf.MessageOptions\030\236\364\003 \001(\010:6\n\013messagen" + + "ame\022\037.google.protobuf.MessageOptions\030\241\364\003" + + " \001(\010:<\n\021goproto_sizecache\022\037.google.proto" + + "buf.MessageOptions\030\242\364\003 \001(\010::\n\017goproto_un" + + "keyed\022\037.google.protobuf.MessageOptions\030\243" + + "\364\003 \001(\010:1\n\010nullable\022\035.google.protobuf.Fie" + + "ldOptions\030\351\373\003 \001(\010:.\n\005embed\022\035.google.prot" + + "obuf.FieldOptions\030\352\373\003 \001(\010:3\n\ncustomtype\022" + + "\035.google.protobuf.FieldOptions\030\353\373\003 \001(\t:3" + + "\n\ncustomname\022\035.google.protobuf.FieldOpti" + + "ons\030\354\373\003 \001(\t:0\n\007jsontag\022\035.google.protobuf" + + ".FieldOptions\030\355\373\003 \001(\t:1\n\010moretags\022\035.goog" + + "le.protobuf.FieldOptions\030\356\373\003 \001(\t:1\n\010cast" + + "type\022\035.google.protobuf.FieldOptions\030\357\373\003 " + + "\001(\t:0\n\007castkey\022\035.google.protobuf.FieldOp" + + "tions\030\360\373\003 \001(\t:2\n\tcastvalue\022\035.google.prot" + + "obuf.FieldOptions\030\361\373\003 \001(\t:0\n\007stdtime\022\035.g" + + "oogle.protobuf.FieldOptions\030\362\373\003 \001(\010:4\n\013s" + + "tdduration\022\035.google.protobuf.FieldOption" + + "s\030\363\373\003 \001(\010:3\n\nwktpointer\022\035.google.protobu" + + "f.FieldOptions\030\364\373\003 \001(\010BE\n\023com.google.pro" + + "tobufB\nGoGoProtosZ\"github.com/gogo/proto" + + "buf/gogoproto" + }; + descriptor = + com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom( + descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + com.google.protobuf.DescriptorProtos.getDescriptor(), + }); + goprotoEnumPrefix.internalInit(descriptor.getExtensions().get(0)); + goprotoEnumStringer.internalInit(descriptor.getExtensions().get(1)); + enumStringer.internalInit(descriptor.getExtensions().get(2)); + enumCustomname.internalInit(descriptor.getExtensions().get(3)); + enumdecl.internalInit(descriptor.getExtensions().get(4)); + enumvalueCustomname.internalInit(descriptor.getExtensions().get(5)); + goprotoGettersAll.internalInit(descriptor.getExtensions().get(6)); + goprotoEnumPrefixAll.internalInit(descriptor.getExtensions().get(7)); + goprotoStringerAll.internalInit(descriptor.getExtensions().get(8)); + verboseEqualAll.internalInit(descriptor.getExtensions().get(9)); + faceAll.internalInit(descriptor.getExtensions().get(10)); + gostringAll.internalInit(descriptor.getExtensions().get(11)); + populateAll.internalInit(descriptor.getExtensions().get(12)); + stringerAll.internalInit(descriptor.getExtensions().get(13)); + onlyoneAll.internalInit(descriptor.getExtensions().get(14)); + equalAll.internalInit(descriptor.getExtensions().get(15)); + descriptionAll.internalInit(descriptor.getExtensions().get(16)); + testgenAll.internalInit(descriptor.getExtensions().get(17)); + benchgenAll.internalInit(descriptor.getExtensions().get(18)); + marshalerAll.internalInit(descriptor.getExtensions().get(19)); + unmarshalerAll.internalInit(descriptor.getExtensions().get(20)); + stableMarshalerAll.internalInit(descriptor.getExtensions().get(21)); + sizerAll.internalInit(descriptor.getExtensions().get(22)); + goprotoEnumStringerAll.internalInit(descriptor.getExtensions().get(23)); + enumStringerAll.internalInit(descriptor.getExtensions().get(24)); + unsafeMarshalerAll.internalInit(descriptor.getExtensions().get(25)); + unsafeUnmarshalerAll.internalInit(descriptor.getExtensions().get(26)); + goprotoExtensionsMapAll.internalInit(descriptor.getExtensions().get(27)); + goprotoUnrecognizedAll.internalInit(descriptor.getExtensions().get(28)); + gogoprotoImport.internalInit(descriptor.getExtensions().get(29)); + protosizerAll.internalInit(descriptor.getExtensions().get(30)); + compareAll.internalInit(descriptor.getExtensions().get(31)); + typedeclAll.internalInit(descriptor.getExtensions().get(32)); + enumdeclAll.internalInit(descriptor.getExtensions().get(33)); + goprotoRegistration.internalInit(descriptor.getExtensions().get(34)); + messagenameAll.internalInit(descriptor.getExtensions().get(35)); + goprotoSizecacheAll.internalInit(descriptor.getExtensions().get(36)); + goprotoUnkeyedAll.internalInit(descriptor.getExtensions().get(37)); + goprotoGetters.internalInit(descriptor.getExtensions().get(38)); + goprotoStringer.internalInit(descriptor.getExtensions().get(39)); + verboseEqual.internalInit(descriptor.getExtensions().get(40)); + face.internalInit(descriptor.getExtensions().get(41)); + gostring.internalInit(descriptor.getExtensions().get(42)); + populate.internalInit(descriptor.getExtensions().get(43)); + stringer.internalInit(descriptor.getExtensions().get(44)); + onlyone.internalInit(descriptor.getExtensions().get(45)); + equal.internalInit(descriptor.getExtensions().get(46)); + description.internalInit(descriptor.getExtensions().get(47)); + testgen.internalInit(descriptor.getExtensions().get(48)); + benchgen.internalInit(descriptor.getExtensions().get(49)); + marshaler.internalInit(descriptor.getExtensions().get(50)); + unmarshaler.internalInit(descriptor.getExtensions().get(51)); + stableMarshaler.internalInit(descriptor.getExtensions().get(52)); + sizer.internalInit(descriptor.getExtensions().get(53)); + unsafeMarshaler.internalInit(descriptor.getExtensions().get(54)); + unsafeUnmarshaler.internalInit(descriptor.getExtensions().get(55)); + goprotoExtensionsMap.internalInit(descriptor.getExtensions().get(56)); + goprotoUnrecognized.internalInit(descriptor.getExtensions().get(57)); + protosizer.internalInit(descriptor.getExtensions().get(58)); + compare.internalInit(descriptor.getExtensions().get(59)); + typedecl.internalInit(descriptor.getExtensions().get(60)); + messagename.internalInit(descriptor.getExtensions().get(61)); + goprotoSizecache.internalInit(descriptor.getExtensions().get(62)); + goprotoUnkeyed.internalInit(descriptor.getExtensions().get(63)); + nullable.internalInit(descriptor.getExtensions().get(64)); + embed.internalInit(descriptor.getExtensions().get(65)); + customtype.internalInit(descriptor.getExtensions().get(66)); + customname.internalInit(descriptor.getExtensions().get(67)); + jsontag.internalInit(descriptor.getExtensions().get(68)); + moretags.internalInit(descriptor.getExtensions().get(69)); + casttype.internalInit(descriptor.getExtensions().get(70)); + castkey.internalInit(descriptor.getExtensions().get(71)); + castvalue.internalInit(descriptor.getExtensions().get(72)); + stdtime.internalInit(descriptor.getExtensions().get(73)); + stdduration.internalInit(descriptor.getExtensions().get(74)); + wktpointer.internalInit(descriptor.getExtensions().get(75)); + com.google.protobuf.DescriptorProtos.getDescriptor(); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/proxy/src/main/java/com/tdunning/math/stats/AgentDigest.java b/proxy/src/main/java/com/tdunning/math/stats/AgentDigest.java index 1c43f498e..abf33f088 100644 --- a/proxy/src/main/java/com/tdunning/math/stats/AgentDigest.java +++ b/proxy/src/main/java/com/tdunning/math/stats/AgentDigest.java @@ -1,10 +1,16 @@ package com.tdunning.math.stats; import com.google.common.base.Preconditions; - import com.yammer.metrics.Metrics; import com.yammer.metrics.core.MetricName; - +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import net.jafama.FastMath; import net.openhft.chronicle.bytes.Bytes; import net.openhft.chronicle.core.io.IORuntimeException; @@ -13,47 +19,43 @@ import net.openhft.chronicle.hash.serialization.SizedWriter; import net.openhft.chronicle.wire.WireIn; import net.openhft.chronicle.wire.WireOut; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - import wavefront.report.Histogram; import wavefront.report.HistogramType; /** - * NOTE: This is a pruned and modified version of {@link MergingDigest}. It does not support queries (cdf/quantiles) or - * the traditional encodings. - *

- * Maintains a t-digest by collecting new points in a buffer that is then sorted occasionally and merged into a sorted - * array that contains previously computed centroids. - *

- * This can be very fast because the cost of sorting and merging is amortized over several insertion. If we keep N - * centroids total and have the input array is k long, then the amortized cost is something like - *

- * N/k + log k - *

- * These costs even out when N/k = log k. Balancing costs is often a good place to start in optimizing an algorithm. - * For different values of compression factor, the following table shows estimated asymptotic values of N and suggested - * values of k: + * NOTE: This is a pruned and modified version of {@link MergingDigest}. It does not support queries + * (cdf/quantiles) or the traditional encodings. + * + *

Maintains a t-digest by collecting new points in a buffer that is then sorted occasionally and + * merged into a sorted array that contains previously computed centroids. + * + *

This can be very fast because the cost of sorting and merging is amortized over several + * insertion. If we keep N centroids total and have the input array is k long, then the amortized + * cost is something like + * + *

N/k + log k + * + *

These costs even out when N/k = log k. Balancing costs is often a good place to start in + * optimizing an algorithm. For different values of compression factor, the following table shows + * estimated asymptotic values of N and suggested values of k: + * + *

CompressionNk
* *
CompressionNk
507825
10015742
20031473
- *

- * The virtues of this kind of t-digest implementation include:

  • No allocation is required after - * initialization
  • The data structure automatically compresses existing centroids when possible
  • No Java - * object overhead is incurred for centroids since data is kept in primitive arrays
- *

- * The current implementation takes the liberty of using ping-pong buffers for implementing the merge resulting in a - * substantial memory penalty, but the complexity of an in place merge was not considered as worthwhile since even with - * the overhead, the memory cost is less than 40 bytes per centroid which is much less than half what the AVLTreeDigest - * uses. Speed tests are still not complete so it is uncertain whether the merge strategy is faster than the tree - * strategy. + * + *

The virtues of this kind of t-digest implementation include: + * + *

    + *
  • No allocation is required after initialization + *
  • The data structure automatically compresses existing centroids when possible + *
  • No Java object overhead is incurred for centroids since data is kept in primitive arrays + *
+ * + *

The current implementation takes the liberty of using ping-pong buffers for implementing the + * merge resulting in a substantial memory penalty, but the complexity of an in place merge was not + * considered as worthwhile since even with the overhead, the memory cost is less than 40 bytes per + * centroid which is much less than half what the AVLTreeDigest uses. Speed tests are still not + * complete so it is uncertain whether the merge strategy is faster than the tree strategy. */ public class AgentDigest extends AbstractTDigest { @@ -85,13 +87,13 @@ public class AgentDigest extends AbstractTDigest { // this is the index of the next temporary centroid // this is a more Java-like convention than lastUsedCell uses private int tempUsed = 0; - private double[] tempWeight; - private double[] tempMean; + private final double[] tempWeight; + private final double[] tempMean; private List> tempData = null; // array used for sorting the temp centroids. This is a field // to avoid allocations during operation - private int[] order; + private final int[] order; private long dispatchTimeMillis; @@ -125,9 +127,7 @@ public AgentDigest(short compression, long dispatchTimeMillis) { this.dispatchTimeMillis = dispatchTimeMillis; } - /** - * Turns on internal data recording. - */ + /** Turns on internal data recording. */ @Override public TDigest recordAllData() { super.recordAllData(); @@ -146,6 +146,16 @@ public void add(double x, int w) { add(x, w, (List) null); } + @Override + public void add(List others) { + for (TDigest other : others) { + setMinMax(Math.min(min, other.getMin()), Math.max(max, other.getMax())); + for (Centroid centroid : other.centroids()) { + add(centroid.mean(), centroid.count(), recordAllData ? centroid.data() : null); + } + } + } + public void add(double x, int w, List history) { if (Double.isNaN(x)) { throw new IllegalArgumentException("Cannot add NaN to t-digest"); @@ -197,7 +207,13 @@ private void mergeNewValues() { int ix = order[i]; if (tempMean[ix] <= mean[j]) { wSoFar += tempWeight[ix]; - k1 = mergeCentroid(wSoFar, k1, tempWeight[ix], tempMean[ix], tempData != null ? tempData.get(ix) : null); + k1 = + mergeCentroid( + wSoFar, + k1, + tempWeight[ix], + tempMean[ix], + tempData != null ? tempData.get(ix) : null); i++; } else { wSoFar += weight[j]; @@ -209,7 +225,13 @@ private void mergeNewValues() { while (i < tempUsed) { int ix = order[i]; wSoFar += tempWeight[ix]; - k1 = mergeCentroid(wSoFar, k1, tempWeight[ix], tempMean[ix], tempData != null ? tempData.get(ix) : null); + k1 = + mergeCentroid( + wSoFar, + k1, + tempWeight[ix], + tempMean[ix], + tempData != null ? tempData.get(ix) : null); i++; } @@ -243,7 +265,8 @@ private double mergeCentroid(double wSoFar, double k1, double w, double m, List< if (k2 - k1 <= 1 || mergeWeight[lastUsedCell] == 0) { // merge into existing centroid mergeWeight[lastUsedCell] += w; - mergeMean[lastUsedCell] = mergeMean[lastUsedCell] + (m - mergeMean[lastUsedCell]) * w / mergeWeight[lastUsedCell]; + mergeMean[lastUsedCell] = + mergeMean[lastUsedCell] + (m - mergeMean[lastUsedCell]) * w / mergeWeight[lastUsedCell]; } else { // create new centroid lastUsedCell++; @@ -261,9 +284,7 @@ private double mergeCentroid(double wSoFar, double k1, double w, double m, List< return k1; } - /** - * Exposed for testing. - */ + /** Exposed for testing. */ int checkWeights() { return checkWeights(weight, totalWeight, lastUsedCell); } @@ -282,11 +303,16 @@ private int checkWeights(double[] w, double total, int last) { double dq = w[i] / total; double k2 = integratedLocation(q + dq); if (k2 - k1 > 1 && w[i] != 1) { - System.out.printf("Oversize centroid at %d, k0=%.2f, k1=%.2f, dk=%.2f, w=%.2f, q=%.4f\n", i, k1, k2, k2 - k1, w[i], q); + System.out.printf( + "Oversize centroid at %d, k0=%.2f, k1=%.2f, dk=%.2f, w=%.2f, q=%.4f\n", + i, k1, k2, k2 - k1, w[i], q); badCount++; } if (k2 - k1 > 1.5 && w[i] != 1) { - throw new IllegalStateException(String.format("Egregiously oversized centroid at %d, k0=%.2f, k1=%.2f, dk=%.2f, w=%.2f, q=%.4f\n", i, k1, k2, k2 - k1, w[i], q)); + throw new IllegalStateException( + String.format( + "Egregiously oversized centroid at %d, k0=%.2f, k1=%.2f, dk=%.2f, w=%.2f, q=%.4f\n", + i, k1, k2, k2 - k1, w[i], q)); } q += dq; k1 = k2; @@ -296,18 +322,17 @@ private int checkWeights(double[] w, double total, int last) { } /** - * Converts a quantile into a centroid scale value. The centroid scale is nominally - * the number k of the centroid that a quantile point q should belong to. Due to - * round-offs, however, we can't align things perfectly without splitting points - * and centroids. We don't want to do that, so we have to allow for offsets. - * In the end, the criterion is that any quantile range that spans a centroid - * scale range more than one should be split across more than one centroid if - * possible. This won't be possible if the quantile range refers to a single point - * or an already existing centroid. - *

- * This mapping is steep near q=0 or q=1 so each centroid there will correspond to - * less q range. Near q=0.5, the mapping is flatter so that centroids there will - * represent a larger chunk of quantiles. + * Converts a quantile into a centroid scale value. The centroid scale is nominally the number k + * of the centroid that a quantile point q should belong to. Due to round-offs, however, we can't + * align things perfectly without splitting points and centroids. We don't want to do that, so we + * have to allow for offsets. In the end, the criterion is that any quantile range that spans a + * centroid scale range more than one should be split across more than one centroid if possible. + * This won't be possible if the quantile range refers to a single point or an already existing + * centroid. + * + *

This mapping is steep near q=0 or q=1 so each centroid there will correspond to less q + * range. Near q=0.5, the mapping is flatter so that centroids there will represent a larger chunk + * of quantiles. * * @param q The quantile scale value to be mapped. * @return The centroid scale value corresponding to q. @@ -338,8 +363,8 @@ public double quantile(double q) { } /** - * Not clear to me that this is a good idea, maybe just add the temp points and existing centroids rather then merging - * first? + * Not clear to me that this is a good idea, maybe just add the temp points and existing centroids + * rather then merging first? */ @Override public Collection centroids() { @@ -367,18 +392,13 @@ public int smallByteSize() { return 0; } - - /** - * Number of centroids of this AgentDigest (does compress if necessary) - */ + /** Number of centroids of this AgentDigest (does compress if necessary) */ public int centroidCount() { mergeNewValues(); return lastUsedCell + (weight[lastUsedCell] == 0 ? 0 : 1); } - /** - * Creates a reporting Histogram from this AgentDigest (marked with the supplied duration). - */ + /** Creates a reporting Histogram from this AgentDigest (marked with the supplied duration). */ public Histogram toHistogram(int duration) { int numCentroids = centroidCount(); // NOTE: now merged as a side-effect @@ -399,36 +419,31 @@ public Histogram toHistogram(int duration) { .build(); } - /** - * Comprises of the dispatch-time (8 bytes) + compression (2 bytes) - */ + /** Comprises of the dispatch-time (8 bytes) + compression (2 bytes) */ private static final int FIXED_SIZE = 8 + 2; - /** - * Weight, mean float pair - */ + /** Weight, mean float pair */ private static final int PER_CENTROID_SIZE = 8; private int encodedSize() { return FIXED_SIZE + centroidCount() * PER_CENTROID_SIZE; } - /** - * Stateless AgentDigest codec for chronicle maps - */ - public static class AgentDigestMarshaller implements SizedReader, SizedWriter, ReadResolvable { + /** Stateless AgentDigest codec for chronicle maps */ + public static class AgentDigestMarshaller + implements SizedReader, + SizedWriter, + ReadResolvable { private static final AgentDigestMarshaller INSTANCE = new AgentDigestMarshaller(); private static final com.yammer.metrics.core.Histogram accumulatorValueSizes = Metrics.newHistogram(new MetricName("histogram", "", "accumulatorValueSize")); - - private AgentDigestMarshaller() { - } + private AgentDigestMarshaller() {} public static AgentDigestMarshaller get() { return INSTANCE; } - @NotNull + @Nonnull @Override public AgentDigest read(Bytes in, long size, @Nullable AgentDigest using) { Preconditions.checkArgument(size >= FIXED_SIZE); @@ -458,14 +473,14 @@ public AgentDigest read(Bytes in, long size, @Nullable AgentDigest using) { } @Override - public long size(@NotNull AgentDigest toWrite) { + public long size(@Nonnull AgentDigest toWrite) { long size = toWrite.encodedSize(); accumulatorValueSizes.update(size); return size; } @Override - public void write(Bytes out, long size, @NotNull AgentDigest toWrite) { + public void write(Bytes out, long size, @Nonnull AgentDigest toWrite) { // Merge in all buffered values int numCentroids = toWrite.centroidCount(); @@ -485,18 +500,19 @@ public void write(Bytes out, long size, @NotNull AgentDigest toWrite) { } } + @Nonnull @Override public AgentDigestMarshaller readResolve() { return INSTANCE; } @Override - public void readMarshallable(@NotNull WireIn wire) throws IORuntimeException { + public void readMarshallable(@Nonnull WireIn wire) throws IORuntimeException { // ignore } @Override - public void writeMarshallable(@NotNull WireOut wire) { + public void writeMarshallable(@Nonnull WireOut wire) { // ignore } } @@ -511,9 +527,7 @@ public void asSmallBytes(ByteBuffer buf) { // Ignore } - /** - * Time at which this digest should be dispatched to wavefront. - */ + /** Time at which this digest should be dispatched to wavefront. */ public long getDispatchTimeMillis() { return dispatchTimeMillis; } diff --git a/proxy/src/main/java/com/wavefront/agent/AbstractAgent.java b/proxy/src/main/java/com/wavefront/agent/AbstractAgent.java index 469ab70d8..46708deb3 100644 --- a/proxy/src/main/java/com/wavefront/agent/AbstractAgent.java +++ b/proxy/src/main/java/com/wavefront/agent/AbstractAgent.java @@ -1,113 +1,60 @@ package com.wavefront.agent; -import com.google.common.base.Charsets; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.base.Strings; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.io.Files; -import com.google.common.util.concurrent.AtomicDouble; -import com.google.common.util.concurrent.RecyclableRateLimiter; -import com.google.gson.Gson; +import static com.wavefront.agent.ProxyUtil.getOrCreateProxyId; +import static com.wavefront.common.Utils.*; +import static java.util.Collections.EMPTY_LIST; +import static org.apache.commons.lang3.StringUtils.isEmpty; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.fasterxml.jackson.databind.JsonNode; +import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.wavefront.agent.auth.TokenAuthenticator; -import com.wavefront.agent.auth.TokenAuthenticatorBuilder; -import com.wavefront.agent.auth.TokenValidationMethod; -import com.wavefront.agent.channel.DisableGZIPEncodingInterceptor; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.Maps; +import com.sun.management.UnixOperatingSystemMXBean; +import com.wavefront.agent.api.APIContainer; import com.wavefront.agent.config.LogsIngestionConfig; -import com.wavefront.agent.config.ReportableConfig; +import com.wavefront.agent.data.EntityPropertiesFactory; +import com.wavefront.agent.data.EntityPropertiesFactoryImpl; import com.wavefront.agent.logsharvesting.InteractiveLogsTester; -import com.wavefront.agent.preprocessor.AgentPreprocessorConfiguration; -import com.wavefront.agent.preprocessor.PointLineBlacklistRegexFilter; -import com.wavefront.agent.preprocessor.PointLineWhitelistRegexFilter; -import com.wavefront.api.WavefrontAPI; +import com.wavefront.agent.preprocessor.InteractivePreprocessorTester; +import com.wavefront.api.agent.preprocessor.LineBasedAllowFilter; +import com.wavefront.api.agent.preprocessor.LineBasedBlockFilter; +import com.wavefront.agent.preprocessor.ProxyPreprocessorConfigManager; +import com.wavefront.api.agent.preprocessor.PreprocessorRuleMetrics; +import com.wavefront.agent.queueing.QueueExporter; +import com.wavefront.agent.queueing.SQSQueueFactoryImpl; +import com.wavefront.agent.queueing.TaskQueueFactory; +import com.wavefront.agent.queueing.TaskQueueFactoryImpl; import com.wavefront.api.agent.AgentConfiguration; -import com.wavefront.api.agent.Constants; -import com.wavefront.common.Clock; -import com.wavefront.common.NamedThreadFactory; +import com.wavefront.api.agent.ValidationConfiguration; import com.wavefront.common.TaggedMetricName; +import com.wavefront.data.ReportableEntityType; import com.wavefront.metrics.ExpectedAgentMetric; -import com.wavefront.metrics.JsonMetricsGenerator; import com.yammer.metrics.Metrics; import com.yammer.metrics.core.Counter; -import com.yammer.metrics.core.Gauge; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.http.HttpRequest; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.config.SocketConfig; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.client.HttpClientBuilder; -import org.jboss.resteasy.client.jaxrs.ClientHttpEngine; -import org.jboss.resteasy.client.jaxrs.ResteasyClient; -import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; -import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; -import org.jboss.resteasy.client.jaxrs.engines.ApacheHttpClient4Engine; -import org.jboss.resteasy.client.jaxrs.internal.ClientInvocation; -import org.jboss.resteasy.plugins.interceptors.encoding.AcceptEncodingGZIPFilter; -import org.jboss.resteasy.plugins.interceptors.encoding.GZIPDecodingInterceptor; -import org.jboss.resteasy.plugins.interceptors.encoding.GZIPEncodingInterceptor; -import org.jboss.resteasy.plugins.providers.jackson.ResteasyJackson2Provider; -import org.jboss.resteasy.spi.ResteasyProviderFactory; - +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; import java.lang.management.ManagementFactory; -import java.lang.management.MemoryNotificationInfo; -import java.lang.management.MemoryPoolMXBean; -import java.lang.management.MemoryType; -import java.net.Authenticator; -import java.net.ConnectException; -import java.net.HttpURLConnection; -import java.net.Inet4Address; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.PasswordAuthentication; -import java.net.SocketException; -import java.net.SocketTimeoutException; -import java.net.UnknownHostException; +import java.lang.management.OperatingSystemMXBean; import java.util.ArrayList; -import java.util.Enumeration; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.ResourceBundle; -import java.util.Set; import java.util.Timer; import java.util.TimerTask; import java.util.UUID; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import javax.annotation.Nullable; -import javax.management.NotificationEmitter; -import javax.net.ssl.HttpsURLConnection; -import javax.ws.rs.ClientErrorException; -import javax.ws.rs.ForbiddenException; -import javax.ws.rs.NotAuthorizedException; -import javax.ws.rs.ProcessingException; +import javax.net.ssl.SSLException; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.ObjectUtils; /** * Agent that runs remotely on a server collecting metrics. @@ -115,1026 +62,174 @@ * @author Clement Pang (clement@wavefront.com) */ public abstract class AbstractAgent { + protected static final Logger logger = Logger.getLogger("proxy"); + final Counter activeListeners = + Metrics.newCounter(ExpectedAgentMetric.ACTIVE_LISTENERS.metricName); + /** A set of commandline parameters to hide when echoing command line arguments */ + protected final ProxyConfig proxyConfig = new ProxyConfig(); - protected static final Logger logger = Logger.getLogger("agent"); - final Counter activeListeners = Metrics.newCounter(ExpectedAgentMetric.ACTIVE_LISTENERS.metricName); - - private static final Gson GSON = new Gson(); - private static final int GRAPHITE_LISTENING_PORT = 2878; - - private static final double MAX_RETRY_BACKOFF_BASE_SECONDS = 60.0; - private static final int MAX_SPLIT_BATCH_SIZE = 50000; // same value as default pushFlushMaxPoints - - @Parameter(names = {"--help"}, help = true) - private boolean help = false; - - @Parameter(names = {"-f", "--file"}, description = - "Proxy configuration file", order = 0) - private String pushConfigFile = null; - - @Parameter(names = {"-c", "--config"}, description = - "Local configuration file to use (overrides using the server to obtain a config file)") - private String configFile = null; - - @Parameter(names = {"-p", "--prefix"}, description = - "Prefix to prepend to all push metrics before reporting.") - protected String prefix = null; - - @Parameter(names = {"-t", "--token"}, description = - "Token to auto-register proxy with an account", order = 2) - protected String token = null; - - @Parameter(names = {"--testLogs"}, description = "Run interactive session for crafting logsIngestionConfig.yaml") - private boolean testLogs = false; - - @Parameter(names = {"-l", "--loglevel", "--pushLogLevel"}, description = - "Log level for push data (NONE/SUMMARY/DETAILED); NONE is default") - protected String pushLogLevel = "NONE"; - - @Parameter(names = {"-v", "--validationlevel", "--pushValidationLevel"}, description = - "Validation level for push data (NO_VALIDATION/NUMERIC_ONLY); NUMERIC_ONLY is default") - protected String pushValidationLevel = "NUMERIC_ONLY"; - - @Parameter(names = {"-h", "--host"}, description = "Server URL", order = 1) - protected String server = "http://localhost:8080/api/"; - - @Parameter(names = {"--buffer"}, description = "File to use for buffering failed transmissions to Wavefront servers" + - ". Defaults to buffer.", order = 6) - private String bufferFile = "buffer"; - - @Parameter(names = {"--retryThreads"}, description = "Number of threads retrying failed transmissions. Defaults to " + - "the number of processors (min. 4). Buffer files are maxed out at 2G each so increasing the number of retry " + - "threads effectively governs the maximum amount of space the proxy will use to buffer points locally", order = 5) - protected Integer retryThreads = Math.min(16, Math.max(4, Runtime.getRuntime().availableProcessors())); - - @Parameter(names = {"--flushThreads"}, description = "Number of threads that flush data to the server. Defaults to" + - "the number of processors (min. 4). Setting this value too large will result in sending batches that are too " + - "small to the server and wasting connections. This setting is per listening port.", order = 4) - protected Integer flushThreads = Math.min(16, Math.max(4, Runtime.getRuntime().availableProcessors())); - - @Parameter(names = {"--purgeBuffer"}, description = "Whether to purge the retry buffer on start-up. Defaults to " + - "false.") - private boolean purgeBuffer = false; - - @Parameter(names = {"--pushFlushInterval"}, description = "Milliseconds between flushes to . Defaults to 1000 ms") - protected AtomicInteger pushFlushInterval = new AtomicInteger(1000); - protected int pushFlushIntervalInitialValue = 1000; // store initially configured value to revert to - - @Parameter(names = {"--pushFlushMaxPoints"}, description = "Maximum allowed points in a single push flush. Defaults" + - " to 40,000") - protected AtomicInteger pushFlushMaxPoints = new AtomicInteger(40000); - protected int pushFlushMaxPointsInitialValue = 40000; // store initially configured value to revert to - - @Parameter(names = {"--pushRateLimit"}, description = "Limit the outgoing point rate at the proxy. Default: " + - "do not throttle.") - protected Integer pushRateLimit = 10_000_000; - - @Parameter(names = {"--pushRateLimitMaxBurstSeconds"}, description = "Max number of burst seconds to allow " + - "when rate limiting to smooth out uneven traffic. Set to 1 when doing data backfills. Default: 10") - protected Integer pushRateLimitMaxBurstSeconds = 10; - - @Parameter(names = {"--pushMemoryBufferLimit"}, description = "Max number of points that can stay in memory buffers" + - " before spooling to disk. Defaults to 16 * pushFlushMaxPoints, minimum size: pushFlushMaxPoints. Setting this " + - " value lower than default reduces memory usage but will force the proxy to spool to disk more frequently if " + - " you have points arriving at the proxy in short bursts") - protected AtomicInteger pushMemoryBufferLimit = new AtomicInteger(16 * pushFlushMaxPoints.get()); - - @Parameter(names = {"--pushBlockedSamples"}, description = "Max number of blocked samples to print to log. Defaults" + - " to 0.") - protected Integer pushBlockedSamples = 0; - - @Parameter(names = {"--pushListenerPorts"}, description = "Comma-separated list of ports to listen on. Defaults to " + - "2878.", order = 3) - protected String pushListenerPorts = "" + GRAPHITE_LISTENING_PORT; - - @Parameter(names = {"--pushListenerMaxReceivedLength"}, description = "Maximum line length for received points in" + - " plaintext format on Wavefront/OpenTSDB/Graphite ports (Default: 4096)") - protected Integer pushListenerMaxReceivedLength = 4096; - - @Parameter(names = {"--pushListenerHttpBufferSize"}, description = "Maximum allowed request size (in bytes) for" + - " incoming HTTP requests on Wavefront/OpenTSDB/Graphite ports (Default: 16MB)") - protected Integer pushListenerHttpBufferSize = 16 * 1024 * 1024; - - @Parameter(names = {"--listenerIdleConnectionTimeout"}, description = "Close idle inbound connections after " + - " specified time in seconds. Default: 300") - protected int listenerIdleConnectionTimeout = 300; - - @Parameter(names = {"--memGuardFlushThreshold"}, description = "If heap usage exceeds this threshold (in percent), " + - "flush pending points to disk as an additional OoM protection measure. Set to 0 to disable. Default: 95") - protected int memGuardFlushThreshold = 95; - - @Parameter( - names = {"--histogramStateDirectory"}, - description = "Directory for persistent proxy state, must be writable.") - protected String histogramStateDirectory = "/var/tmp"; - - @Parameter( - names = {"--histogramAccumulatorResolveInterval"}, - description = "Interval to write-back accumulation changes from memory cache to disk in millis (only " + - "applicable when memory cache is enabled") - protected Long histogramAccumulatorResolveInterval = 100L; - - @Parameter( - names = {"--histogramAccumulatorFlushInterval"}, - description = "Interval to check for histograms to send to Wavefront in millis (Default: 1000)") - protected Long histogramAccumulatorFlushInterval = 1000L; - - @Parameter( - names = {"--histogramAccumulatorFlushMaxBatchSize"}, - description = "Max number of histograms to send to Wavefront in one flush (Default: no limit)") - protected Integer histogramAccumulatorFlushMaxBatchSize = -1; - - @Parameter( - names = {"--histogramReceiveBufferFlushInterval"}, - description = "Interval to send received points to the processing queue in millis (Default: 100)") - protected Integer histogramReceiveBufferFlushInterval = 100; - - @Parameter( - names = {"--histogramProcessingQueueScanInterval"}, - description = "Processing queue scan interval in millis (Default: 20)") - protected Integer histogramProcessingQueueScanInterval = 20; - - @Parameter( - names = {"--histogramMaxReceivedLength"}, - description = "Maximum line length for received histogram data (Default: 65536)") - protected Integer histogramMaxReceivedLength = 64 * 1024; - - @Parameter( - names = {"--histogramMinuteListenerPorts"}, - description = "Comma-separated list of ports to listen on. Defaults to none.") - protected String histogramMinuteListenerPorts = ""; - - @Parameter( - names = {"--histogramMinuteAccumulators"}, - description = "Number of accumulators per minute port") - protected Integer histogramMinuteAccumulators = Runtime.getRuntime().availableProcessors(); - - @Parameter( - names = {"--histogramMinuteFlushSecs"}, - description = "Number of seconds to keep a minute granularity accumulator open for new samples.") - protected Integer histogramMinuteFlushSecs = 70; - - @Parameter( - names = {"--histogramMinuteCompression"}, - description = "Controls allowable number of centroids per histogram. Must be in [20;1000]") - protected Short histogramMinuteCompression = 100; - - @Parameter( - names = {"--histogramMinuteAvgKeyBytes"}, - description = "Average number of bytes in a [UTF-8] encoded histogram key. Generally corresponds to a metric, " + - "source and tags concatenation.") - protected Integer histogramMinuteAvgKeyBytes = 150; - - @Parameter( - names = {"--histogramMinuteAvgDigestBytes"}, - description = "Average number of bytes in a encoded histogram.") - protected Integer histogramMinuteAvgDigestBytes = 500; - - @Parameter( - names = {"--histogramMinuteAccumulatorSize"}, - description = "Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel reporting bins") - protected Long histogramMinuteAccumulatorSize = 100000L; - - @Parameter( - names = {"--histogramMinuteMemoryCache"}, - description = "Enabling memory cache reduces I/O load with fewer time series and higher frequency data " + - "(more than 1 point per second per time series). Default: false") - protected boolean histogramMinuteMemoryCache = false; - - @Parameter( - names = {"--histogramHourListenerPorts"}, - description = "Comma-separated list of ports to listen on. Defaults to none.") - protected String histogramHourListenerPorts = ""; - - @Parameter( - names = {"--histogramHourAccumulators"}, - description = "Number of accumulators per hour port") - protected Integer histogramHourAccumulators = Runtime.getRuntime().availableProcessors(); - - @Parameter( - names = {"--histogramHourFlushSecs"}, - description = "Number of seconds to keep an hour granularity accumulator open for new samples.") - protected Integer histogramHourFlushSecs = 4200; - - @Parameter( - names = {"--histogramHourCompression"}, - description = "Controls allowable number of centroids per histogram. Must be in [20;1000]") - protected Short histogramHourCompression = 100; - - @Parameter( - names = {"--histogramHourAvgKeyBytes"}, - description = "Average number of bytes in a [UTF-8] encoded histogram key. Generally corresponds to a metric, " + - "source and tags concatenation.") - protected Integer histogramHourAvgKeyBytes = 150; - - @Parameter( - names = {"--histogramHourAvgDigestBytes"}, - description = "Average number of bytes in a encoded histogram.") - protected Integer histogramHourAvgDigestBytes = 500; - - @Parameter( - names = {"--histogramHourAccumulatorSize"}, - description = "Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel reporting bins") - protected Long histogramHourAccumulatorSize = 100000L; - - @Parameter( - names = {"--histogramHourMemoryCache"}, - description = "Enabling memory cache reduces I/O load with fewer time series and higher frequency data " + - "(more than 1 point per second per time series). Default: false") - protected boolean histogramHourMemoryCache = false; - - @Parameter( - names = {"--histogramDayListenerPorts"}, - description = "Comma-separated list of ports to listen on. Defaults to none.") - protected String histogramDayListenerPorts = ""; - - @Parameter( - names = {"--histogramDayAccumulators"}, - description = "Number of accumulators per day port") - protected Integer histogramDayAccumulators = Runtime.getRuntime().availableProcessors(); - - @Parameter( - names = {"--histogramDayFlushSecs"}, - description = "Number of seconds to keep a day granularity accumulator open for new samples.") - protected Integer histogramDayFlushSecs = 18000; - - @Parameter( - names = {"--histogramDayCompression"}, - description = "Controls allowable number of centroids per histogram. Must be in [20;1000]") - protected Short histogramDayCompression = 100; - - @Parameter( - names = {"--histogramDayAvgKeyBytes"}, - description = "Average number of bytes in a [UTF-8] encoded histogram key. Generally corresponds to a metric, " + - "source and tags concatenation.") - protected Integer histogramDayAvgKeyBytes = 150; - - @Parameter( - names = {"--histogramDayAvgHistogramDigestBytes"}, - description = "Average number of bytes in a encoded histogram.") - protected Integer histogramDayAvgDigestBytes = 500; - - @Parameter( - names = {"--histogramDayAccumulatorSize"}, - description = "Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel reporting bins") - protected Long histogramDayAccumulatorSize = 100000L; - - @Parameter( - names = {"--histogramDayMemoryCache"}, - description = "Enabling memory cache reduces I/O load with fewer time series and higher frequency data " + - "(more than 1 point per second per time series). Default: false") - protected boolean histogramDayMemoryCache = false; - - @Parameter( - names = {"--histogramDistListenerPorts"}, - description = "Comma-separated list of ports to listen on. Defaults to none.") - protected String histogramDistListenerPorts = ""; - - @Parameter( - names = {"--histogramDistAccumulators"}, - description = "Number of accumulators per distribution port") - protected Integer histogramDistAccumulators = Runtime.getRuntime().availableProcessors(); - - @Parameter( - names = {"--histogramDistFlushSecs"}, - description = "Number of seconds to keep a new distribution bin open for new samples.") - protected Integer histogramDistFlushSecs = 70; - - @Parameter( - names = {"--histogramDistCompression"}, - description = "Controls allowable number of centroids per histogram. Must be in [20;1000]") - protected Short histogramDistCompression = 100; - - @Parameter( - names = {"--histogramDistAvgKeyBytes"}, - description = "Average number of bytes in a [UTF-8] encoded histogram key. Generally corresponds to a metric, " + - "source and tags concatenation.") - protected Integer histogramDistAvgKeyBytes = 150; - - @Parameter( - names = {"--histogramDistAvgDigestBytes"}, - description = "Average number of bytes in a encoded histogram.") - protected Integer histogramDistAvgDigestBytes = 500; - - @Parameter( - names = {"--histogramDistAccumulatorSize"}, - description = "Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel reporting bins") - protected Long histogramDistAccumulatorSize = 100000L; - - @Parameter( - names = {"--histogramDistMemoryCache"}, - description = "Enabling memory cache reduces I/O load with fewer time series and higher frequency data " + - "(more than 1 point per second per time series). Default: false") - protected boolean histogramDistMemoryCache = false; - - @Parameter( - names = {"--histogramAccumulatorSize"}, hidden = true, - description = "(DEPRECATED FOR histogramMinuteAccumulatorSize/histogramHourAccumulatorSize/" + - "histogramDayAccumulatorSize/histogramDistAccumulatorSize)") - protected Long histogramAccumulatorSize = null; - - @Parameter( - names = {"--avgHistogramKeyBytes"}, hidden = true, - description = "(DEPRECATED FOR histogramMinuteAvgKeyBytes/histogramHourAvgKeyBytes/" + - "histogramDayAvgHistogramKeyBytes/histogramDistAvgKeyBytes)") - protected Integer avgHistogramKeyBytes = null; - - @Parameter( - names = {"--avgHistogramDigestBytes"}, hidden = true, - description = "(DEPRECATED FOR histogramMinuteAvgDigestBytes/histogramHourAvgDigestBytes/" + - "histogramDayAvgHistogramDigestBytes/histogramDistAvgDigestBytes)") - protected Integer avgHistogramDigestBytes = null; - - @Parameter( - names = {"--persistMessages"}, - description = "Whether histogram samples or distributions should be persisted to disk") - protected boolean persistMessages = true; - - @Parameter(names = {"--persistMessagesCompression"}, description = "Enable LZ4 compression for histogram samples " + - "persisted to disk. (Default: true)") - protected boolean persistMessagesCompression = true; - - @Parameter( - names = {"--persistAccumulator"}, - description = "Whether the accumulator should persist to disk") - protected boolean persistAccumulator = true; - - @Parameter( - names = {"--histogramCompression"}, hidden = true, - description = "(DEPRECATED FOR histogramMinuteCompression/histogramHourCompression/" + - "histogramDayCompression/histogramDistCompression)") - protected Short histogramCompression = null; - - @Parameter(names = {"--graphitePorts"}, description = "Comma-separated list of ports to listen on for graphite " + - "data. Defaults to empty list.") - protected String graphitePorts = ""; - - @Parameter(names = {"--graphiteFormat"}, description = "Comma-separated list of metric segments to extract and " + - "reassemble as the hostname (1-based).") - protected String graphiteFormat = ""; - - @Parameter(names = {"--graphiteDelimiters"}, description = "Concatenated delimiters that should be replaced in the " + - "extracted hostname with dots. Defaults to underscores (_).") - protected String graphiteDelimiters = "_"; - - @Parameter(names = {"--graphiteFieldsToRemove"}, description = "Comma-separated list of metric segments to remove (1-based)") - protected String graphiteFieldsToRemove; - - @Parameter(names = {"--jsonListenerPorts", "--httpJsonPorts"}, description = "Comma-separated list of ports to " + - "listen on for json metrics data. Binds, by default, to none.") - protected String jsonListenerPorts = ""; - - @Parameter(names = {"--dataDogJsonPorts"}, description = "Comma-separated list of ports to listen on for JSON " + - "metrics data in DataDog format. Binds, by default, to none.") - protected String dataDogJsonPorts = ""; - - @Parameter(names = {"--dataDogRequestRelayTarget"}, description = "HTTP/HTTPS target for relaying all incoming " + - "requests on dataDogJsonPorts to. Defaults to none (do not relay incoming requests)") - protected String dataDogRequestRelayTarget = null; - - @Parameter(names = {"--dataDogProcessSystemMetrics"}, description = "If true, handle system metrics as reported by " + - "DataDog collection agent. Defaults to false.") - protected boolean dataDogProcessSystemMetrics = false; - - @Parameter(names = {"--dataDogProcessServiceChecks"}, description = "If true, convert service checks to metrics. " + - "Defaults to true.") - protected boolean dataDogProcessServiceChecks = true; - - @Parameter(names = {"--writeHttpJsonListenerPorts", "--writeHttpJsonPorts"}, description = "Comma-separated list " + - "of ports to listen on for json metrics from collectd write_http json format data. Binds, by default, to none.") - protected String writeHttpJsonListenerPorts = ""; - - @Parameter(names = {"--filebeatPort"}, description = "Port on which to listen for filebeat data.") - protected Integer filebeatPort = 0; - - @Parameter(names = {"--rawLogsPort"}, description = "Port on which to listen for raw logs data.") - protected Integer rawLogsPort = 0; - - @Parameter(names = {"--rawLogsMaxReceivedLength"}, description = "Maximum line length for received raw logs (Default: 4096)") - protected Integer rawLogsMaxReceivedLength = 4096; - - @Parameter(names = {"--hostname"}, description = "Hostname for the proxy. Defaults to FQDN of machine.") - protected String hostname; - - @Parameter(names = {"--idFile"}, description = "File to read proxy id from. Defaults to ~/.dshell/id." + - "This property is ignored if ephemeral=true.") - protected String idFile = null; - - @Parameter(names = {"--graphiteWhitelistRegex"}, description = "(DEPRECATED for whitelistRegex)", hidden = true) - protected String graphiteWhitelistRegex; - - @Parameter(names = {"--graphiteBlacklistRegex"}, description = "(DEPRECATED for blacklistRegex)", hidden = true) - protected String graphiteBlacklistRegex; - - @Parameter(names = {"--whitelistRegex"}, description = "Regex pattern (java.util.regex) that graphite input lines must match to be accepted") - protected String whitelistRegex; - - @Parameter(names = {"--blacklistRegex"}, description = "Regex pattern (java.util.regex) that graphite input lines must NOT match to be accepted") - protected String blacklistRegex; - - @Parameter(names = {"--opentsdbPorts"}, description = "Comma-separated list of ports to listen on for opentsdb data. " + - "Binds, by default, to none.") - protected String opentsdbPorts = ""; - - @Parameter(names = {"--opentsdbWhitelistRegex"}, description = "Regex pattern (java.util.regex) that opentsdb input lines must match to be accepted") - protected String opentsdbWhitelistRegex; - - @Parameter(names = {"--opentsdbBlacklistRegex"}, description = "Regex pattern (java.util.regex) that opentsdb input lines must NOT match to be accepted") - protected String opentsdbBlacklistRegex; - - @Parameter(names = {"--picklePorts"}, description = "Comma-separated list of ports to listen on for pickle protocol " + - "data. Defaults to none.") - protected String picklePorts; - - @Parameter(names = {"--traceListenerPorts"}, description = "Comma-separated list of ports to listen on for trace " + - "data. Defaults to none.") - protected String traceListenerPorts; - - @Parameter(names = {"--traceJaegerListenerPorts"}, description = "Comma-separated list of ports on which to listen " + - "on for jaeger thrift formatted data over TChannel protocol. Defaults to none.") - protected String traceJaegerListenerPorts; - - @Parameter(names = {"--traceZipkinListenerPorts"}, description = "Comma-separated list of ports on which to listen " + - "on for zipkin trace data over HTTP. Defaults to none.") - protected String traceZipkinListenerPorts; - - @Parameter(names = {"--traceSamplingRate"}, description = "Value between 0.0 and 1.0. " + - "Defaults to 1.0 (allow all spans).") - protected double traceSamplingRate = 1.0d; - - @Parameter(names = {"--traceSamplingDuration"}, description = "Sample spans by duration in " + - "milliseconds. " + "Defaults to 0 (ignore duration based sampling).") - protected Integer traceSamplingDuration = 0; - - @Parameter(names = {"--pushRelayListenerPorts"}, description = "Comma-separated list of ports on which to listen " + - "on for proxy chaining data. For internal use. Defaults to none.") - protected String pushRelayListenerPorts; - - @Parameter(names = {"--splitPushWhenRateLimited"}, description = "Whether to split the push batch size when the push is rejected by Wavefront due to rate limit. Default false.") - protected boolean splitPushWhenRateLimited = false; - - @Parameter(names = {"--retryBackoffBaseSeconds"}, description = "For exponential backoff when retry threads are throttled, the base (a in a^b) in seconds. Default 2.0") - protected AtomicDouble retryBackoffBaseSeconds = new AtomicDouble(2.0); - protected double retryBackoffBaseSecondsInitialValue = 2.0d; - - @Parameter(names = {"--customSourceTags"}, description = "Comma separated list of point tag keys that should be treated as the source in Wavefront in the absence of a tag named source or host") - protected String customSourceTagsProperty = "fqdn"; - - @Parameter(names = {"--agentMetricsPointTags"}, description = "Additional point tags and their respective values to be included into internal agent's metrics (comma-separated list, ex: dc=west,env=prod)") - protected String agentMetricsPointTags = null; - - @Parameter(names = {"--ephemeral"}, description = "If true, this proxy is removed from Wavefront after 24 hours of inactivity.") - protected boolean ephemeral = false; - - @Parameter(names = {"--disableRdnsLookup"}, description = "When receiving Wavefront-formatted data without source/host specified, use remote IP address as source instead of trying to resolve the DNS name. Default false.") - protected boolean disableRdnsLookup = false; - - @Parameter(names = {"--javaNetConnection"}, description = "If true, use JRE's own http client when making connections instead of Apache HTTP Client") - protected boolean javaNetConnection = false; - - @Parameter(names = {"--gzipCompression"}, description = "If true, enables gzip compression for traffic sent to Wavefront (Default: true)") - protected boolean gzipCompression = true; - - @Parameter(names = {"--soLingerTime"}, description = "If provided, enables SO_LINGER with the specified linger time in seconds (default: SO_LINGER disabled)") - protected Integer soLingerTime = -1; - - @Parameter(names = {"--proxyHost"}, description = "Proxy host for routing traffic through a http proxy") - protected String proxyHost = null; - - @Parameter(names = {"--proxyPort"}, description = "Proxy port for routing traffic through a http proxy") - protected Integer proxyPort = 0; - - @Parameter(names = {"--proxyUser"}, description = "If proxy authentication is necessary, this is the username that will be passed along") - protected String proxyUser = null; - - @Parameter(names = {"--proxyPassword"}, description = "If proxy authentication is necessary, this is the password that will be passed along") - protected String proxyPassword = null; - - @Parameter(names = {"--httpUserAgent"}, description = "Override User-Agent in request headers") - protected String httpUserAgent = null; - - @Parameter(names = {"--httpConnectTimeout"}, description = "Connect timeout in milliseconds (default: 5000)") - protected Integer httpConnectTimeout = 5000; - - @Parameter(names = {"--httpRequestTimeout"}, description = "Request timeout in milliseconds (default: 10000)") - protected Integer httpRequestTimeout = 10000; - - @Parameter(names = {"--httpMaxConnTotal"}, description = "Max connections to keep open (default: 200)") - protected Integer httpMaxConnTotal = 200; - - @Parameter(names = {"--httpMaxConnPerRoute"}, description = "Max connections per route to keep open (default: 100)") - protected Integer httpMaxConnPerRoute = 100; - - @Parameter(names = {"--httpAutoRetries"}, description = "Number of times to retry http requests before queueing, set to 0 to disable (default: 1)") - protected Integer httpAutoRetries = 3; - - @Parameter(names = {"--preprocessorConfigFile"}, description = "Optional YAML file with additional configuration options for filtering and pre-processing points") - protected String preprocessorConfigFile = null; - - @Parameter(names = {"--dataBackfillCutoffHours"}, description = "The cut-off point for what is considered a valid timestamp for back-dated points. Default is 8760 (1 year)") - protected Integer dataBackfillCutoffHours = 8760; - - @Parameter(names = {"--dataPrefillCutoffHours"}, description = "The cut-off point for what is considered a valid timestamp for pre-dated points. Default is 24 (1 day)") - protected Integer dataPrefillCutoffHours = 24; - - @Parameter(names = {"--logsIngestionConfigFile"}, description = "Location of logs ingestions config yaml file.") - protected String logsIngestionConfigFile = null; - - @Parameter(names = {"--authMethod"}, converter = TokenValidationMethod.TokenValidationMethodConverter.class, - description = "Authenticate all incoming HTTP requests and disables TCP streams when set to a value " + - "other than NONE. Allowed values are: NONE, STATIC_TOKEN, HTTP_GET, OAUTH2. Default: NONE") - protected TokenValidationMethod authMethod = TokenValidationMethod.NONE; - - @Parameter(names = {"--authTokenIntrospectionServiceUrl"}, description = "URL for the token introspection endpoint " + - "used to validate tokens for incoming HTTP requests. Required for authMethod = OAUTH2 (endpoint must be " + - "RFC7662-compliant) and authMethod = HTTP_GET (use {{token}} placeholder in the URL to pass token to the " + - "service, endpoint must return any 2xx status for valid tokens, any other response code is a fail)") - protected String authTokenIntrospectionServiceUrl = null; - - @Parameter(names = {"--authTokenIntrospectionAuthorizationHeader"}, description = "Optional credentials for use " + - "with the token introspection endpoint.") - protected String authTokenIntrospectionAuthorizationHeader = null; - - @Parameter(names = {"--authResponseRefreshInterval"}, description = "Cache TTL (in seconds) for token validation " + - "results (re-authenticate when expired). Default: 600 seconds") - protected int authResponseRefreshInterval = 600; - - @Parameter(names = {"--authResponseMaxTtl"}, description = "Maximum allowed cache TTL (in seconds) for token " + - "validation results when token introspection service is unavailable. Default: 86400 seconds (1 day)") - protected int authResponseMaxTtl = 86400; - - @Parameter(names = {"--authStaticToken"}, description = "Static token that is considered valid for all incoming " + - "HTTP requests. Required when authMethod = STATIC_TOKEN.") - protected String authStaticToken = null; - - @Parameter(description = "") - protected List unparsed_params; - - /** - * A set of commandline parameters to hide when echoing command line arguments - */ - protected static final Set PARAMETERS_TO_HIDE = ImmutableSet.of("-t", "--token", "--proxyPassword"); - - protected QueuedAgentService agentAPI; - protected ResourceBundle props; - protected final AtomicLong bufferSpaceLeft = new AtomicLong(); - protected List customSourceTags = new ArrayList<>(); - protected final List managedTasks = new ArrayList<>(); + protected APIContainer apiContainer; protected final List managedExecutors = new ArrayList<>(); protected final List shutdownTasks = new ArrayList<>(); - protected final AgentPreprocessorConfiguration preprocessors = new AgentPreprocessorConfiguration(); - protected RecyclableRateLimiter pushRateLimiter = null; - protected TokenAuthenticator tokenAuthenticator = TokenAuthenticatorBuilder.create(). - setTokenValidationMethod(TokenValidationMethod.NONE).build(); - protected final MemoryPoolMXBean tenuredGenPool = getTenuredGenPool(); - protected JsonNode agentMetrics; - protected long agentMetricsCaptureTs; - protected AtomicBoolean hadSuccessfulCheckin = new AtomicBoolean(false); - protected boolean shuttingDown = false; - - /** - * A unique process ID value (PID, when available, or a random hexadecimal string), assigned at proxy start-up, - * to be reported with all ~proxy metrics as a "processId" point tag to prevent potential ~proxy metrics - * collisions caused by users spinning up multiple proxies with duplicate names. - */ - protected final String processId = getProcessId(); - - protected final boolean localAgent; - protected final boolean pushAgent; - - // Will be updated inside processConfiguration method and the new configuration is periodically - // loaded from the server by invoking agentAPI.checkin - protected final AtomicBoolean histogramDisabled = new AtomicBoolean(false); - protected final AtomicBoolean traceDisabled = new AtomicBoolean(false); - - /** - * Executors for support tasks. - */ - private final ScheduledExecutorService agentConfigurationExecutor = Executors.newScheduledThreadPool(2, - new NamedThreadFactory("proxy-configuration")); - private final ScheduledExecutorService queuedAgentExecutor = Executors.newScheduledThreadPool(retryThreads + 1, - new NamedThreadFactory("submitter-queue")); + protected final ProxyPreprocessorConfigManager preprocessors = + new ProxyPreprocessorConfigManager(); + protected final ValidationConfiguration validationConfiguration = new ValidationConfiguration(); + protected final Map entityPropertiesFactoryMap = + Maps.newHashMap(); + protected final AtomicBoolean shuttingDown = new AtomicBoolean(false); + protected final AtomicBoolean truncate = new AtomicBoolean(false); + protected ProxyCheckInScheduler proxyCheckinScheduler; protected UUID agentId; - private final Runnable updateConfiguration = () -> { - boolean doShutDown = false; - try { - AgentConfiguration config = fetchConfig(); - if (config != null) { - processConfiguration(config); - doShutDown = config.getShutOffAgents(); - } - } catch (Exception e) { - logger.log(Level.SEVERE, "Exception occurred during configuration update", e); - } finally { - if (doShutDown) { - logger.warning("Shutting down: Server side flag indicating proxy has to shut down."); - System.exit(1); - } - } - }; - - private final Runnable updateAgentMetrics = () -> { - @Nullable Map pointTags = new HashMap<>(); - try { - // calculate disk space available for queueing - long maxAvailableSpace = 0; - try { - File bufferDirectory = new File(bufferFile).getAbsoluteFile(); - while (bufferDirectory != null && bufferDirectory.getUsableSpace() == 0) { - bufferDirectory = bufferDirectory.getParentFile(); - } - for (int i = 0; i < retryThreads; i++) { - File buffer = new File(bufferFile + "." + i); - if (buffer.exists()) { - maxAvailableSpace += Integer.MAX_VALUE - buffer.length(); // 2GB max file size minus size used - } - } - if (bufferDirectory != null) { - // lesser of: available disk space or available buffer space - bufferSpaceLeft.set(Math.min(maxAvailableSpace, bufferDirectory.getUsableSpace())); - } - } catch (Throwable t) { - logger.warning("cannot compute remaining space in buffer file partition: " + t); - } - - if (agentMetricsPointTags != null) { - pointTags.putAll(Splitter.on(",").withKeyValueSeparator("=").split(agentMetricsPointTags)); - } - pointTags.put("processId", processId); - synchronized (agentConfigurationExecutor) { - agentMetricsCaptureTs = System.currentTimeMillis(); - agentMetrics = JsonMetricsGenerator.generateJsonMetrics(Metrics.defaultRegistry(), - true, true, true, pointTags, null); - } - } catch (Exception ex) { - logger.log(Level.SEVERE, "Could not generate proxy metrics", ex); - } - }; - - public AbstractAgent() { - this(false, false); - } + protected SslContext sslContext; + protected List tlsPorts = EMPTY_LIST; + protected boolean secureAllPorts = false; + @Deprecated public AbstractAgent(boolean localAgent, boolean pushAgent) { - this.pushAgent = pushAgent; - this.localAgent = localAgent; - this.hostname = getLocalHostName(); - Metrics.newGauge(ExpectedAgentMetric.BUFFER_BYTES_LEFT.metricName, - new Gauge() { - @Override - public Long value() { - return bufferSpaceLeft.get(); - } - } - ); + this(); } - protected abstract void startListeners(); - - protected abstract void stopListeners(); + public AbstractAgent() { + entityPropertiesFactoryMap.put( + APIContainer.CENTRAL_TENANT_NAME, new EntityPropertiesFactoryImpl(proxyConfig)); + } - private void initPreprocessors() throws IOException { - // convert blacklist/whitelist fields to filters for full backwards compatibility - // blacklistRegex and whitelistRegex are applied to pushListenerPorts, graphitePorts and picklePorts - if (whitelistRegex != null || blacklistRegex != null) { - String allPorts = StringUtils.join(new String[]{ - pushListenerPorts == null ? "" : pushListenerPorts, - graphitePorts == null ? "" : graphitePorts, - picklePorts == null ? "" : picklePorts, - traceListenerPorts == null ? "" : traceListenerPorts - }, ","); - Iterable ports = Splitter.on(",").omitEmptyStrings().trimResults().split(allPorts); - for (String strPort : ports) { - if (blacklistRegex != null) { - preprocessors.forPort(strPort).forPointLine().addFilter( - new PointLineBlacklistRegexFilter(blacklistRegex, - Metrics.newCounter(new TaggedMetricName("validationRegex", "points-rejected", "port", strPort)) - )); + private void addPreprocessorFilters(String ports, String allowList, String blockList) { + if (ports != null && (allowList != null || blockList != null)) { + for (String strPort : Splitter.on(",").omitEmptyStrings().trimResults().split(ports)) { + PreprocessorRuleMetrics ruleMetrics = + new PreprocessorRuleMetrics( + Metrics.newCounter( + new TaggedMetricName("validationRegex", "points-rejected", "port", strPort)), + Metrics.newCounter( + new TaggedMetricName("validationRegex", "cpu-nanos", "port", strPort)), + Metrics.newCounter( + new TaggedMetricName("validationRegex", "points-checked", "port", strPort))); + if (blockList != null) { + preprocessors + .getSystemPreprocessor(strPort) + .forPointLine() + .addFilter(new LineBasedBlockFilter(blockList, ruleMetrics)); } - if (whitelistRegex != null) { - preprocessors.forPort(strPort).forPointLine().addFilter( - new PointLineWhitelistRegexFilter(whitelistRegex, - Metrics.newCounter(new TaggedMetricName("validationRegex", "points-rejected", "port", strPort)) - )); + if (allowList != null) { + preprocessors + .getSystemPreprocessor(strPort) + .forPointLine() + .addFilter(new LineBasedAllowFilter(allowList, ruleMetrics)); } } } + } - // opentsdbBlacklistRegex and opentsdbWhitelistRegex are applied to opentsdbPorts only - if (opentsdbPorts != null && (opentsdbWhitelistRegex != null || opentsdbBlacklistRegex != null)) { - Iterable ports = Splitter.on(",").omitEmptyStrings().trimResults().split(opentsdbPorts); - for (String strPort : ports) { - if (opentsdbBlacklistRegex != null) { - preprocessors.forPort(strPort).forPointLine().addFilter( - new PointLineBlacklistRegexFilter(opentsdbBlacklistRegex, - Metrics.newCounter(new TaggedMetricName("validationRegex", "points-rejected", "port", strPort)) - )); - } - if (opentsdbWhitelistRegex != null) { - preprocessors.forPort(strPort).forPointLine().addFilter( - new PointLineWhitelistRegexFilter(opentsdbWhitelistRegex, - Metrics.newCounter(new TaggedMetricName("validationRegex", "points-rejected", "port", strPort)) - )); - } - } + @VisibleForTesting + void initSslContext() throws SSLException { + if (!isEmpty(proxyConfig.getPrivateCertPath()) && !isEmpty(proxyConfig.getPrivateKeyPath())) { + sslContext = + SslContextBuilder.forServer( + new File(proxyConfig.getPrivateCertPath()), + new File(proxyConfig.getPrivateKeyPath())) + .build(); + } + if (!isEmpty(proxyConfig.getTlsPorts()) && sslContext == null) { + Preconditions.checkArgument( + sslContext != null, "Missing TLS certificate/private key configuration."); + } + if (StringUtils.equals(proxyConfig.getTlsPorts(), "*")) { + secureAllPorts = true; + } else { + tlsPorts = csvToList(proxyConfig.getTlsPorts()); } + } - if (preprocessorConfigFile != null) { - FileInputStream stream = new FileInputStream(preprocessorConfigFile); - preprocessors.loadFromStream(stream); - logger.info("Preprocessor configuration loaded from " + preprocessorConfigFile); - } + private void initPreprocessors() { + String configFileName = proxyConfig.getPreprocessorConfigFile(); + if (ProxyCheckInScheduler.isRulesSetInFE.get()) { + logger.info("Preprocessor configuration was set in FE. Skipping reading from file " + configFileName); + } else if (configFileName != null) { + try { + preprocessors.loadFile(configFileName); + preprocessors.setUpConfigFileMonitoring(configFileName, 5000); // check every 5s + } catch (FileNotFoundException ex) { + throw new RuntimeException( + "Unable to load preprocessor rules - file does not exist: " + configFileName); + } + logger.log(Level.INFO, () -> String.format("Preprocessor configuration loaded from %s", + (ProxyCheckInScheduler.isRulesSetInFE.get()) ? "FE rule" : configFileName)); + } + + // convert block/allow list fields to filters for full backwards compatibility. + // "block" and "allow" regexes are applied to pushListenerPorts, graphitePorts and + // picklePorts + String allPorts = + StringUtils.join( + new String[] { + ObjectUtils.firstNonNull(proxyConfig.getPushListenerPorts(), ""), + ObjectUtils.firstNonNull(proxyConfig.getGraphitePorts(), ""), + ObjectUtils.firstNonNull(proxyConfig.getPicklePorts(), ""), + ObjectUtils.firstNonNull(proxyConfig.getTraceListenerPorts(), "") + }, + ","); + addPreprocessorFilters(allPorts, proxyConfig.getAllowRegex(), proxyConfig.getBlockRegex()); + // opentsdb block/allow lists are applied to opentsdbPorts only + addPreprocessorFilters( + proxyConfig.getOpentsdbPorts(), + proxyConfig.getOpentsdbAllowRegex(), + proxyConfig.getOpentsdbBlockRegex()); } // Returns null on any exception, and logs the exception. protected LogsIngestionConfig loadLogsIngestionConfig() { try { - if (logsIngestionConfigFile == null) { + if (proxyConfig.getLogsIngestionConfigFile() == null) { return null; } ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); - return objectMapper.readValue(new File(logsIngestionConfigFile), LogsIngestionConfig.class); + return objectMapper.readValue( + new File(proxyConfig.getLogsIngestionConfigFile()), LogsIngestionConfig.class); + } catch (UnrecognizedPropertyException e) { + logger.severe("Unable to load logs ingestion config: " + e.getMessage()); } catch (Exception e) { logger.log(Level.SEVERE, "Could not load logs ingestion config", e); - return null; } + return null; } - private void loadListenerConfigurationFile() throws IOException { - ReportableConfig config; - // If they've specified a push configuration file, override the command line values - try { - if (pushConfigFile != null) { - config = new ReportableConfig(pushConfigFile); - } else { - config = new ReportableConfig(); // dummy config - } - prefix = Strings.emptyToNull(config.getString("prefix", prefix)); - pushLogLevel = config.getString("pushLogLevel", pushLogLevel); - pushValidationLevel = config.getString("pushValidationLevel", pushValidationLevel); - token = ObjectUtils.firstNonNull(config.getRawProperty("token", token), "undefined").trim(); // don't track - server = config.getRawProperty("server", server).trim(); // don't track - hostname = config.getString("hostname", hostname); - idFile = config.getString("idFile", idFile); - pushRateLimit = config.getNumber("pushRateLimit", pushRateLimit).intValue(); - pushRateLimitMaxBurstSeconds = config.getNumber("pushRateLimitMaxBurstSeconds", pushRateLimitMaxBurstSeconds). - intValue(); - pushBlockedSamples = config.getNumber("pushBlockedSamples", pushBlockedSamples).intValue(); - pushListenerPorts = config.getString("pushListenerPorts", pushListenerPorts); - pushListenerMaxReceivedLength = config.getNumber("pushListenerMaxReceivedLength", - pushListenerMaxReceivedLength).intValue(); - pushListenerHttpBufferSize = config.getNumber("pushListenerHttpBufferSize", - pushListenerHttpBufferSize).intValue(); - listenerIdleConnectionTimeout = config.getNumber("listenerIdleConnectionTimeout", - listenerIdleConnectionTimeout).intValue(); - memGuardFlushThreshold = config.getNumber("memGuardFlushThreshold", memGuardFlushThreshold).intValue(); + private void postProcessConfig() { + // disable useless info messages when httpClient has to retry a request due to a stale + // connection. the alternative is to always validate connections before reuse, but since + // it happens fairly infrequently, and connection re-validation performance penalty is + // incurred every time, suppressing that message seems to be a more reasonable approach. + // org.apache.log4j.Logger.getLogger("org.apache.http.impl.execchain.RetryExec"). + // setLevel(org.apache.log4j.Level.WARN); + // Logger.getLogger("org.apache.http.impl.execchain.RetryExec").setLevel(Level.WARNING); - // Histogram: global settings - histogramStateDirectory = config.getString("histogramStateDirectory", histogramStateDirectory); - histogramAccumulatorResolveInterval = config.getNumber("histogramAccumulatorResolveInterval", - histogramAccumulatorResolveInterval).longValue(); - histogramAccumulatorFlushInterval = config.getNumber("histogramAccumulatorFlushInterval", - histogramAccumulatorFlushInterval).longValue(); - histogramAccumulatorFlushMaxBatchSize = config.getNumber("histogramAccumulatorFlushMaxBatchSize", - histogramAccumulatorFlushMaxBatchSize).intValue(); - histogramReceiveBufferFlushInterval = config.getNumber("histogramReceiveBufferFlushInterval", - histogramReceiveBufferFlushInterval).intValue(); - histogramProcessingQueueScanInterval = config.getNumber("histogramProcessingQueueScanInterval", - histogramProcessingQueueScanInterval).intValue(); - histogramMaxReceivedLength = config.getNumber("histogramMaxReceivedLength", - histogramMaxReceivedLength).intValue(); - persistAccumulator = config.getBoolean("persistAccumulator", persistAccumulator); - persistMessages = config.getBoolean("persistMessages", persistMessages); - persistMessagesCompression = config.getBoolean("persistMessagesCompression", - persistMessagesCompression); + if (StringUtils.isBlank(proxyConfig.getHostname())) { + throw new IllegalArgumentException( + "hostname cannot be blank! Please correct your configuration settings."); + } - // Histogram: deprecated settings - fall back for backwards compatibility - if (config.isDefined("avgHistogramKeyBytes")) { - histogramMinuteAvgKeyBytes = histogramHourAvgKeyBytes = histogramDayAvgKeyBytes = - histogramDistAvgKeyBytes = config.getNumber("avgHistogramKeyBytes", avgHistogramKeyBytes).intValue(); - } - if (config.isDefined("avgHistogramDigestBytes")) { - histogramMinuteAvgDigestBytes = histogramHourAvgDigestBytes = histogramDayAvgDigestBytes = - histogramDistAvgDigestBytes = config.getNumber("avgHistogramDigestBytes", avgHistogramDigestBytes). - intValue(); - } - if (config.isDefined("histogramAccumulatorSize")) { - histogramMinuteAccumulatorSize = histogramHourAccumulatorSize = histogramDayAccumulatorSize = - histogramDistAccumulatorSize = config.getNumber("histogramAccumulatorSize", - histogramAccumulatorSize).longValue(); + if (proxyConfig.isSqsQueueBuffer()) { + if (StringUtils.isBlank(proxyConfig.getSqsQueueIdentifier())) { + throw new IllegalArgumentException( + "sqsQueueIdentifier cannot be blank! Please correct " + "your configuration settings."); } - if (config.isDefined("histogramCompression")) { - histogramMinuteCompression = histogramHourCompression = histogramDayCompression = - histogramDistCompression = config.getNumber("histogramCompression", null, 20, 1000).shortValue(); + if (!SQSQueueFactoryImpl.isValidSQSTemplate(proxyConfig.getSqsQueueNameTemplate())) { + throw new IllegalArgumentException( + "sqsQueueNameTemplate is invalid! Must contain " + + "{{id}} {{entity}} and {{port}} replacements."); } - - // Histogram: minute accumulator settings - histogramMinuteListenerPorts = config.getString("histogramMinuteListenerPorts", histogramMinuteListenerPorts); - histogramMinuteAccumulators = config.getNumber("histogramMinuteAccumulators", histogramMinuteAccumulators). - intValue(); - histogramMinuteFlushSecs = config.getNumber("histogramMinuteFlushSecs", histogramMinuteFlushSecs).intValue(); - histogramMinuteCompression = config.getNumber("histogramMinuteCompression", - histogramMinuteCompression, 20, 1000).shortValue(); - histogramMinuteAvgKeyBytes = config.getNumber("histogramMinuteAvgKeyBytes", histogramMinuteAvgKeyBytes). - intValue(); - histogramMinuteAvgDigestBytes = 32 + histogramMinuteCompression * 7; - histogramMinuteAvgDigestBytes = config.getNumber("histogramMinuteAvgDigestBytes", - histogramMinuteAvgDigestBytes).intValue(); - histogramMinuteAccumulatorSize = config.getNumber("histogramMinuteAccumulatorSize", - histogramMinuteAccumulatorSize).longValue(); - histogramMinuteMemoryCache = config.getBoolean("histogramMinuteMemoryCache", histogramMinuteMemoryCache); - - // Histogram: hour accumulator settings - histogramHourListenerPorts = config.getString("histogramHourListenerPorts", histogramHourListenerPorts); - histogramHourAccumulators = config.getNumber("histogramHourAccumulators", histogramHourAccumulators).intValue(); - histogramHourFlushSecs = config.getNumber("histogramHourFlushSecs", histogramHourFlushSecs).intValue(); - histogramHourCompression = config.getNumber("histogramHourCompression", - histogramHourCompression, 20, 1000).shortValue(); - histogramHourAvgKeyBytes = config.getNumber("histogramHourAvgKeyBytes", histogramHourAvgKeyBytes).intValue(); - histogramHourAvgDigestBytes = 32 + histogramHourCompression * 7; - histogramHourAvgDigestBytes = config.getNumber("histogramHourAvgDigestBytes", histogramHourAvgDigestBytes). - intValue(); - histogramHourAccumulatorSize = config.getNumber("histogramHourAccumulatorSize", histogramHourAccumulatorSize). - longValue(); - histogramHourMemoryCache = config.getBoolean("histogramHourMemoryCache", histogramHourMemoryCache); - - // Histogram: day accumulator settings - histogramDayListenerPorts = config.getString("histogramDayListenerPorts", histogramDayListenerPorts); - histogramDayAccumulators = config.getNumber("histogramDayAccumulators", histogramDayAccumulators).intValue(); - histogramDayFlushSecs = config.getNumber("histogramDayFlushSecs", histogramDayFlushSecs).intValue(); - histogramDayCompression = config.getNumber("histogramDayCompression", - histogramDayCompression, 20, 1000).shortValue(); - histogramDayAvgKeyBytes = config.getNumber("histogramDayAvgKeyBytes", histogramDayAvgKeyBytes).intValue(); - histogramDayAvgDigestBytes = 32 + histogramDayCompression * 7; - histogramDayAvgDigestBytes = config.getNumber("histogramDayAvgDigestBytes", histogramDayAvgDigestBytes). - intValue(); - histogramDayAccumulatorSize = config.getNumber("histogramDayAccumulatorSize", histogramDayAccumulatorSize). - longValue(); - histogramDayMemoryCache = config.getBoolean("histogramDayMemoryCache", histogramDayMemoryCache); - - // Histogram: dist accumulator settings - histogramDistListenerPorts = config.getString("histogramDistListenerPorts", histogramDistListenerPorts); - histogramDistAccumulators = config.getNumber("histogramDistAccumulators", histogramDistAccumulators).intValue(); - histogramDistFlushSecs = config.getNumber("histogramDistFlushSecs", histogramDistFlushSecs).intValue(); - histogramDistCompression = config.getNumber("histogramDistCompression", - histogramDistCompression, 20, 1000).shortValue(); - histogramDistAvgKeyBytes = config.getNumber("histogramDistAvgKeyBytes", histogramDistAvgKeyBytes).intValue(); - histogramDistAvgDigestBytes = 32 + histogramDistCompression * 7; - histogramDistAvgDigestBytes = config.getNumber("histogramDistAvgDigestBytes", histogramDistAvgDigestBytes). - intValue(); - histogramDistAccumulatorSize = config.getNumber("histogramDistAccumulatorSize", histogramDistAccumulatorSize). - longValue(); - histogramDistMemoryCache = config.getBoolean("histogramDistMemoryCache", histogramDistMemoryCache); - - retryThreads = config.getNumber("retryThreads", retryThreads).intValue(); - flushThreads = config.getNumber("flushThreads", flushThreads).intValue(); - jsonListenerPorts = config.getString("jsonListenerPorts", jsonListenerPorts); - writeHttpJsonListenerPorts = config.getString("writeHttpJsonListenerPorts", writeHttpJsonListenerPorts); - dataDogJsonPorts = config.getString("dataDogJsonPorts", dataDogJsonPorts); - dataDogRequestRelayTarget = config.getString("dataDogRequestRelayTarget", dataDogRequestRelayTarget); - dataDogProcessSystemMetrics = config.getBoolean("dataDogProcessSystemMetrics", dataDogProcessSystemMetrics); - dataDogProcessServiceChecks = config.getBoolean("dataDogProcessServiceChecks", dataDogProcessServiceChecks); - graphitePorts = config.getString("graphitePorts", graphitePorts); - graphiteFormat = config.getString("graphiteFormat", graphiteFormat); - graphiteFieldsToRemove = config.getString("graphiteFieldsToRemove", graphiteFieldsToRemove); - graphiteDelimiters = config.getString("graphiteDelimiters", graphiteDelimiters); - graphiteWhitelistRegex = config.getString("graphiteWhitelistRegex", graphiteWhitelistRegex); - graphiteBlacklistRegex = config.getString("graphiteBlacklistRegex", graphiteBlacklistRegex); - whitelistRegex = config.getString("whitelistRegex", whitelistRegex); - blacklistRegex = config.getString("blacklistRegex", blacklistRegex); - opentsdbPorts = config.getString("opentsdbPorts", opentsdbPorts); - opentsdbWhitelistRegex = config.getString("opentsdbWhitelistRegex", opentsdbWhitelistRegex); - opentsdbBlacklistRegex = config.getString("opentsdbBlacklistRegex", opentsdbBlacklistRegex); - proxyHost = config.getString("proxyHost", proxyHost); - proxyPort = config.getNumber("proxyPort", proxyPort).intValue(); - proxyPassword = config.getString("proxyPassword", proxyPassword, s -> ""); - proxyUser = config.getString("proxyUser", proxyUser); - httpUserAgent = config.getString("httpUserAgent", httpUserAgent); - httpConnectTimeout = config.getNumber("httpConnectTimeout", httpConnectTimeout).intValue(); - httpRequestTimeout = config.getNumber("httpRequestTimeout", httpRequestTimeout).intValue(); - httpMaxConnTotal = Math.min(200, config.getNumber("httpMaxConnTotal", httpMaxConnTotal).intValue()); - httpMaxConnPerRoute = Math.min(100, config.getNumber("httpMaxConnPerRoute", httpMaxConnPerRoute).intValue()); - httpAutoRetries = config.getNumber("httpAutoRetries", httpAutoRetries).intValue(); - javaNetConnection = config.getBoolean("javaNetConnection", javaNetConnection); - gzipCompression = config.getBoolean("gzipCompression", gzipCompression); - soLingerTime = config.getNumber("soLingerTime", soLingerTime).intValue(); - splitPushWhenRateLimited = config.getBoolean("splitPushWhenRateLimited", splitPushWhenRateLimited); - customSourceTagsProperty = config.getString("customSourceTags", customSourceTagsProperty); - agentMetricsPointTags = config.getString("agentMetricsPointTags", agentMetricsPointTags); - ephemeral = config.getBoolean("ephemeral", ephemeral); - disableRdnsLookup = config.getBoolean("disableRdnsLookup", disableRdnsLookup); - picklePorts = config.getString("picklePorts", picklePorts); - traceListenerPorts = config.getString("traceListenerPorts", traceListenerPorts); - traceJaegerListenerPorts = config.getString("traceJaegerListenerPorts", traceJaegerListenerPorts); - traceSamplingRate = Double.parseDouble(config.getRawProperty("traceSamplingRate", - String.valueOf(traceSamplingRate)).trim()); - traceSamplingDuration = config.getNumber("traceSamplingDuration", traceSamplingDuration).intValue(); - pushRelayListenerPorts = config.getString("pushRelayListenerPorts", pushRelayListenerPorts); - bufferFile = config.getString("buffer", bufferFile); - preprocessorConfigFile = config.getString("preprocessorConfigFile", preprocessorConfigFile); - dataBackfillCutoffHours = config.getNumber("dataBackfillCutoffHours", dataBackfillCutoffHours).intValue(); - dataPrefillCutoffHours = config.getNumber("dataPrefillCutoffHours", dataPrefillCutoffHours).intValue(); - filebeatPort = config.getNumber("filebeatPort", filebeatPort).intValue(); - rawLogsPort = config.getNumber("rawLogsPort", rawLogsPort).intValue(); - rawLogsMaxReceivedLength = config.getNumber("rawLogsMaxReceivedLength", rawLogsMaxReceivedLength).intValue(); - logsIngestionConfigFile = config.getString("logsIngestionConfigFile", logsIngestionConfigFile); - - authMethod = TokenValidationMethod.fromString(config.getString("authMethod", authMethod.toString())); - authTokenIntrospectionServiceUrl = config.getString("authTokenIntrospectionServiceUrl", - authTokenIntrospectionServiceUrl); - authTokenIntrospectionAuthorizationHeader = config.getString("authTokenIntrospectionAuthorizationHeader", - authTokenIntrospectionAuthorizationHeader); - authResponseRefreshInterval = config.getNumber("authResponseRefreshInterval", authResponseRefreshInterval). - intValue(); - authResponseMaxTtl = config.getNumber("authResponseMaxTtl", authResponseMaxTtl).intValue(); - authStaticToken = config.getString("authStaticToken", authStaticToken); - - // track mutable settings - pushFlushIntervalInitialValue = Integer.parseInt(config.getRawProperty("pushFlushInterval", - String.valueOf(pushFlushInterval.get())).trim()); - pushFlushInterval.set(pushFlushIntervalInitialValue); - config.reportSettingAsGauge(pushFlushInterval, "pushFlushInterval"); - - pushFlushMaxPointsInitialValue = Integer.parseInt(config.getRawProperty("pushFlushMaxPoints", - String.valueOf(pushFlushMaxPoints.get())).trim()); - // clamp values for pushFlushMaxPoints between 1..50000 - pushFlushMaxPointsInitialValue = Math.max(Math.min(pushFlushMaxPointsInitialValue, MAX_SPLIT_BATCH_SIZE), 1); - pushFlushMaxPoints.set(pushFlushMaxPointsInitialValue); - config.reportSettingAsGauge(pushFlushMaxPoints, "pushFlushMaxPoints"); - - retryBackoffBaseSecondsInitialValue = Double.parseDouble(config.getRawProperty("retryBackoffBaseSeconds", - String.valueOf(retryBackoffBaseSeconds.get())).trim()); - retryBackoffBaseSeconds.set(retryBackoffBaseSecondsInitialValue); - config.reportSettingAsGauge(retryBackoffBaseSeconds, "retryBackoffBaseSeconds"); - - /* - default value for pushMemoryBufferLimit is 16 * pushFlushMaxPoints, but no more than 25% of available heap - memory. 25% is chosen heuristically as a safe number for scenarios with limited system resources (4 CPU cores - or less, heap size less than 4GB) to prevent OOM. this is a conservative estimate, budgeting 200 characters - (400 bytes) per per point line. Also, it shouldn't be less than 1 batch size (pushFlushMaxPoints). - */ - int listeningPorts = Iterables.size(Splitter.on(",").omitEmptyStrings().trimResults().split(pushListenerPorts)); - long calculatedMemoryBufferLimit = Math.max(Math.min(16 * pushFlushMaxPoints.get(), - Runtime.getRuntime().maxMemory() / (listeningPorts > 0 ? listeningPorts : 1) / 4 / flushThreads / 400), - pushFlushMaxPoints.get()); - logger.fine("Calculated pushMemoryBufferLimit: " + calculatedMemoryBufferLimit); - pushMemoryBufferLimit.set(Integer.parseInt( - config.getRawProperty("pushMemoryBufferLimit", String.valueOf(pushMemoryBufferLimit.get())).trim())); - config.reportSettingAsGauge(pushMemoryBufferLimit, "pushMemoryBufferLimit"); - logger.fine("Configured pushMemoryBufferLimit: " + pushMemoryBufferLimit); - - logger.warning("Loaded configuration file " + pushConfigFile); - } catch (Throwable exception) { - logger.severe("Could not load configuration file " + pushConfigFile); - throw exception; - } - - // Compatibility with deprecated fields - if (whitelistRegex == null && graphiteWhitelistRegex != null) { - whitelistRegex = graphiteWhitelistRegex; - } - - if (blacklistRegex == null && graphiteBlacklistRegex != null) { - blacklistRegex = graphiteBlacklistRegex; - } - - initPreprocessors(); - - if (!persistMessages) { - persistMessagesCompression = false; - } - if (pushRateLimit > 0) { - pushRateLimiter = RecyclableRateLimiter.create(pushRateLimit, pushRateLimitMaxBurstSeconds); } + } - pushMemoryBufferLimit.set(Math.max(pushMemoryBufferLimit.get(), pushFlushMaxPoints.get())); - - PostPushDataTimedTask.setPointsPerBatch(pushFlushMaxPoints); - PostPushDataTimedTask.setMemoryBufferLimit(pushMemoryBufferLimit); - QueuedAgentService.setSplitBatchSize(pushFlushMaxPoints); - - retryBackoffBaseSeconds.set(Math.max( - Math.min(retryBackoffBaseSeconds.get(), MAX_RETRY_BACKOFF_BASE_SECONDS), - 1.0)); - QueuedAgentService.setRetryBackoffBaseSeconds(retryBackoffBaseSeconds); - - // for backwards compatibility - if pushLogLevel is defined in the config file, change log level programmatically - Level level = null; - switch (pushLogLevel) { - case "NONE": - level = Level.WARNING; - break; - case "SUMMARY": - level = Level.INFO; - break; - case "DETAILED": - level = Level.FINE; - break; - } - if (level != null) { - Logger.getLogger("agent").setLevel(level); - Logger.getLogger(PostPushDataTimedTask.class.getCanonicalName()).setLevel(level); - Logger.getLogger(QueuedAgentService.class.getCanonicalName()).setLevel(level); + @VisibleForTesting + void parseArguments(String[] args) { + try { + if (!proxyConfig.parseArguments(args, this.getClass().getCanonicalName())) { + System.exit(0); + } + } catch (ParameterException e) { + logger.severe("Parameter exception: " + e.getMessage()); + System.exit(1); } } @@ -1143,520 +238,195 @@ private void loadListenerConfigurationFile() throws IOException { * * @param args Command-line parameters passed on to JCommander to configure the daemon. */ - public void start(String[] args) throws IOException { - try { - // read build information and print version. - props = ResourceBundle.getBundle("build"); - logger.info("Starting proxy version " + props.getString("build.version")); + public void start(String[] args) { + String versionStr = + "Wavefront Proxy version " + + getBuildVersion() + + " (pkg:" + + getPackage() + + ")" + + ", runtime: " + + getJavaVersion(); + logger.info(versionStr); + + OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean(); + if (os instanceof UnixOperatingSystemMXBean) { + UnixOperatingSystemMXBean os1 = (UnixOperatingSystemMXBean) os; + logger.info("OS Max File Descriptors: " + os1.getMaxFileDescriptorCount()); + } - logger.info("Arguments: " + IntStream.range(0, args.length). - mapToObj(i -> (i > 0 && PARAMETERS_TO_HIDE.contains(args[i - 1])) ? "" : args[i]). - collect(Collectors.joining(", "))); - JCommander jCommander = JCommander.newBuilder(). - programName(this.getClass().getCanonicalName()). - addObject(this). - allowParameterOverwriting(true). - build(); - jCommander.parse(args); - if (help) { - jCommander.usage(); - System.exit(0); - } - if (unparsed_params != null) { - logger.info("Unparsed arguments: " + Joiner.on(", ").join(unparsed_params)); - } + try { /* ------------------------------------------------------------------------------------ * Configuration Setup. * ------------------------------------------------------------------------------------ */ - // 1. Load the listener configurations. - loadListenerConfigurationFile(); - loadLogsIngestionConfig(); - configureTokenAuthenticator(); - - managedExecutors.add(agentConfigurationExecutor); - - // Conditionally enter an interactive debugging session for logsIngestionConfig.yaml - if (testLogs) { - InteractiveLogsTester interactiveLogsTester = new InteractiveLogsTester(this::loadLogsIngestionConfig, prefix); - logger.info("Reading line-by-line sample log messages from STDIN"); - while (interactiveLogsTester.interactiveTest()) { + // Parse commandline arguments and load configuration file + parseArguments(args); + postProcessConfig(); + initSslContext(); + initPreprocessors(); + + if (proxyConfig.isTestLogs() + || proxyConfig.getTestPreprocessorForPort() != null + || proxyConfig.getTestSpanPreprocessorForPort() != null) { + InteractiveTester interactiveTester; + if (proxyConfig.isTestLogs()) { + logger.info("Reading line-by-line sample log messages from STDIN"); + interactiveTester = + new InteractiveLogsTester(this::loadLogsIngestionConfig, proxyConfig.getPrefix()); + } else if (proxyConfig.getTestPreprocessorForPort() != null) { + logger.info("Reading line-by-line points from STDIN"); + interactiveTester = + new InteractivePreprocessorTester( + preprocessors.get(proxyConfig.getTestPreprocessorForPort()), + ReportableEntityType.POINT, + proxyConfig.getTestPreprocessorForPort(), + proxyConfig.getCustomSourceTags()); + } else if (proxyConfig.getTestSpanPreprocessorForPort() != null) { + logger.info("Reading line-by-line spans from STDIN"); + interactiveTester = + new InteractivePreprocessorTester( + preprocessors.get(String.valueOf(proxyConfig.getTestPreprocessorForPort())), + ReportableEntityType.TRACE, + proxyConfig.getTestPreprocessorForPort(), + proxyConfig.getCustomSourceTags()); + } else { + throw new IllegalStateException(); + } + //noinspection StatementWithEmptyBody + while (interactiveTester.interactiveTest()) { // empty } System.exit(0); } - // 2. Read or create the unique Id for the daemon running on this machine. - if (ephemeral) { - agentId = UUID.randomUUID(); // don't need to store one - logger.info("Ephemeral proxy id created: " + agentId); - } else { - readOrCreateDaemonId(); - } - - if (proxyHost != null) { - System.setProperty("http.proxyHost", proxyHost); - System.setProperty("https.proxyHost", proxyHost); - System.setProperty("http.proxyPort", String.valueOf(proxyPort)); - System.setProperty("https.proxyPort", String.valueOf(proxyPort)); - } - if (proxyUser != null && proxyPassword != null) { - Authenticator.setDefault( - new Authenticator() { - @Override - public PasswordAuthentication getPasswordAuthentication() { - if (getRequestorType() == RequestorType.PROXY) { - return new PasswordAuthentication(proxyUser, proxyPassword.toCharArray()); - } else { - return null; - } - } - } - ); - } - - // create List of custom tags from the configuration string - String[] tags = customSourceTagsProperty.split(","); - for (String tag : tags) { - tag = tag.trim(); - if (!customSourceTags.contains(tag)) { - customSourceTags.add(tag); - } else { - logger.warning("Custom source tag: " + tag + " was repeated. Check the customSourceTags property in " + - "wavefront.conf"); - } - } - - // 3. Setup proxies. - WavefrontAPI service = createAgentService(); - try { - setupQueueing(service); - } catch (IOException e) { - logger.log(Level.SEVERE, "Cannot setup local file for queueing due to IO error", e); - throw e; + // If we are exporting data from the queue, run export and exit + if (proxyConfig.getExportQueueOutputFile() != null + && proxyConfig.getExportQueuePorts() != null) { + TaskQueueFactory tqFactory = + new TaskQueueFactoryImpl( + proxyConfig.getBufferFile(), false, false, proxyConfig.getBufferShardSize()); + EntityPropertiesFactory epFactory = new EntityPropertiesFactoryImpl(proxyConfig); + QueueExporter queueExporter = + new QueueExporter( + proxyConfig.getBufferFile(), + proxyConfig.getExportQueuePorts(), + proxyConfig.getExportQueueOutputFile(), + proxyConfig.isExportQueueRetainData(), + tqFactory, + epFactory); + logger.info("Starting queue export for ports: " + proxyConfig.getExportQueuePorts()); + queueExporter.export(); + logger.info("Done"); + System.exit(0); } - // 4. Start the (push) listening endpoints + // 2. Read or create the unique Id for the daemon running on this machine. + agentId = getOrCreateProxyId(proxyConfig); + apiContainer = new APIContainer(proxyConfig, proxyConfig.isUseNoopSender()); + TokenManager.start(apiContainer); + // config the entityPropertiesFactoryMap + for (String tenantName : TokenManager.getMulticastingTenantList().keySet()) { + entityPropertiesFactoryMap.put(tenantName, new EntityPropertiesFactoryImpl(proxyConfig)); + } + // Perform initial proxy check-in and schedule regular check-ins (once a minute) + proxyCheckinScheduler = + new ProxyCheckInScheduler( + agentId, + proxyConfig, + apiContainer, + this::processConfiguration, + () -> System.exit(1), + this::truncateBacklog); + proxyCheckinScheduler.scheduleCheckins(); + + // Start the listening endpoints startListeners(); - // set up OoM memory guard - if (memGuardFlushThreshold > 0) { - setupMemoryGuard((float)memGuardFlushThreshold / 100); - } - - new Timer().schedule( + Timer startupTimer = new Timer("Timer-startup"); + shutdownTasks.add(startupTimer::cancel); + startupTimer.schedule( new TimerTask() { @Override public void run() { - try { - // exit if no active listeners - if (activeListeners.count() == 0) { - logger.severe("**** All listener threads failed to start - there is already a running instance " + - "listening on configured ports, or no listening ports configured!"); - logger.severe("Aborting start-up"); - System.exit(1); - } - - // 5. Poll or read the configuration file to use. - AgentConfiguration config; - if (configFile != null) { - logger.info("Loading configuration file from: " + configFile); - try { - config = GSON.fromJson(new FileReader(configFile), - AgentConfiguration.class); - } catch (FileNotFoundException e) { - throw new RuntimeException("Cannot read config file: " + configFile); - } - try { - config.validate(localAgent); - } catch (RuntimeException ex) { - logger.log(Level.SEVERE, "cannot parse config file", ex); - throw new RuntimeException("cannot parse config file", ex); - } - agentId = null; - } else { - updateAgentMetrics.run(); - config = fetchConfig(); - logger.info("scheduling regular configuration polls"); - agentConfigurationExecutor.scheduleAtFixedRate(updateAgentMetrics, 10, 60, TimeUnit.SECONDS); - agentConfigurationExecutor.scheduleWithFixedDelay(updateConfiguration, 0, 1, TimeUnit.SECONDS); - } - // 6. Setup work units and targets based on the configuration. - if (config != null) { - logger.info("initial configuration is available, setting up proxy"); - processConfiguration(config); - } - - Runtime.getRuntime().addShutdownHook(new Thread("proxy-shutdown-hook") { - @Override - public void run() { - shutdown(); - } - }); - - logger.info("setup complete"); - } catch (Throwable t) { - logger.log(Level.SEVERE, "Aborting start-up", t); + // exit if no active listeners + if (activeListeners.count() == 0) { + logger.severe( + "**** All listener threads failed to start - there is already a " + + "running instance listening on configured ports, or no listening ports " + + "configured!"); + logger.severe("Aborting start-up"); System.exit(1); } - } - }, - 5000 - ); - } catch (Throwable t) { - logger.log(Level.SEVERE, "Aborting start-up", t); - System.exit(1); - } - } - - protected void configureTokenAuthenticator() { - HttpClient httpClient = HttpClientBuilder.create(). - useSystemProperties(). - setUserAgent(httpUserAgent). - setMaxConnPerRoute(10). - setMaxConnTotal(10). - setConnectionTimeToLive(1, TimeUnit.MINUTES). - setRetryHandler(new DefaultHttpRequestRetryHandler(httpAutoRetries, true)). - setDefaultRequestConfig( - RequestConfig.custom(). - setContentCompressionEnabled(true). - setRedirectsEnabled(true). - setConnectTimeout(httpConnectTimeout). - setConnectionRequestTimeout(httpConnectTimeout). - setSocketTimeout(httpRequestTimeout).build()). - build(); - this.tokenAuthenticator = TokenAuthenticatorBuilder.create(). - setTokenValidationMethod(authMethod). - setHttpClient(httpClient). - setTokenIntrospectionServiceUrl(authTokenIntrospectionServiceUrl). - setTokenIntrospectionAuthorizationHeader(authTokenIntrospectionAuthorizationHeader). - setAuthResponseRefreshInterval(authResponseRefreshInterval). - setAuthResponseMaxTtl(authResponseMaxTtl). - setStaticToken(authStaticToken). - build(); - } + Runtime.getRuntime() + .addShutdownHook( + new Thread("proxy-shutdown-hook") { + @Override + public void run() { + shutdown(); + } + }); - /** - * Create RESTeasy proxies for remote calls via HTTP. - */ - protected WavefrontAPI createAgentService() { - ResteasyProviderFactory factory = ResteasyProviderFactory.getInstance(); - factory.registerProvider(JsonNodeWriter.class); - if (!factory.getClasses().contains(ResteasyJackson2Provider.class)) { - factory.registerProvider(ResteasyJackson2Provider.class); - } - if (httpUserAgent == null) { - httpUserAgent = "Wavefront-Proxy/" + props.getString("build.version"); - } - ClientHttpEngine httpEngine; - if (javaNetConnection) { - httpEngine = new JavaNetConnectionEngine() { - @Override - protected HttpURLConnection createConnection(ClientInvocation request) throws IOException { - HttpURLConnection connection = (HttpURLConnection) request.getUri().toURL().openConnection(); - connection.setRequestProperty("User-Agent", httpUserAgent); - connection.setRequestMethod(request.getMethod()); - connection.setConnectTimeout(httpConnectTimeout); // 5s - connection.setReadTimeout(httpRequestTimeout); // 60s - if (connection instanceof HttpsURLConnection) { - HttpsURLConnection secureConnection = (HttpsURLConnection) connection; - secureConnection.setSSLSocketFactory(new SSLSocketFactoryImpl( - HttpsURLConnection.getDefaultSSLSocketFactory(), - httpRequestTimeout)); - } - return connection; - } - }; - } else { - HttpClient httpClient = HttpClientBuilder.create(). - useSystemProperties(). - setUserAgent(httpUserAgent). - setMaxConnTotal(httpMaxConnTotal). - setMaxConnPerRoute(httpMaxConnPerRoute). - setConnectionTimeToLive(1, TimeUnit.MINUTES). - setDefaultSocketConfig( - SocketConfig.custom(). - setSoTimeout(httpRequestTimeout).build()). - setSSLSocketFactory(new SSLConnectionSocketFactoryImpl( - SSLConnectionSocketFactory.getSystemSocketFactory(), - httpRequestTimeout)). - setRetryHandler(new DefaultHttpRequestRetryHandler(httpAutoRetries, true) { - @Override - protected boolean handleAsIdempotent(HttpRequest request) { - // by default, retry all http calls (submissions are idempotent). - return true; + logger.info("setup complete"); } - }). - setDefaultRequestConfig( - RequestConfig.custom(). - setContentCompressionEnabled(true). - setRedirectsEnabled(true). - setConnectTimeout(httpConnectTimeout). - setConnectionRequestTimeout(httpConnectTimeout). - setSocketTimeout(httpRequestTimeout).build()). - build(); - final ApacheHttpClient4Engine apacheHttpClient4Engine = new ApacheHttpClient4Engine(httpClient, true); - // avoid using disk at all - apacheHttpClient4Engine.setFileUploadInMemoryThresholdLimit(100); - apacheHttpClient4Engine.setFileUploadMemoryUnit(ApacheHttpClient4Engine.MemoryUnit.MB); - httpEngine = apacheHttpClient4Engine; - } - ResteasyClient client = new ResteasyClientBuilder(). - httpEngine(httpEngine). - providerFactory(factory). - register(GZIPDecodingInterceptor.class). - register(gzipCompression ? GZIPEncodingInterceptor.class : DisableGZIPEncodingInterceptor.class). - register(AcceptEncodingGZIPFilter.class). - build(); - ResteasyWebTarget target = client.target(server); - return target.proxy(WavefrontAPI.class); - } - - private void setupQueueing(WavefrontAPI service) throws IOException { - shutdownTasks.add(() -> { - try { - queuedAgentExecutor.shutdownNow(); - // wait for up to httpRequestTimeout - queuedAgentExecutor.awaitTermination(httpRequestTimeout, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - // ignore - } - }); - agentAPI = new QueuedAgentService(service, bufferFile, retryThreads, queuedAgentExecutor, purgeBuffer, - agentId, splitPushWhenRateLimited, pushRateLimiter, token); - } - - /** - * Read or create the Daemon id for this machine. Reads from ~/.dshell/id. - */ - private void readOrCreateDaemonId() { - File agentIdFile; - if (idFile != null) { - agentIdFile = new File(idFile); - } else { - File userHome = new File(System.getProperty("user.home")); - if (!userHome.exists() || !userHome.isDirectory()) { - logger.severe("Cannot read from user.home, quitting"); - System.exit(1); - } - File configDirectory = new File(userHome, ".dshell"); - if (configDirectory.exists()) { - if (!configDirectory.isDirectory()) { - logger.severe(configDirectory + " must be a directory!"); - System.exit(1); - } - } else { - if (!configDirectory.mkdir()) { - logger.severe("Cannot create .dshell directory under " + userHome); - System.exit(1); - } - } - agentIdFile = new File(configDirectory, "id"); - } - if (agentIdFile.exists()) { - if (agentIdFile.isFile()) { - try { - agentId = UUID.fromString(Files.readFirstLine(agentIdFile, Charsets.UTF_8)); - logger.info("Proxy Id read from file: " + agentId); - } catch (IllegalArgumentException ex) { - logger.severe("Cannot read proxy id from " + agentIdFile + - ", content is malformed"); - System.exit(1); - } catch (IOException e) { - logger.log(Level.SEVERE, "Cannot read from " + agentIdFile, e); - System.exit(1); - } - } else { - logger.severe(agentIdFile + " is not a file!"); - System.exit(1); - } - } else { - agentId = UUID.randomUUID(); - logger.info("Proxy Id created: " + agentId); - try { - Files.write(agentId.toString(), agentIdFile, Charsets.UTF_8); - } catch (IOException e) { - logger.severe("Cannot write to " + agentIdFile); - System.exit(1); - } - } - } - - private void fetchConfigError(String errMsg, @Nullable String secondErrMsg) { - if (hadSuccessfulCheckin.get()) { - logger.severe(errMsg + (secondErrMsg == null ? "" : " " + secondErrMsg)); - } else { - logger.severe(Strings.repeat("*", errMsg.length())); - logger.severe(errMsg); - if (secondErrMsg != null) { - logger.severe(secondErrMsg); - } - logger.severe(Strings.repeat("*", errMsg.length())); - } - } - - /** - * Fetch configuration of the daemon from remote server. - * - * @return Fetched configuration. {@code null} if the configuration is invalid. - */ - @SuppressWarnings("ThrowableResultOfMethodCallIgnored") - private AgentConfiguration fetchConfig() { - AgentConfiguration newConfig = null; - JsonNode agentMetricsWorkingCopy; - long agentMetricsCaptureTsWorkingCopy; - synchronized(agentConfigurationExecutor) { - if (agentMetrics == null) return null; - agentMetricsWorkingCopy = agentMetrics; - agentMetricsCaptureTsWorkingCopy = agentMetricsCaptureTs; - agentMetrics = null; - } - logger.info("fetching configuration from server at: " + server); - try { - newConfig = agentAPI.checkin(agentId, hostname, token, props.getString("build.version"), - agentMetricsCaptureTsWorkingCopy, localAgent, agentMetricsWorkingCopy, pushAgent, ephemeral); - agentMetricsWorkingCopy = null; - hadSuccessfulCheckin.set(true); - } catch (NotAuthorizedException ex) { - fetchConfigError("HTTP 401 Unauthorized: Please verify that your server and token settings", - "are correct and that the token has Proxy Management permission!"); - agentMetricsWorkingCopy = null; - return new AgentConfiguration(); // return empty configuration to prevent checking in every second - } catch (ForbiddenException ex) { - fetchConfigError("HTTP 403 Forbidden: Please verify that your token has Proxy Management permission!", null); - agentMetricsWorkingCopy = null; - return new AgentConfiguration(); // return empty configuration to prevent checking in every second - } catch (ClientErrorException ex) { - if (ex.getResponse().getStatus() == 407) { - fetchConfigError("HTTP 407 Proxy Authentication Required: Please verify that proxyUser and proxyPassword", - "settings are correct and make sure your HTTP proxy is not rate limiting!"); - return null; - } - if (ex.getResponse().getStatus() == 404) { - fetchConfigError("HTTP 404 Not Found: Please verify that your server setting is correct: " + server, null); - return null; - } - fetchConfigError("HTTP " + ex.getResponse().getStatus() + " error: Unable to retrieve proxy configuration!", - server + ": " + Throwables.getRootCause(ex).getMessage()); - return null; - } catch (ProcessingException ex) { - Throwable rootCause = Throwables.getRootCause(ex); - if (rootCause instanceof UnknownHostException) { - fetchConfigError("Unknown host: " + server + ". Please verify your DNS and network settings!", null); - return null; - } - if (rootCause instanceof ConnectException || - rootCause instanceof SocketTimeoutException) { - fetchConfigError("Unable to connect to " + server + ": " + rootCause.getMessage(), - "Please verify your network/firewall settings!"); - return null; - } - fetchConfigError("Request processing error: Unable to retrieve proxy configuration!", - server + ": " + rootCause); - return null; - } catch (Exception ex) { - fetchConfigError("Unable to retrieve proxy configuration from remote server!", - server + ": " + Throwables.getRootCause(ex)); - return null; - } finally { - synchronized(agentConfigurationExecutor) { - // if check-in process failed (agentMetricsWorkingCopy is not null) and agent metrics have - // not been updated yet, restore last known set of agent metrics to be retried - if (agentMetricsWorkingCopy != null && agentMetrics == null) { - agentMetrics = agentMetricsWorkingCopy; - } - } - } - try { - if (newConfig.currentTime != null) { - Clock.set(newConfig.currentTime); - } - newConfig.validate(localAgent); - } catch (Exception ex) { - logger.log(Level.WARNING, "configuration file read from server is invalid", ex); - try { - agentAPI.agentError(agentId, "Configuration file is invalid: " + ex.toString()); - } catch (Exception e) { - logger.log(Level.WARNING, "cannot report error to collector", e); - } - return null; - } - return newConfig; - } - - protected PostPushDataTimedTask[] getFlushTasks(String handle) { - return getFlushTasks(Constants.PUSH_FORMAT_GRAPHITE_V2, handle); - } - - protected PostPushDataTimedTask[] getFlushTasks(String pushFormat, String handle) { - PostPushDataTimedTask[] toReturn = new PostPushDataTimedTask[flushThreads]; - logger.info("Using " + flushThreads + " flush threads to send batched " + pushFormat + - " data to Wavefront for data received on port: " + handle); - for (int i = 0; i < flushThreads; i++) { - final PostPushDataTimedTask postPushDataTimedTask = - new PostPushDataTimedTask(pushFormat, agentAPI, agentId, handle, i, pushRateLimiter, pushFlushInterval.get()); - toReturn[i] = postPushDataTimedTask; - managedTasks.add(postPushDataTimedTask); + }, + 5000); + } catch (Exception e) { + logger.log(Level.SEVERE, e.getMessage(), e); + // logger.severe(e.getMessage()); + System.exit(1); } - return toReturn; } /** * Actual agents can do additional configuration. * + * @param tenantName The tenant name * @param config The configuration to process. */ - protected void processConfiguration(AgentConfiguration config) { + protected void processConfiguration(String tenantName, AgentConfiguration config) { try { - agentAPI.agentConfigProcessed(agentId); + // for all ProxyV2API + for (String tn : TokenManager.getMulticastingTenantList().keySet()) { + apiContainer.getProxyV2APIForTenant(tn).proxyConfigProcessed(agentId); + } } catch (RuntimeException e) { // cannot throw or else configuration update thread would die. } } - private static void safeLogInfo(String msg) { - try { - logger.info(msg); - } catch (Throwable t) { - // ignore logging errors - } - } - + /** Best-effort graceful shutdown. */ public void shutdown() { - if (shuttingDown) { - return; // we need it only once - } - shuttingDown = true; + if (!shuttingDown.compareAndSet(false, true)) return; try { - safeLogInfo("Shutting down: Stopping listeners..."); - - stopListeners(); - - safeLogInfo("Shutting down: Stopping schedulers..."); - - for (ExecutorService executor : managedExecutors) { - executor.shutdownNow(); - // wait for up to request timeout - executor.awaitTermination(httpRequestTimeout, TimeUnit.MILLISECONDS); + try { + logger.info("Shutting down the proxy..."); + } catch (Throwable t) { + // ignore logging errors } - managedTasks.forEach(PostPushDataTimedTask::shutdown); - - safeLogInfo("Shutting down: Flushing pending points..."); - - for (PostPushDataTimedTask task : managedTasks) { - while (task.getNumPointsToSend() > 0) { - task.drainBuffersToQueue(); - } - } + System.out.println("Shutting down: Stopping listeners..."); + stopListeners(); - safeLogInfo("Shutting down: Running finalizing tasks..."); + System.out.println("Shutting down: Stopping schedulers..."); + if (proxyCheckinScheduler != null) proxyCheckinScheduler.shutdown(); + managedExecutors.forEach(ExecutorService::shutdownNow); + // wait for up to request timeout + managedExecutors.forEach( + x -> { + try { + x.awaitTermination(proxyConfig.getHttpRequestTimeout(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + // ignore + } + }); + System.out.println("Shutting down: Running finalizing tasks..."); shutdownTasks.forEach(Runnable::run); - safeLogInfo("Shutdown complete"); + System.out.println("Shutdown complete."); } catch (Throwable t) { try { logger.log(Level.SEVERE, "Error during shutdown: ", t); @@ -1667,86 +437,18 @@ public void shutdown() { } } - private static String getLocalHostName() { - InetAddress localAddress = null; - try { - Enumeration nics = NetworkInterface.getNetworkInterfaces(); - while (nics.hasMoreElements()) { - NetworkInterface network = nics.nextElement(); - if (!network.isUp() || network.isLoopback()) { - continue; - } - for (Enumeration addresses = network.getInetAddresses(); addresses.hasMoreElements(); ) { - InetAddress address = addresses.nextElement(); - if (address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isMulticastAddress()) { - continue; - } - if (address instanceof Inet4Address) { // prefer ipv4 - localAddress = address; - break; - } - if (localAddress == null) { - localAddress = address; - } - } - } - } catch (SocketException ex) { - // ignore - } - if (localAddress != null) { - return localAddress.getCanonicalHostName(); - } - return "localhost"; - } - - private MemoryPoolMXBean getTenuredGenPool() { - for (MemoryPoolMXBean pool : ManagementFactory.getMemoryPoolMXBeans()) { - if (pool.getType() == MemoryType.HEAP && pool.isUsageThresholdSupported()) { - return pool; - } - } - return null; - } - - private void setupMemoryGuard(double threshold) { - if (tenuredGenPool == null) return; - tenuredGenPool.setUsageThreshold((long) (tenuredGenPool.getUsage().getMax() * threshold)); + /** Starts all listeners as configured. */ + protected abstract void startListeners() throws Exception; - NotificationEmitter emitter = (NotificationEmitter) ManagementFactory.getMemoryMXBean(); - emitter.addNotificationListener((notification, obj) -> { - if (notification.getType().equals( - MemoryNotificationInfo.MEMORY_THRESHOLD_EXCEEDED)) { - logger.warning("Heap usage threshold exceeded - draining buffers to disk!"); - for (PostPushDataTimedTask task : managedTasks) { - if (task.getNumPointsToSend() > 0) { - task.drainBuffersToQueue(); - } - } - logger.info("Draining buffers to disk: finished"); - } - }, null, null); - } + /** Stops all listeners before terminating the process. */ + protected abstract void stopListeners(); /** - * Return a unique process identifier used to prevent collisions in ~proxy metrics. - * Try to extract system PID from RuntimeMXBean name string (usually in the "11111@hostname" format). - * If it's not parsable or an extracted PID is too low, for example, when running in containerized - * environment, chances of ID collision are much higher, so we use a random 32bit hex string instead. + * Shut down specific listener pipeline. * - * @return unique process identifier string + * @param port port number. */ - private static String getProcessId() { - try { - final String runtime = ManagementFactory.getRuntimeMXBean().getName(); - if (runtime.indexOf("@") >= 1) { - long id = Long.parseLong(runtime.substring(0, runtime.indexOf("@"))); - if (id > 1000) { - return Long.toString(id); - } - } - } catch (Exception e) { - // can't resolve process ID, fall back to using random ID - } - return Integer.toHexString((int) (Math.random() * Integer.MAX_VALUE)); - } + protected abstract void stopListener(int port); + + protected abstract void truncateBacklog(); } diff --git a/proxy/src/main/java/com/wavefront/agent/InteractiveTester.java b/proxy/src/main/java/com/wavefront/agent/InteractiveTester.java new file mode 100644 index 000000000..91e2378d1 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/InteractiveTester.java @@ -0,0 +1,19 @@ +package com.wavefront.agent; + +import com.wavefront.agent.config.ConfigurationException; + +/** + * Base interface for all interactive testers (logs and preprocessor at the moment). + * + * @author vasily@wavefront.com + */ +public interface InteractiveTester { + + /** + * Read line from stdin and process it. + * + * @return true if there's more input to process + * @throws ConfigurationException + */ + boolean interactiveTest() throws ConfigurationException; +} diff --git a/proxy/src/main/java/com/wavefront/agent/JavaNetConnectionEngine.java b/proxy/src/main/java/com/wavefront/agent/JavaNetConnectionEngine.java deleted file mode 100644 index e41e1d1f2..000000000 --- a/proxy/src/main/java/com/wavefront/agent/JavaNetConnectionEngine.java +++ /dev/null @@ -1,180 +0,0 @@ -package com.wavefront.agent; - -import org.jboss.resteasy.client.jaxrs.ClientHttpEngine; -import org.jboss.resteasy.client.jaxrs.i18n.Messages; -import org.jboss.resteasy.client.jaxrs.internal.ClientInvocation; -import org.jboss.resteasy.client.jaxrs.internal.ClientResponse; -import org.jboss.resteasy.util.CaseInsensitiveMap; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; -import javax.ws.rs.ProcessingException; -import javax.ws.rs.core.MultivaluedMap; - -/** - * {@link ClientHttpEngine} that uses {@link HttpURLConnection} to connect to an Http endpoint. - * - * @author Clement Pang (clement@wavefront.com). - */ -public class JavaNetConnectionEngine implements ClientHttpEngine { - - protected SSLContext sslContext; - protected HostnameVerifier hostnameVerifier; - - public JavaNetConnectionEngine() { - } - - public ClientResponse invoke(ClientInvocation request) { - final HttpURLConnection connection; - int status; - try { - connection = this.createConnection(request); - this.executeRequest(request, connection); - status = connection.getResponseCode(); - } catch (IOException ex) { - throw new ProcessingException(Messages.MESSAGES.unableToInvokeRequest(), ex); - } - - ClientResponse response = new JavaNetConnectionClientResponse(request, connection); - response.setStatus(status); - response.setHeaders(this.getHeaders(connection)); - return response; - } - - protected MultivaluedMap getHeaders(HttpURLConnection connection) { - CaseInsensitiveMap headers = new CaseInsensitiveMap<>(); - final Iterator>> headerFieldsIter = - connection.getHeaderFields().entrySet().iterator(); - - while (true) { - Map.Entry> header; - do { - if (!headerFieldsIter.hasNext()) { - return headers; - } - - header = headerFieldsIter.next(); - } while (header.getKey() == null); - - final Iterator valuesIterator = header.getValue().iterator(); - - while (valuesIterator.hasNext()) { - String value = valuesIterator.next(); - headers.add(header.getKey(), value); - } - } - } - - public void close() { - } - - protected HttpURLConnection createConnection(ClientInvocation request) throws IOException { - HttpURLConnection connection = (HttpURLConnection) request.getUri().toURL().openConnection(); - connection.setRequestMethod(request.getMethod()); - return connection; - } - - protected void executeRequest(ClientInvocation request, HttpURLConnection connection) { - connection.setInstanceFollowRedirects(request.getMethod().equals("GET")); - if (request.getEntity() != null) { - if (request.getMethod().equals("GET")) { - throw new ProcessingException(Messages.MESSAGES.getRequestCannotHaveBody()); - } - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - request.getDelegatingOutputStream().setDelegate(baos); - - try { - request.writeRequestBody(request.getEntityStream()); - baos.close(); - this.commitHeaders(request, connection); - connection.setDoOutput(true); - OutputStream e = connection.getOutputStream(); - e.write(baos.toByteArray()); - e.flush(); - e.close(); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } else { - this.commitHeaders(request, connection); - } - - } - - protected void commitHeaders(ClientInvocation request, HttpURLConnection connection) { - final MultivaluedMap headers = request.getHeaders().asMap(); - - for (Map.Entry> header : headers.entrySet()) { - final List values = header.getValue(); - for (String value : values) { - connection.addRequestProperty(header.getKey(), value); - } - } - } - - public SSLContext getSslContext() { - return this.sslContext; - } - - public HostnameVerifier getHostnameVerifier() { - return this.hostnameVerifier; - } - - public void setSslContext(SSLContext sslContext) { - this.sslContext = sslContext; - } - - public void setHostnameVerifier(HostnameVerifier hostnameVerifier) { - this.hostnameVerifier = hostnameVerifier; - } - - private static class JavaNetConnectionClientResponse extends ClientResponse { - private HttpURLConnection connection; - private InputStream stream; - - public JavaNetConnectionClientResponse(ClientInvocation request, HttpURLConnection connection) { - super(request.getClientConfiguration()); - this.connection = connection; - } - - protected InputStream getInputStream() { - if (this.stream == null) { - try { - this.stream = this.status < 300 ? connection.getInputStream() : connection.getErrorStream(); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } - - return this.stream; - } - - protected void setInputStream(InputStream is) { - this.stream = is; - } - - public void releaseConnection() throws IOException { - InputStream is = this.getInputStream(); - if (is != null) { - is.close(); - } - connection.disconnect(); - - this.stream = null; - this.connection = null; - this.properties = null; - this.configuration = null; - this.bufferedEntity = null; - } - } -} diff --git a/proxy/src/main/java/com/wavefront/agent/JsonNodeWriter.java b/proxy/src/main/java/com/wavefront/agent/JsonNodeWriter.java index 5f0f10c68..bd68b0cba 100644 --- a/proxy/src/main/java/com/wavefront/agent/JsonNodeWriter.java +++ b/proxy/src/main/java/com/wavefront/agent/JsonNodeWriter.java @@ -4,12 +4,10 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; - import java.io.IOException; import java.io.OutputStream; import java.lang.annotation.Annotation; import java.lang.reflect.Type; - import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MultivaluedMap; @@ -26,20 +24,31 @@ public class JsonNodeWriter implements MessageBodyWriter { private final JsonFactory factory = new JsonFactory(); @Override - public boolean isWriteable(Class type, Type genericType, Annotation[] annotations, MediaType mediaType) { + public boolean isWriteable( + Class type, Type genericType, Annotation[] annotations, MediaType mediaType) { return JsonNode.class.isAssignableFrom(type); } @Override - public long getSize(JsonNode jsonNode, Class type, Type genericType, Annotation[] annotations, - MediaType mediaType) { + public long getSize( + JsonNode jsonNode, + Class type, + Type genericType, + Annotation[] annotations, + MediaType mediaType) { return -1; } @Override - public void writeTo(JsonNode jsonNode, Class type, Type genericType, Annotation[] annotations, - MediaType mediaType, MultivaluedMap httpHeaders, - OutputStream entityStream) throws IOException, WebApplicationException { + public void writeTo( + JsonNode jsonNode, + Class type, + Type genericType, + Annotation[] annotations, + MediaType mediaType, + MultivaluedMap httpHeaders, + OutputStream entityStream) + throws IOException, WebApplicationException { JsonGenerator generator = factory.createGenerator(entityStream); mapper.writeTree(generator, jsonNode); } diff --git a/proxy/src/main/java/com/wavefront/agent/LogsUtil.java b/proxy/src/main/java/com/wavefront/agent/LogsUtil.java new file mode 100644 index 000000000..0506dffe0 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/LogsUtil.java @@ -0,0 +1,33 @@ +package com.wavefront.agent; + +import com.wavefront.agent.formatter.DataFormat; +import com.wavefront.common.TaggedMetricName; +import com.yammer.metrics.core.Counter; +import com.yammer.metrics.core.Histogram; +import com.yammer.metrics.core.MetricsRegistry; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** @author Sumit Deo (deosu@vmware.com) */ +public class LogsUtil { + + public static final Set LOGS_DATA_FORMATS = + new HashSet<>( + Arrays.asList( + DataFormat.LOGS_JSON_ARR, + DataFormat.LOGS_JSON_LINES, + DataFormat.LOGS_JSON_CLOUDWATCH)); + + public static Counter getOrCreateLogsCounterFromRegistry( + MetricsRegistry registry, DataFormat format, String group, String name) { + return registry.newCounter( + new TaggedMetricName(group, name, "format", format.name().toLowerCase())); + } + + public static Histogram getOrCreateLogsHistogramFromRegistry( + MetricsRegistry registry, DataFormat format, String group, String name) { + return registry.newHistogram( + new TaggedMetricName(group, name, "format", format.name().toLowerCase()), false); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/PointHandler.java b/proxy/src/main/java/com/wavefront/agent/PointHandler.java deleted file mode 100644 index 497da6267..000000000 --- a/proxy/src/main/java/com/wavefront/agent/PointHandler.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.wavefront.agent; - -import java.util.List; - -import javax.annotation.Nullable; - -import wavefront.report.ReportPoint; - -/** - * Interface for a handler of Report Points. - * - * @author Clement Pang (clement@wavefront.com). - */ -public interface PointHandler { - /** - * Send a point for reporting. - * - * @param point Point to report. - * @param debugLine Debug information to print to console when the line is rejected. - * If null, then use the entire point converted to string. - */ - void reportPoint(ReportPoint point, @Nullable String debugLine); - - /** - * Send a collection of points for reporting. - * - * @param points Points to report. - */ - void reportPoints(List points); - - /** - * Called when a blocked line is encountered. - * - * @param pointLine Line encountered. If null, it will increment the blocked points counter - * but won't write to the log - */ - void handleBlockedPoint(@Nullable String pointLine); -} diff --git a/proxy/src/main/java/com/wavefront/agent/PointHandlerImpl.java b/proxy/src/main/java/com/wavefront/agent/PointHandlerImpl.java deleted file mode 100644 index 1b1d87423..000000000 --- a/proxy/src/main/java/com/wavefront/agent/PointHandlerImpl.java +++ /dev/null @@ -1,236 +0,0 @@ -package com.wavefront.agent; - -import com.wavefront.common.Clock; -import com.wavefront.data.Validation; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Histogram; -import com.yammer.metrics.core.MetricName; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.NumberUtils; -import org.apache.commons.lang.time.DateUtils; - -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.annotation.Nullable; - -import wavefront.report.ReportPoint; - -import static com.wavefront.data.Validation.validatePoint; - -/** - * Adds all graphite strings to a working list, and batches them up on a set schedule (100ms) to be sent (through the - * daemon's logic) up to the collector on the server side. - */ -public class PointHandlerImpl implements PointHandler { - - private static final Logger logger = Logger.getLogger(PointHandlerImpl.class.getCanonicalName()); - private static final Logger blockedPointsLogger = Logger.getLogger("RawBlockedPoints"); - private static final Logger validPointsLogger = Logger.getLogger("RawValidPoints"); - private static final Random RANDOM = new Random(); - - private final Histogram receivedPointLag; - private final String validationLevel; - private final String handle; - private boolean logPoints = false; - private final double logPointsSampleRate; - private volatile long logPointsUpdatedMillis = 0L; - - /** - * Value of system property wavefront.proxy.logging (for backwards compatibility) - */ - private final boolean logPointsFlag; - - @Nullable - private final String prefix; - - protected final int blockedPointsPerBatch; - protected final PostPushDataTimedTask[] sendDataTasks; - - public PointHandlerImpl(final String handle, - final String validationLevel, - final int blockedPointsPerBatch, - final PostPushDataTimedTask[] sendDataTasks) { - this(handle, validationLevel, blockedPointsPerBatch, null, sendDataTasks); - } - - public PointHandlerImpl(final String handle, - final String validationLevel, - final int blockedPointsPerBatch, - @Nullable final String prefix, - final PostPushDataTimedTask[] sendDataTasks) { - this.validationLevel = validationLevel; - this.handle = handle; - this.blockedPointsPerBatch = blockedPointsPerBatch; - this.prefix = prefix; - String logPointsProperty = System.getProperty("wavefront.proxy.logpoints"); - this.logPointsFlag = logPointsProperty != null && logPointsProperty.equalsIgnoreCase("true"); - String logPointsSampleRateProperty = System.getProperty("wavefront.proxy.logpoints.sample-rate"); - this.logPointsSampleRate = logPointsSampleRateProperty != null && - NumberUtils.isNumber(logPointsSampleRateProperty) ? Double.parseDouble(logPointsSampleRateProperty) : 1.0d; - - this.receivedPointLag = Metrics.newHistogram(new MetricName("points." + handle + ".received", "", "lag")); - - this.sendDataTasks = sendDataTasks; - } - - @Override - public void reportPoint(ReportPoint point, @Nullable String debugLine) { - final PostPushDataTimedTask randomPostTask = getRandomPostTask(); - try { - if (prefix != null) { - point.setMetric(prefix + "." + point.getMetric()); - } - validatePoint( - point, - handle, - validationLevel == null ? null : Validation.Level.valueOf(validationLevel)); - - String strPoint = pointToString(point); - - if (logPointsUpdatedMillis + TimeUnit.SECONDS.toMillis(1) < System.currentTimeMillis()) { - // refresh validPointsLogger level once a second - if (logPoints != validPointsLogger.isLoggable(Level.FINEST)) { - logPoints = !logPoints; - logger.info("Valid points logging is now " + (logPoints ? - "enabled with " + (logPointsSampleRate * 100) + "% sampling": - "disabled")); - } - logPointsUpdatedMillis = System.currentTimeMillis(); - } - if ((logPoints || logPointsFlag) && - (logPointsSampleRate >= 1.0d || (logPointsSampleRate > 0.0d && RANDOM.nextDouble() < logPointsSampleRate))) { - // we log valid points only if system property wavefront.proxy.logpoints is true or RawValidPoints log level is - // set to "ALL". this is done to prevent introducing overhead and accidentally logging points to the main log - // Additionally, honor sample rate limit, if set. - validPointsLogger.info(strPoint); - } - randomPostTask.addPoint(strPoint); - randomPostTask.enforceBufferLimits(); - receivedPointLag.update(Clock.now() - point.getTimestamp()); - - } catch (IllegalArgumentException e) { - String pointString = pointToString(point); - blockedPointsLogger.warning(pointString); - this.handleBlockedPoint(e.getMessage() + " (" + (debugLine == null ? pointString : debugLine) + ")"); - } catch (Exception ex) { - logger.log(Level.SEVERE, "WF-500 Uncaught exception when handling point (" + - (debugLine == null ? pointToString(point) : debugLine) + ")", ex); - } - } - - @Override - public void reportPoints(List points) { - for (final ReportPoint point : points) { - reportPoint(point, null); - } - } - - public PostPushDataTimedTask getRandomPostTask() { - // return the task with the lowest number of pending points and, if possible, not currently flushing to retry queue - long min = Long.MAX_VALUE; - PostPushDataTimedTask randomPostTask = null; - PostPushDataTimedTask firstChoicePostTask = null; - for (int i = 0; i < this.sendDataTasks.length; i++) { - long pointsToSend = this.sendDataTasks[i].getNumPointsToSend(); - if (pointsToSend < min) { - min = pointsToSend; - randomPostTask = this.sendDataTasks[i]; - if (!this.sendDataTasks[i].getFlushingToQueueFlag()) { - firstChoicePostTask = this.sendDataTasks[i]; - } - } - } - return firstChoicePostTask == null ? randomPostTask : firstChoicePostTask; - } - - @Override - public void handleBlockedPoint(@Nullable String pointLine) { - final PostPushDataTimedTask randomPostTask = getRandomPostTask(); - if (pointLine != null && randomPostTask.getBlockedSampleSize() < this.blockedPointsPerBatch) { - randomPostTask.addBlockedSample(pointLine); - } - randomPostTask.incrementBlockedPoints(); - } - - private static String quote = "\""; - private static String escapedQuote = "\\\""; - - private static String escapeQuotes(String raw) { - return StringUtils.replace(raw, quote, escapedQuote); - } - - private static void appendTagMap(StringBuilder sb, @Nullable Map tags) { - if (tags == null) { - return; - } - for (Map.Entry entry : tags.entrySet()) { - sb.append(' ').append(quote).append(escapeQuotes(entry.getKey())).append(quote) - .append("=") - .append(quote).append(escapeQuotes(entry.getValue())).append(quote); - } - } - - private static String pointToStringSB(ReportPoint point) { - if (point.getValue() instanceof Double || point.getValue() instanceof Long || point.getValue() instanceof String) { - StringBuilder sb = new StringBuilder(quote) - .append(escapeQuotes(point.getMetric())).append(quote).append(" ") - .append(point.getValue()).append(" ") - .append(point.getTimestamp() / 1000).append(" ") - .append("source=").append(quote).append(escapeQuotes(point.getHost())).append(quote); - appendTagMap(sb, point.getAnnotations()); - return sb.toString(); - } else if (point.getValue() instanceof wavefront.report.Histogram){ - wavefront.report.Histogram h = (wavefront.report.Histogram) point.getValue(); - - StringBuilder sb = new StringBuilder(); - - // BinType - switch (h.getDuration()) { - case (int) DateUtils.MILLIS_PER_MINUTE: - sb.append("!M "); - break; - case (int) DateUtils.MILLIS_PER_HOUR: - sb.append("!H "); - break; - case (int) DateUtils.MILLIS_PER_DAY: - sb.append("!D "); - break; - default: - throw new RuntimeException("Unexpected histogram duration " + h.getDuration()); - } - - // Timestamp - sb.append(point.getTimestamp() / 1000).append(' '); - - // Centroids - int numCentroids = Math.min(CollectionUtils.size(h.getBins()), CollectionUtils.size(h.getCounts())); - for (int i=0; i points = new ArrayList<>(); - private final Object pointsMutex = new Object(); - private final List blockedSamples = new ArrayList<>(); - - private final String pushFormat; - private final Object blockedSamplesMutex = new Object(); - - /** - * Warn about exceeding the rate limit no more than once per 10 seconds (per thread) - */ - private final RateLimiter warningMessageRateLimiter = RateLimiter.create(0.1); - - /** - * Print summary once a minute - */ - private final RateLimiter summaryMessageRateLimiter = RateLimiter.create(0.017); - - /** - * Write a sample of blocked points to log once a minute - */ - private final RateLimiter blockedSamplesRateLimiter = RateLimiter.create(0.017); - - /** - * Attempt to schedule drainBuffersToQueueTask no more than once every 100ms to reduce - * scheduler overhead under memory pressure - */ - private final RateLimiter drainBuffersRateLimiter = RateLimiter.create(10); - - private final RecyclableRateLimiter pushRateLimiter; - - private final Counter pointsReceived; - private final Counter pointsAttempted; - private final Counter pointsQueued; - private final Counter pointsBlocked; - private final Counter permitsGranted; - private final Counter permitsDenied; - private final Counter permitsRetried; - private final Counter batchesAttempted; - private final Counter bufferFlushCount; - private final Timer batchSendTime; - - private long numApiCalls = 0; - - private UUID daemonId; - private String handle; - private final int threadId; - private long pushFlushInterval; - private final ScheduledExecutorService scheduler; - private final ExecutorService flushExecutor; - - private static AtomicInteger pointsPerBatch = new AtomicInteger(50000); - private static AtomicInteger memoryBufferLimit = new AtomicInteger(50000 * 32); - private boolean isFlushingToQueue = false; - - private ForceQueueEnabledAgentAPI agentAPI; - - static void setPointsPerBatch(AtomicInteger newSize) { - pointsPerBatch = newSize; - } - - static void setMemoryBufferLimit(AtomicInteger newSize) { - memoryBufferLimit = newSize; - } - - public void addPoint(String metricString) { - pointsReceived.inc(); - synchronized (pointsMutex) { - this.points.add(metricString); - } - } - - public void addPoints(List metricStrings) { - pointsReceived.inc(metricStrings.size()); - synchronized (pointsMutex) { - this.points.addAll(metricStrings); - } - } - - public int getBlockedSampleSize() { - synchronized (blockedSamplesMutex) { - return blockedSamples.size(); - } - } - - public void addBlockedSample(String blockedSample) { - synchronized (blockedSamplesMutex) { - blockedSamples.add(blockedSample); - } - } - - public void incrementBlockedPoints() { - this.pointsBlocked.inc(); - } - - public long getAttemptedPoints() { - return this.pointsAttempted.count(); - } - - public long getNumPointsQueued() { - return this.pointsQueued.count(); - } - - public long getNumPointsToSend() { - return this.points.size(); - } - - public boolean getFlushingToQueueFlag() { - return isFlushingToQueue; - } - - public long getNumApiCalls() { - return numApiCalls; - } - - public UUID getDaemonId() { - return daemonId; - } - - @Deprecated - public PostPushDataTimedTask(String pushFormat, ForceQueueEnabledAgentAPI agentAPI, String logLevel, - UUID daemonId, String handle, int threadId, RecyclableRateLimiter pushRateLimiter, - long pushFlushInterval) { - this(pushFormat, agentAPI, daemonId, handle, threadId, pushRateLimiter, pushFlushInterval); - } - - public PostPushDataTimedTask(String pushFormat, ForceQueueEnabledAgentAPI agentAPI, - UUID daemonId, String handle, int threadId, RecyclableRateLimiter pushRateLimiter, - long pushFlushInterval) { - this.pushFormat = pushFormat; - this.daemonId = daemonId; - this.handle = handle; - this.threadId = threadId; - this.pushFlushInterval = pushFlushInterval; - this.agentAPI = agentAPI; - this.pushRateLimiter = pushRateLimiter; - this.scheduler = Executors.newScheduledThreadPool(1, - new NamedThreadFactory("submitter-main-" + handle + "-" + String.valueOf(threadId))); - this.flushExecutor = new ThreadPoolExecutor(1, 1, 60L, TimeUnit.MINUTES, - new SynchronousQueue<>(), new NamedThreadFactory("flush-" + handle + "-" + String.valueOf(threadId))); - - this.pointsAttempted = Metrics.newCounter(new MetricName("points." + handle, "", "sent")); - this.pointsQueued = Metrics.newCounter(new MetricName("points." + handle, "", "queued")); - this.pointsBlocked = Metrics.newCounter(new MetricName("points." + handle, "", "blocked")); - this.pointsReceived = Metrics.newCounter(new MetricName("points." + handle, "", "received")); - this.permitsGranted = Metrics.newCounter(new MetricName("limiter", "", "permits-granted")); - this.permitsDenied = Metrics.newCounter(new MetricName("limiter", "", "permits-denied")); - this.permitsRetried = Metrics.newCounter(new MetricName("limiter", "", "permits-retried")); - this.batchesAttempted = Metrics.newCounter( - new MetricName("push." + String.valueOf(handle) + ".thread-" + String.valueOf(threadId), "", "batches")); - this.batchSendTime = Metrics.newTimer(new MetricName("push." + handle, "", "duration"), - TimeUnit.MILLISECONDS, TimeUnit.MINUTES); - this.bufferFlushCount = Metrics.newCounter(new MetricName("buffer", "", "flush-count")); - this.scheduler.schedule(this, pushFlushInterval, TimeUnit.MILLISECONDS); - } - - @Override - public void run() { - long nextRunMillis = this.pushFlushInterval; - try { - List current = createAgentPostBatch(); - batchesAttempted.inc(); - if (current.size() == 0) { - return; - } - if (pushRateLimiter == null || pushRateLimiter.tryAcquire(current.size())) { - if (pushRateLimiter != null) this.permitsGranted.inc(current.size()); - - TimerContext timerContext = this.batchSendTime.time(); - Response response = null; - try { - response = agentAPI.postPushData( - daemonId, - Constants.GRAPHITE_BLOCK_WORK_UNIT, - System.currentTimeMillis(), - pushFormat, - StringLineIngester.joinPushData(current)); - int pointsInList = current.size(); - this.pointsAttempted.inc(pointsInList); - if (response.getStatus() == Response.Status.NOT_ACCEPTABLE.getStatusCode()) { - if (pushRateLimiter != null) { - this.pushRateLimiter.recyclePermits(pointsInList); - this.permitsRetried.inc(pointsInList); - } - this.pointsQueued.inc(pointsInList); - } - } finally { - numApiCalls++; - timerContext.stop(); - if (response != null) response.close(); - } - enforceBufferLimits(); - } else { - this.permitsDenied.inc(current.size()); - // if proxy rate limit exceeded, try again in 250..500ms (to introduce some degree of fairness) - nextRunMillis = 250 + (int) (Math.random() * 250); - if (warningMessageRateLimiter.tryAcquire()) { - logger.warning("[FLUSH THREAD " + threadId + "]: WF-4 Proxy rate limit exceeded " + - "(pending points: " + points.size() + "), will retry"); - } - synchronized (pointsMutex) { // return the batch to the beginning of the queue - points.addAll(0, current); - } - } - } catch (Throwable t) { - logger.log(Level.SEVERE, "Unexpected error in flush loop", t); - } finally { - scheduler.schedule(this, nextRunMillis, TimeUnit.MILLISECONDS); - } - } - - /** - * Shut down the scheduler for this task (prevent future scheduled runs) - */ - public void shutdown() { - try { - scheduler.shutdownNow(); - scheduler.awaitTermination(1000L, TimeUnit.MILLISECONDS); - } catch (Throwable t) { - logger.log(Level.SEVERE, "Error during shutdown", t); - } - } - - public void enforceBufferLimits() { - if (points.size() > memoryBufferLimit.get() && drainBuffersRateLimiter.tryAcquire()) { - try { - flushExecutor.submit(drainBuffersToQueueTask); - } catch (RejectedExecutionException e) { - // ignore - another task is already being executed - } - } - } - - private Runnable drainBuffersToQueueTask = new Runnable() { - @Override - public void run() { - if (points.size() > memoryBufferLimit.get()) { - // there are going to be too many points to be able to flush w/o the agent blowing up - // drain the leftovers straight to the retry queue (i.e. to disk) - // don't let anyone add any more to points while we're draining it. - logger.warning("[FLUSH THREAD " + threadId + "]: WF-3 Too many pending points (" + points.size() + - "), block size: " + pointsPerBatch + ". flushing to retry queue"); - drainBuffersToQueue(); - logger.info("[FLUSH THREAD " + threadId + "]: flushing to retry queue complete. " + - "Pending points: " + points.size()); - } - } - }; - - public void drainBuffersToQueue() { - try { - isFlushingToQueue = true; - int lastBatchSize = Integer.MIN_VALUE; - // roughly limit number of points to flush to the the current buffer size (+1 blockSize max) - // if too many points arrive at the proxy while it's draining, they will be taken care of in the next run - int pointsToFlush = points.size(); - while (pointsToFlush > 0) { - List pushData = createAgentPostBatch(); - int pushDataPointCount = pushData.size(); - if (pushDataPointCount > 0) { - agentAPI.postPushData(daemonId, Constants.GRAPHITE_BLOCK_WORK_UNIT, - System.currentTimeMillis(), Constants.PUSH_FORMAT_GRAPHITE_V2, - StringLineIngester.joinPushData(pushData), true); - - // update the counters as if this was a failed call to the API - this.pointsAttempted.inc(pushDataPointCount); - this.pointsQueued.inc(pushDataPointCount); - if (pushRateLimiter != null) { - this.permitsDenied.inc(pushDataPointCount); - } - numApiCalls++; - pointsToFlush -= pushDataPointCount; - - // stop draining buffers if the batch is smaller than the previous one - if (pushDataPointCount < lastBatchSize) { - break; - } - lastBatchSize = pushDataPointCount; - } else { - break; - } - } - } finally { - isFlushingToQueue = false; - bufferFlushCount.inc(); - } - } - - private void logBlockedPoints() { - if (blockedSamplesRateLimiter.tryAcquire()) { - List currentBlockedSamples = new ArrayList<>(); - if (!blockedSamples.isEmpty()) { - synchronized (blockedSamplesMutex) { - // Copy this to a temp structure that we can iterate over for printing below - currentBlockedSamples.addAll(blockedSamples); - blockedSamples.clear(); - } - } - for (String blockedLine : currentBlockedSamples) { - logger.info("[" + handle + "] blocked input: [" + blockedLine + "]"); - } - } - } - - private List createAgentPostBatch() { - List current; - int blockSize; - synchronized (pointsMutex) { - blockSize = Math.min(points.size(), pointsPerBatch.get()); - current = points.subList(0, blockSize); - points = new ArrayList<>(points.subList(blockSize, points.size())); - } - if (summaryMessageRateLimiter.tryAcquire()) { - logger.info("[" + handle + "] (SUMMARY): points attempted: " + getAttemptedPoints() + - "; blocked: " + this.pointsBlocked.count()); - } - logBlockedPoints(); - logger.fine("[" + handle + "] (DETAILED): sending " + current.size() + " valid points" + - "; points in memory: " + points.size() + - "; total attempted points: " + getAttemptedPoints() + - "; total blocked: " + this.pointsBlocked.count() + - "; total queued: " + getNumPointsQueued()); - return current; - } -} diff --git a/proxy/src/main/java/com/wavefront/agent/ProxyCheckInScheduler.java b/proxy/src/main/java/com/wavefront/agent/ProxyCheckInScheduler.java new file mode 100644 index 000000000..358dfc0b7 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/ProxyCheckInScheduler.java @@ -0,0 +1,487 @@ +package com.wavefront.agent; + +import static com.wavefront.common.Utils.getBuildVersion; +import static org.apache.commons.lang3.ObjectUtils.firstNonNull; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.base.Throwables; +import com.google.common.collect.Maps; +import com.wavefront.agent.api.APIContainer; +import com.wavefront.agent.preprocessor.ProxyPreprocessorConfigManager; +import com.wavefront.api.agent.AgentConfiguration; +import com.wavefront.api.agent.ValidationConfiguration; +import com.wavefront.common.Clock; +import com.wavefront.common.NamedThreadFactory; +import com.wavefront.metrics.JsonMetricsGenerator; +import com.yammer.metrics.Metrics; +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; +import javax.ws.rs.ClientErrorException; +import javax.ws.rs.ProcessingException; +import org.apache.commons.lang.StringUtils; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Registers the proxy with the back-end, sets up regular "check-ins" (every minute), transmits + * proxy metrics to the back-end. + * + * @author vasily@wavefront.com + */ +public class ProxyCheckInScheduler { + private static final Logger logger = Logger.getLogger("proxy"); + private static final int MAX_CHECKIN_ATTEMPTS = 5; + + /** + * A unique value (a random hexadecimal string), assigned at proxy start-up, to be reported with + * all ~proxy metrics as a "processId" point tag to prevent potential ~proxy metrics collisions + * caused by users spinning up multiple proxies with duplicate names. + */ + private static final String ID = Integer.toHexString((int) (Math.random() * Integer.MAX_VALUE)); + + private final UUID proxyId; + private final ProxyConfig proxyConfig; + private final APIContainer apiContainer; + private final BiConsumer agentConfigurationConsumer; + private final Runnable shutdownHook; + private final Runnable truncateBacklog; + private final AtomicInteger retries = new AtomicInteger(0); + private final AtomicLong successfulCheckIns = new AtomicLong(0); + /** Executors for support tasks. */ + private final ScheduledExecutorService executor = + Executors.newScheduledThreadPool(2, new NamedThreadFactory("proxy-configuration")); + + private String serverEndpointUrl = null; + private volatile JsonNode agentMetrics; + private boolean retryImmediately = false; + + // check if preprocessor rules need to be sent to update BE + public static AtomicBoolean preprocessorRulesNeedUpdate = new AtomicBoolean(false); + // check if rules are set from FE/API + public static AtomicBoolean isRulesSetInFE = new AtomicBoolean(false); + + /** + * @param proxyId Proxy UUID. + * @param proxyConfig Proxy settings. + * @param apiContainer API container object. + * @param agentConfigurationConsumer Configuration processor, invoked after each successful + * configuration fetch. + * @param shutdownHook Invoked when proxy receives a shutdown directive from the back-end. + */ + public ProxyCheckInScheduler( + UUID proxyId, + ProxyConfig proxyConfig, + APIContainer apiContainer, + BiConsumer agentConfigurationConsumer, + Runnable shutdownHook, + Runnable truncateBacklog) { + this.proxyId = proxyId; + this.proxyConfig = proxyConfig; + this.apiContainer = apiContainer; + this.agentConfigurationConsumer = agentConfigurationConsumer; + this.shutdownHook = shutdownHook; + this.truncateBacklog = truncateBacklog; + + updateProxyMetrics(); + + Map configList = checkin(); + new ProxySendConfigScheduler(apiContainer, proxyId, proxyConfig).start(); + + if (configList == null && retryImmediately) { + // immediately retry check-ins if we need to re-attempt + // due to changing the server endpoint URL + updateProxyMetrics(); + configList = checkin(); + sendPreprocessorRules(); + } + if (configList != null && !configList.isEmpty()) { + logger.info("initial configuration is available, setting up proxy"); + for (Map.Entry configEntry : configList.entrySet()) { + agentConfigurationConsumer.accept(configEntry.getKey(), configEntry.getValue()); + successfulCheckIns.incrementAndGet(); + } + } + } + + /** Set up and schedule regular check-ins. */ + public void scheduleCheckins() { + logger.info("scheduling regular check-ins"); + executor.scheduleAtFixedRate(this::updateProxyMetrics, 60, 60, TimeUnit.SECONDS); + executor.scheduleWithFixedDelay(this::updateConfiguration, 0, 1, TimeUnit.SECONDS); + } + + /** + * Returns the number of successful check-ins. + * + * @return true if this proxy had at least one successful check-in. + */ + public long getSuccessfulCheckinCount() { + return successfulCheckIns.get(); + } + + /** Stops regular check-ins. */ + public void shutdown() { + executor.shutdown(); + } + + /** + * Initial sending of preprocessor rules. Will always check local location for preprocessor rule. + */ + public void sendPreprocessorRules() { + if (preprocessorRulesNeedUpdate.getAndSet(false)) { + try { + JsonNode rulesNode = createRulesNode(ProxyPreprocessorConfigManager.getProxyConfigRules(), null); + apiContainer + .getProxyV2APIForTenant(APIContainer.CENTRAL_TENANT_NAME) + .proxySavePreprocessorRules( + proxyId, + rulesNode + ); + } catch (javax.ws.rs.NotFoundException ex) { + logger.warning("'proxySavePreprocessorRules' api end point not found"); + } + } + } + + /** Send preprocessor rules */ + private void sendPreprocessorRules(AgentConfiguration agentConfiguration) { + if (preprocessorRulesNeedUpdate.getAndSet(false)) { + String preprocessorRules = null; + if (agentConfiguration.getPreprocessorRules() != null) { + // reading rules from BE if sent from BE + preprocessorRules = agentConfiguration.getPreprocessorRules(); + } else { + // reading local file's rule + preprocessorRules = ProxyPreprocessorConfigManager.getProxyConfigRules(); + } + try { + JsonNode rulesNode = createRulesNode(preprocessorRules, agentConfiguration.getPreprocessorRulesId()); + apiContainer + .getProxyV2APIForTenant(APIContainer.CENTRAL_TENANT_NAME) + .proxySavePreprocessorRules( + proxyId, + rulesNode + ); + } catch (javax.ws.rs.NotFoundException ex) { + logger.warning("'proxySavePreprocessorRules' api end point not found"); + } + } + } + + private JsonNode createRulesNode(String preprocessorRules, String proxyId) { + Map fieldsMap = new HashMap<>(); + fieldsMap.put("proxyRules", preprocessorRules); + if (proxyConfig.getPreprocessorConfigFile() != null) fieldsMap.put("proxyRulesFilePath", proxyConfig.getPreprocessorConfigFile()); + if (proxyId != null) fieldsMap.put("proxyRulesId", proxyId); + + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.valueToTree(fieldsMap); + } + + /** + * Perform agent check-in and fetch configuration of the daemon from remote server. + * + * @return Fetched configuration map {tenant_name: config instance}. {@code null} if the + * configuration is invalid. + */ + private Map checkin() { + Map configurationList = Maps.newHashMap(); + JsonNode agentMetricsWorkingCopy; + synchronized (executor) { + if (agentMetrics == null) return null; + agentMetricsWorkingCopy = agentMetrics; + agentMetrics = null; + if (retries.incrementAndGet() > MAX_CHECKIN_ATTEMPTS) return null; + } + // MONIT-25479: check-in for central and multicasting tenants / clusters + Map multicastingTenantList = TokenManager.getMulticastingTenantList(); + // Initialize tenantName and multicastingTenantProxyConfig here to track current checking + // tenant for better exception handling message + String tenantName = APIContainer.CENTRAL_TENANT_NAME; + TenantInfo multicastingTenantProxyConfig = + multicastingTenantList.get(APIContainer.CENTRAL_TENANT_NAME); + try { + AgentConfiguration multicastingConfig; + for (Map.Entry multicastingTenantEntry : + multicastingTenantList.entrySet()) { + tenantName = multicastingTenantEntry.getKey(); + multicastingTenantProxyConfig = multicastingTenantEntry.getValue(); + logger.info("Checking in tenants: " + multicastingTenantProxyConfig.getWFServer()); + multicastingConfig = + apiContainer + .getProxyV2APIForTenant(tenantName) + .proxyCheckin( + proxyId, + "Bearer " + multicastingTenantProxyConfig.getBearerToken(), + proxyConfig.getHostname() + + (multicastingTenantList.size() > 1 ? "-multi_tenant" : ""), + proxyConfig.getProxyname(), + getBuildVersion(), + System.currentTimeMillis(), + agentMetricsWorkingCopy, + proxyConfig.isEphemeral()); + configurationList.put(tenantName, multicastingConfig); + } + agentMetricsWorkingCopy = null; + } catch (ClientErrorException ex) { + agentMetricsWorkingCopy = null; + switch (ex.getResponse().getStatus()) { + case 401: + checkinError( + "HTTP 401 Unauthorized: Please verify that your server and token settings" + + " are correct and that the token has Proxy Management permission!"); + if (successfulCheckIns.get() == 0) { + throw new RuntimeException("Aborting start-up"); + } + break; + case 403: + checkinError( + "HTTP 403 Forbidden: Please verify that your token has Proxy Management " + + "permission!"); + if (successfulCheckIns.get() == 0) { + throw new RuntimeException("Aborting start-up"); + } + break; + case 404: + case 405: + String serverUrl = multicastingTenantProxyConfig.getWFServer().replaceAll("/$", ""); + if (successfulCheckIns.get() == 0 && !retryImmediately && !serverUrl.endsWith("/api")) { + this.serverEndpointUrl = serverUrl + "/api/"; + checkinError( + "Possible server endpoint misconfiguration detected, attempting to use " + + serverEndpointUrl); + apiContainer.updateServerEndpointURL(tenantName, serverEndpointUrl); + retryImmediately = true; + return null; + } + String secondaryMessage = + serverUrl.endsWith("/api") + ? "Current setting: " + multicastingTenantProxyConfig.getWFServer() + : "Server endpoint URLs normally end with '/api/'. Current setting: " + + multicastingTenantProxyConfig.getBearerToken(); + checkinError( + "HTTP " + + ex.getResponse().getStatus() + + ": Misconfiguration detected, " + + "please verify that your server setting is correct. " + + secondaryMessage); + if (successfulCheckIns.get() == 0) { + throw new RuntimeException("Aborting start-up"); + } + break; + case 407: + checkinError( + "HTTP 407 Proxy Authentication Required: Please verify that " + + "proxyUser and proxyPassword settings are correct and make sure your HTTP proxy" + + " is not rate limiting!"); + if (successfulCheckIns.get() == 0) { + throw new RuntimeException("Aborting start-up"); + } + break; + case 429: + // 429s are retried silently. + return null; + default: + checkinError( + "HTTP " + + ex.getResponse().getStatus() + + " error: Unable to check in with Wavefront! " + + multicastingTenantProxyConfig.getWFServer() + + ": " + + Throwables.getRootCause(ex).getMessage()); + } + return Maps.newHashMap(); // return empty configuration to prevent checking in every 1s + } catch (ProcessingException ex) { + Throwable rootCause = Throwables.getRootCause(ex); + if (rootCause instanceof UnknownHostException) { + checkinError( + "Unknown host: " + + multicastingTenantProxyConfig.getWFServer() + + ". Please verify your DNS and network settings!"); + return null; + } + if (rootCause instanceof ConnectException) { + checkinError( + "Unable to connect to " + + multicastingTenantProxyConfig.getWFServer() + + ": " + + rootCause.getMessage() + + " Please verify your network/firewall settings!"); + return null; + } + if (rootCause instanceof SocketTimeoutException) { + checkinError( + "Unable to check in with " + + multicastingTenantProxyConfig.getWFServer() + + ": " + + rootCause.getMessage() + + " Please verify your network/firewall settings!"); + return null; + } + checkinError( + "Request processing error: Unable to retrieve proxy configuration! " + + multicastingTenantProxyConfig.getWFServer() + + ": " + + rootCause); + return null; + } catch (Exception ex) { + checkinError( + "Unable to retrieve proxy configuration from remote server! " + + multicastingTenantProxyConfig.getWFServer() + + ": " + + Throwables.getRootCause(ex)); + return null; + } finally { + synchronized (executor) { + // if check-in process failed (agentMetricsWorkingCopy is not null) and agent + // metrics have + // not been updated yet, restore last known set of agent metrics to be retried + if (agentMetricsWorkingCopy != null && agentMetrics == null) { + agentMetrics = agentMetricsWorkingCopy; + } + } + } + if (configurationList.get(APIContainer.CENTRAL_TENANT_NAME).currentTime != null) { + Clock.set(configurationList.get(APIContainer.CENTRAL_TENANT_NAME).currentTime); + } + + // Always update the log server url / token in case they've changed + String logServerIngestionURL = + configurationList.get(APIContainer.CENTRAL_TENANT_NAME).getLogServerEndpointUrl(); + String logServerIngestionToken = + configurationList.get(APIContainer.CENTRAL_TENANT_NAME).getLogServerToken(); + + // MONIT-33770 - For converged CSP tenants, a user needs to provide vRLIC Ingestion token & + // URL as proxy configuration. + String WARNING_MSG = "Missing either logServerIngestionToken/logServerIngestionURL or both."; + if (StringUtils.isBlank(logServerIngestionURL) + && StringUtils.isBlank(logServerIngestionToken)) { + ValidationConfiguration validationConfiguration = + configurationList.get(APIContainer.CENTRAL_TENANT_NAME).getValidationConfiguration(); + if (validationConfiguration != null + && validationConfiguration.enableHyperlogsConvergedCsp()) { + proxyConfig.setEnableHyperlogsConvergedCsp(true); + logServerIngestionURL = proxyConfig.getLogServerIngestionURL(); + logServerIngestionToken = proxyConfig.getLogServerIngestionToken(); + if (StringUtils.isBlank(logServerIngestionURL) + || StringUtils.isBlank(logServerIngestionToken)) { + proxyConfig.setReceivedLogServerDetails(false); + logger.severe( + WARNING_MSG + + " To ingest logs to the log server, please provide " + + "logServerIngestionToken & logServerIngestionURL in the proxy configuration."); + } + } + } else if (StringUtils.isBlank(logServerIngestionURL) + || StringUtils.isBlank(logServerIngestionToken)) { + logger.severe( + WARNING_MSG + + " Proxy will not be ingesting data to the log server as it did " + + "not receive at least one of the values during check-in."); + } + + apiContainer.updateLogServerEndpointURLandToken(logServerIngestionURL, logServerIngestionToken); + + return configurationList; + } + + @VisibleForTesting + void updateConfiguration() { + try { + Map configList = checkin(); + if (configList != null && !configList.isEmpty()) { + AgentConfiguration config; + for (Map.Entry configEntry : configList.entrySet()) { + config = configEntry.getValue(); + // For shutdown the proxy / truncate queue, only check the central tenant's flag + if (config == null) { + continue; + } + if (configEntry.getKey().equals(APIContainer.CENTRAL_TENANT_NAME)) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("Server configuration getShutOffAgents: " + config.getShutOffAgents()); + logger.fine("Server configuration isTruncateQueue: " + config.isTruncateQueue()); + } + if (config.getShutOffAgents()) { + logger.severe( + firstNonNull( + config.getShutOffMessage(), + "Shutting down: Server side flag indicating proxy has to shut down.")); + shutdownHook.run(); + } else if (config.isTruncateQueue()) { + logger.severe( + "Truncating queue: Server side flag indicating proxy queue has to be truncated."); + truncateBacklog.run(); + } + } + agentConfigurationConsumer.accept(configEntry.getKey(), config); + + // Check if preprocessor rules were set on server side + String checkPreprocessorRules = config.getPreprocessorRules(); + if (checkPreprocessorRules != null && !checkPreprocessorRules.isEmpty()) { + AgentConfiguration finalConfig = config; + logger.log(Level.INFO, () -> String.format("New preprocessor rules detected during checkin. Setting new preprocessor rule %s", + (finalConfig.getPreprocessorRulesId() != null && !finalConfig.getPreprocessorRulesId().isEmpty()) ? finalConfig.getPreprocessorRulesId() : "")); + // future implementation, can send timestamp through AgentConfig and skip reloading if rule unchanged + isRulesSetInFE.set(true); + // indicates will need to sendPreprocessorRules() + preprocessorRulesNeedUpdate.set(true); + } else { + // was previously reading from BE + if (isRulesSetInFE.get()) { + if (proxyConfig.getPreprocessorConfigFile() == null || proxyConfig.getPreprocessorConfigFile().isEmpty()) { + logger.info("No preprocessor rules detected during checkin, and no rules file found."); + } else { + logger.log(Level.INFO, () -> String.format("Reverting back to reading rules from file %s", proxyConfig.getPreprocessorConfigFile())); + } + // indicates that previously read from BE, now switching back to reading from file. + isRulesSetInFE.set(false); + preprocessorRulesNeedUpdate.set(true); + } + } + // will always send to BE in order to update Agent with latest rule + sendPreprocessorRules(config); + } + } + } catch (Exception e) { + logger.log(Level.SEVERE, "Exception occurred during configuration update", e); + } + } + + @VisibleForTesting + void updateProxyMetrics() { + try { + Map pointTags = new HashMap<>(proxyConfig.getAgentMetricsPointTags()); + pointTags.put("processId", ID); + pointTags.put("hostname", proxyConfig.getHostname()); + synchronized (executor) { + agentMetrics = + JsonMetricsGenerator.generateJsonMetrics( + Metrics.defaultRegistry(), true, true, true, pointTags, null); + retries.set(0); + } + } catch (Exception ex) { + logger.log(Level.SEVERE, "Could not generate proxy metrics", ex); + } + } + + private void checkinError(String errMsg) { + if (successfulCheckIns.get() == 0) logger.severe(Strings.repeat("*", errMsg.length())); + logger.severe(errMsg); + if (successfulCheckIns.get() == 0) logger.severe(Strings.repeat("*", errMsg.length())); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/ProxyConfig.java b/proxy/src/main/java/com/wavefront/agent/ProxyConfig.java new file mode 100644 index 000000000..2247e19f3 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/ProxyConfig.java @@ -0,0 +1,1509 @@ +package com.wavefront.agent; + +import static com.wavefront.agent.api.APIContainer.CENTRAL_TENANT_NAME; +import static com.wavefront.agent.config.ReportableConfig.reportGauge; +import static com.wavefront.agent.data.EntityProperties.*; +import static com.wavefront.common.Utils.getBuildVersion; +import static com.wavefront.common.Utils.getLocalHostName; +import static io.opentracing.tag.Tags.SPAN_KIND; + +import com.beust.jcommander.IStringConverter; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.wavefront.agent.api.APIContainer; +import com.wavefront.agent.auth.TokenValidationMethod; +import com.wavefront.agent.config.Categories; +import com.wavefront.agent.config.ProxyConfigOption; +import com.wavefront.agent.config.ReportableConfig; +import com.wavefront.agent.config.SubCategories; +import com.wavefront.agent.data.TaskQueueLevel; +import com.wavefront.common.TaggedMetricName; +import com.wavefront.common.TimeProvider; +import com.yammer.metrics.core.MetricName; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.*; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.jetbrains.annotations.NotNull; + +/** + * Proxy configuration (refactored from {@link com.wavefront.agent.AbstractAgent}). + * + * @author vasily@wavefront.com + */ +@SuppressWarnings("CanBeFinal") +public class ProxyConfig extends ProxyConfigDef { + static final int GRAPHITE_LISTENING_PORT = 2878; + private static final Logger logger = Logger.getLogger(ProxyConfig.class.getCanonicalName()); + private static final double MAX_RETRY_BACKOFF_BASE_SECONDS = 60.0; + @VisibleForTesting public static final Integer NUMBER_OF_VISIBLE_DIGITS = 4; + private final List modifyByArgs = new ArrayList<>(); + private final List modifyByFile = new ArrayList<>(); + + TimeProvider timeProvider = System::currentTimeMillis; + + public String getCSPBaseUrl() { + return cspBaseUrl; + } + + public boolean isHelp() { + return help; + } + + public boolean isVersion() { + return version; + } + + public String getPrefix() { + return prefix; + } + + public boolean isTestLogs() { + return testLogs; + } + + public String getTestPreprocessorForPort() { + return testPreprocessorForPort; + } + + public String getTestSpanPreprocessorForPort() { + return testSpanPreprocessorForPort; + } + + public String getServer() { + return server; + } + + public String getBufferFile() { + return bufferFile; + } + + public int getBufferShardSize() { + return bufferShardSize; + } + + public boolean isDisableBufferSharding() { + return disableBufferSharding; + } + + public boolean isSqsQueueBuffer() { + return sqsQueueBuffer; + } + + public String getSqsQueueNameTemplate() { + return sqsQueueNameTemplate; + } + + public String getSqsQueueRegion() { + return sqsQueueRegion; + } + + public String getSqsQueueIdentifier() { + return sqsQueueIdentifier; + } + + public TaskQueueLevel getTaskQueueLevel() { + return taskQueueLevel; + } + + public String getExportQueuePorts() { + return exportQueuePorts; + } + + public String getExportQueueOutputFile() { + return exportQueueOutputFile; + } + + public boolean isExportQueueRetainData() { + return exportQueueRetainData; + } + + public boolean isUseNoopSender() { + return useNoopSender; + } + + public int getFlushThreads() { + return flushThreads; + } + + public int getFlushThreadsSourceTags() { + return flushThreadsSourceTags; + } + + public int getFlushThreadsEvents() { + return flushThreadsEvents; + } + + public int getFlushThreadsLogs() { + return flushThreadsLogs; + } + + public int getPushFlushIntervalLogs() { + return pushFlushIntervalLogs; + } + + public boolean isPurgeBuffer() { + return purgeBuffer; + } + + public int getPushFlushInterval() { + return pushFlushInterval; + } + + public int getPushFlushMaxPoints() { + return pushFlushMaxPoints; + } + + public int getPushFlushMaxHistograms() { + return pushFlushMaxHistograms; + } + + public int getPushFlushMaxSourceTags() { + return pushFlushMaxSourceTags; + } + + public int getPushFlushMaxSpans() { + return pushFlushMaxSpans; + } + + public int getPushFlushMaxSpanLogs() { + return pushFlushMaxSpanLogs; + } + + public int getPushFlushMaxEvents() { + return pushFlushMaxEvents; + } + + public int getPushFlushMaxLogs() { + return pushFlushMaxLogs; + } + + public double getPushRateLimit() { + return pushRateLimit; + } + + public double getPushRateLimitHistograms() { + return pushRateLimitHistograms; + } + + public double getPushRateLimitSourceTags() { + return pushRateLimitSourceTags; + } + + public double getPushRateLimitSpans() { + return pushRateLimitSpans; + } + + public double getPushRateLimitSpanLogs() { + return pushRateLimitSpanLogs; + } + + public double getPushRateLimitEvents() { + return pushRateLimitEvents; + } + + public double getPushRateLimitLogs() { + return pushRateLimitLogs; + } + + public int getPushRateLimitMaxBurstSeconds() { + return pushRateLimitMaxBurstSeconds; + } + + public int getPushMemoryBufferLimit() { + return pushMemoryBufferLimit; + } + + public int getPushMemoryBufferLimitLogs() { + return pushMemoryBufferLimitLogs; + } + + public int getPushBlockedSamples() { + return pushBlockedSamples; + } + + public String getBlockedPointsLoggerName() { + return blockedPointsLoggerName; + } + + public String getBlockedHistogramsLoggerName() { + return blockedHistogramsLoggerName; + } + + public String getBlockedSpansLoggerName() { + return blockedSpansLoggerName; + } + + public String getBlockedLogsLoggerName() { + return blockedLogsLoggerName; + } + + public String getPushListenerPorts() { + return pushListenerPorts; + } + + public int getPushListenerMaxReceivedLength() { + return pushListenerMaxReceivedLength; + } + + public int getPushListenerHttpBufferSize() { + return pushListenerHttpBufferSize; + } + + public int getTraceListenerMaxReceivedLength() { + return traceListenerMaxReceivedLength; + } + + public int getTraceListenerHttpBufferSize() { + return traceListenerHttpBufferSize; + } + + public int getListenerIdleConnectionTimeout() { + return listenerIdleConnectionTimeout; + } + + public int getMemGuardFlushThreshold() { + return memGuardFlushThreshold; + } + + public boolean isHistogramPassthroughRecompression() { + return histogramPassthroughRecompression; + } + + public String getHistogramStateDirectory() { + return histogramStateDirectory; + } + + public long getHistogramAccumulatorResolveInterval() { + return histogramAccumulatorResolveInterval; + } + + public long getHistogramAccumulatorFlushInterval() { + return histogramAccumulatorFlushInterval; + } + + public int getHistogramAccumulatorFlushMaxBatchSize() { + return histogramAccumulatorFlushMaxBatchSize; + } + + public int getHistogramMaxReceivedLength() { + return histogramMaxReceivedLength; + } + + public int getHistogramHttpBufferSize() { + return histogramHttpBufferSize; + } + + public String getHistogramMinuteListenerPorts() { + return histogramMinuteListenerPorts; + } + + public int getHistogramMinuteFlushSecs() { + return histogramMinuteFlushSecs; + } + + public short getHistogramMinuteCompression() { + return histogramMinuteCompression; + } + + public int getHistogramMinuteAvgKeyBytes() { + return histogramMinuteAvgKeyBytes; + } + + public int getHistogramMinuteAvgDigestBytes() { + return histogramMinuteAvgDigestBytes; + } + + public long getHistogramMinuteAccumulatorSize() { + return histogramMinuteAccumulatorSize; + } + + public boolean isHistogramMinuteAccumulatorPersisted() { + return histogramMinuteAccumulatorPersisted; + } + + public boolean isHistogramMinuteMemoryCache() { + return histogramMinuteMemoryCache; + } + + public String getHistogramHourListenerPorts() { + return histogramHourListenerPorts; + } + + public int getHistogramHourFlushSecs() { + return histogramHourFlushSecs; + } + + public short getHistogramHourCompression() { + return histogramHourCompression; + } + + public int getHistogramHourAvgKeyBytes() { + return histogramHourAvgKeyBytes; + } + + public int getHistogramHourAvgDigestBytes() { + return histogramHourAvgDigestBytes; + } + + public long getHistogramHourAccumulatorSize() { + return histogramHourAccumulatorSize; + } + + public boolean isHistogramHourAccumulatorPersisted() { + return histogramHourAccumulatorPersisted; + } + + public boolean isHistogramHourMemoryCache() { + return histogramHourMemoryCache; + } + + public String getHistogramDayListenerPorts() { + return histogramDayListenerPorts; + } + + public int getHistogramDayFlushSecs() { + return histogramDayFlushSecs; + } + + public short getHistogramDayCompression() { + return histogramDayCompression; + } + + public int getHistogramDayAvgKeyBytes() { + return histogramDayAvgKeyBytes; + } + + public int getHistogramDayAvgDigestBytes() { + return histogramDayAvgDigestBytes; + } + + public long getHistogramDayAccumulatorSize() { + return histogramDayAccumulatorSize; + } + + public boolean isHistogramDayAccumulatorPersisted() { + return histogramDayAccumulatorPersisted; + } + + public boolean isHistogramDayMemoryCache() { + return histogramDayMemoryCache; + } + + public String getHistogramDistListenerPorts() { + return histogramDistListenerPorts; + } + + public int getHistogramDistFlushSecs() { + return histogramDistFlushSecs; + } + + public short getHistogramDistCompression() { + return histogramDistCompression; + } + + public int getHistogramDistAvgKeyBytes() { + return histogramDistAvgKeyBytes; + } + + public int getHistogramDistAvgDigestBytes() { + return histogramDistAvgDigestBytes; + } + + public long getHistogramDistAccumulatorSize() { + return histogramDistAccumulatorSize; + } + + public boolean isHistogramDistAccumulatorPersisted() { + return histogramDistAccumulatorPersisted; + } + + public boolean isHistogramDistMemoryCache() { + return histogramDistMemoryCache; + } + + public String getGraphitePorts() { + return graphitePorts; + } + + public String getGraphiteFormat() { + return graphiteFormat; + } + + public String getGraphiteDelimiters() { + return graphiteDelimiters; + } + + public String getGraphiteFieldsToRemove() { + return graphiteFieldsToRemove; + } + + public String getJsonListenerPorts() { + return jsonListenerPorts; + } + + public String getDataDogJsonPorts() { + return dataDogJsonPorts; + } + + public String getDataDogRequestRelayTarget() { + return dataDogRequestRelayTarget; + } + + public int getDataDogRequestRelayAsyncThreads() { + return dataDogRequestRelayAsyncThreads; + } + + public boolean isDataDogRequestRelaySyncMode() { + return dataDogRequestRelaySyncMode; + } + + public boolean isDataDogProcessSystemMetrics() { + return dataDogProcessSystemMetrics; + } + + public boolean isDataDogProcessServiceChecks() { + return dataDogProcessServiceChecks; + } + + public String getWriteHttpJsonListenerPorts() { + return writeHttpJsonListenerPorts; + } + + public String getOtlpGrpcListenerPorts() { + return otlpGrpcListenerPorts; + } + + public String getOtlpHttpListenerPorts() { + return otlpHttpListenerPorts; + } + + public boolean isOtlpResourceAttrsOnMetricsIncluded() { + return otlpResourceAttrsOnMetricsIncluded; + } + + public boolean isOtlpAppTagsOnMetricsIncluded() { + return otlpAppTagsOnMetricsIncluded; + } + + public int getFilebeatPort() { + return filebeatPort; + } + + public int getRawLogsPort() { + return rawLogsPort; + } + + public int getRawLogsMaxReceivedLength() { + return rawLogsMaxReceivedLength; + } + + public int getRawLogsHttpBufferSize() { + return rawLogsHttpBufferSize; + } + + public String getLogsIngestionConfigFile() { + return logsIngestionConfigFile; + } + + public String getHostname() { + return hostname; + } + + public String getProxyname() { + return proxyname; + } + + public String getIdFile() { + return idFile; + } + + public String getAllowRegex() { + return allowRegex; + } + + public String getBlockRegex() { + return blockRegex; + } + + public String getOpentsdbPorts() { + return opentsdbPorts; + } + + public String getOpentsdbAllowRegex() { + return opentsdbAllowRegex; + } + + public String getOpentsdbBlockRegex() { + return opentsdbBlockRegex; + } + + public String getPicklePorts() { + return picklePorts; + } + + public String getTraceListenerPorts() { + return traceListenerPorts; + } + + public String getTraceJaegerListenerPorts() { + return traceJaegerListenerPorts; + } + + public String getTraceJaegerHttpListenerPorts() { + return traceJaegerHttpListenerPorts; + } + + public String getTraceJaegerGrpcListenerPorts() { + return traceJaegerGrpcListenerPorts; + } + + public String getTraceJaegerApplicationName() { + return traceJaegerApplicationName; + } + + public String getTraceZipkinListenerPorts() { + return traceZipkinListenerPorts; + } + + public String getTraceZipkinApplicationName() { + return traceZipkinApplicationName; + } + + public String getCustomTracingListenerPorts() { + return customTracingListenerPorts; + } + + public String getCustomTracingApplicationName() { + return customTracingApplicationName; + } + + public String getCustomTracingServiceName() { + return customTracingServiceName; + } + + public double getTraceSamplingRate() { + return traceSamplingRate; + } + + public int getTraceSamplingDuration() { + return traceSamplingDuration; + } + + public Set getTraceDerivedCustomTagKeys() { + Set customTagKeys = + new HashSet<>( + Splitter.on(",") + .trimResults() + .omitEmptyStrings() + .splitToList(ObjectUtils.firstNonNull(traceDerivedCustomTagKeys, ""))); + customTagKeys.add(SPAN_KIND.getKey()); // add span.kind tag by default + return customTagKeys; + } + + public boolean isBackendSpanHeadSamplingPercentIgnored() { + return backendSpanHeadSamplingPercentIgnored; + } + + public String getPushRelayListenerPorts() { + return pushRelayListenerPorts; + } + + public boolean isPushRelayHistogramAggregator() { + return pushRelayHistogramAggregator; + } + + public long getPushRelayHistogramAggregatorAccumulatorSize() { + return pushRelayHistogramAggregatorAccumulatorSize; + } + + public int getPushRelayHistogramAggregatorFlushSecs() { + return pushRelayHistogramAggregatorFlushSecs; + } + + public short getPushRelayHistogramAggregatorCompression() { + return pushRelayHistogramAggregatorCompression; + } + + public boolean isSplitPushWhenRateLimited() { + return splitPushWhenRateLimited; + } + + public double getRetryBackoffBaseSeconds() { + return retryBackoffBaseSeconds; + } + + public List getCustomSourceTags() { + // create List of custom tags from the configuration string + Set tagSet = new LinkedHashSet<>(); + Splitter.on(",") + .trimResults() + .omitEmptyStrings() + .split(customSourceTags) + .forEach( + x -> { + if (!tagSet.add(x)) { + logger.warning( + "Duplicate tag " + x + " specified in customSourceTags config setting"); + } + }); + return new ArrayList<>(tagSet); + } + + public List getCustomTimestampTags() { + // create List of timestamp tags from the configuration string + Set tagSet = new LinkedHashSet<>(); + Splitter.on(",") + .trimResults() + .omitEmptyStrings() + .split(customTimestampTags) + .forEach( + x -> { + if (!tagSet.add(x)) { + logger.warning( + "Duplicate tag " + x + " specified in customTimestampTags config setting"); + } + }); + return new ArrayList<>(tagSet); + } + + public List getCustomMessageTags() { + // create List of message tags from the configuration string + Set tagSet = new LinkedHashSet<>(); + Splitter.on(",") + .trimResults() + .omitEmptyStrings() + .split(customMessageTags) + .forEach( + x -> { + if (!tagSet.add(x)) { + logger.warning( + "Duplicate tag " + x + " specified in customMessageTags config setting"); + } + }); + return new ArrayList<>(tagSet); + } + + public List getCustomApplicationTags() { + // create List of application tags from the configuration string + Set tagSet = new LinkedHashSet<>(); + Splitter.on(",") + .trimResults() + .omitEmptyStrings() + .split(customApplicationTags) + .forEach( + x -> { + if (!tagSet.add(x)) { + logger.warning( + "Duplicate tag " + x + " specified in customApplicationTags config setting"); + } + }); + return new ArrayList<>(tagSet); + } + + public List getCustomServiceTags() { + // create List of service tags from the configuration string + Set tagSet = new LinkedHashSet<>(); + Splitter.on(",") + .trimResults() + .omitEmptyStrings() + .split(customServiceTags) + .forEach( + x -> { + if (!tagSet.add(x)) { + logger.warning( + "Duplicate tag " + x + " specified in customServiceTags config setting"); + } + }); + return new ArrayList<>(tagSet); + } + + public List getCustomExceptionTags() { + Set tagSet = new LinkedHashSet<>(); + Splitter.on(",") + .trimResults() + .omitEmptyStrings() + .split(customExceptionTags) + .forEach( + x -> { + if (!tagSet.add(x)) { + logger.warning( + "Duplicate tag " + x + " specified in customExceptionTags config setting"); + } + }); + return new ArrayList<>(tagSet); + } + + public List getCustomLevelTags() { + // create List of level tags from the configuration string + Set tagSet = new LinkedHashSet<>(); + Splitter.on(",") + .trimResults() + .omitEmptyStrings() + .split(customLevelTags) + .forEach( + x -> { + if (!tagSet.add(x)) { + logger.warning( + "Duplicate tag " + x + " specified in customLevelTags config setting"); + } + }); + return new ArrayList<>(tagSet); + } + + public Map getAgentMetricsPointTags() { + //noinspection UnstableApiUsage + return agentMetricsPointTags == null + ? Collections.emptyMap() + : Splitter.on(",") + .trimResults() + .omitEmptyStrings() + .withKeyValueSeparator("=") + .split(agentMetricsPointTags); + } + + public boolean isEphemeral() { + return ephemeral; + } + + public boolean isDisableRdnsLookup() { + return disableRdnsLookup; + } + + public boolean isGzipCompression() { + return gzipCompression; + } + + public int getGzipCompressionLevel() { + return gzipCompressionLevel; + } + + public int getSoLingerTime() { + return soLingerTime; + } + + public String getProxyHost() { + return proxyHost; + } + + public int getProxyPort() { + return proxyPort; + } + + public String getProxyUser() { + return proxyUser; + } + + public String getProxyPassword() { + return proxyPassword; + } + + public String getHttpUserAgent() { + return httpUserAgent; + } + + public int getHttpConnectTimeout() { + return httpConnectTimeout; + } + + public int getHttpRequestTimeout() { + return httpRequestTimeout; + } + + public int getHttpMaxConnTotal() { + return httpMaxConnTotal; + } + + public int getHttpMaxConnPerRoute() { + return httpMaxConnPerRoute; + } + + public int getHttpAutoRetries() { + return httpAutoRetries; + } + + public String getPreprocessorConfigFile() { + return preprocessorConfigFile; + } + + public int getDataBackfillCutoffHours() { + return dataBackfillCutoffHours; + } + + public int getDataPrefillCutoffHours() { + return dataPrefillCutoffHours; + } + + public TokenValidationMethod getAuthMethod() { + return authMethod; + } + + public String getAuthTokenIntrospectionServiceUrl() { + return authTokenIntrospectionServiceUrl; + } + + public String getAuthTokenIntrospectionAuthorizationHeader() { + return authTokenIntrospectionAuthorizationHeader; + } + + public int getAuthResponseRefreshInterval() { + return authResponseRefreshInterval; + } + + public int getAuthResponseMaxTtl() { + return authResponseMaxTtl; + } + + public String getAuthStaticToken() { + return authStaticToken; + } + + public int getAdminApiListenerPort() { + return adminApiListenerPort; + } + + public String getAdminApiRemoteIpAllowRegex() { + return adminApiRemoteIpAllowRegex; + } + + public String getHttpHealthCheckPorts() { + return httpHealthCheckPorts; + } + + public boolean isHttpHealthCheckAllPorts() { + return httpHealthCheckAllPorts; + } + + public String getHttpHealthCheckPath() { + return httpHealthCheckPath; + } + + public String getHttpHealthCheckResponseContentType() { + return httpHealthCheckResponseContentType; + } + + public int getHttpHealthCheckPassStatusCode() { + return httpHealthCheckPassStatusCode; + } + + public String getHttpHealthCheckPassResponseBody() { + return httpHealthCheckPassResponseBody; + } + + public int getHttpHealthCheckFailStatusCode() { + return httpHealthCheckFailStatusCode; + } + + public String getHttpHealthCheckFailResponseBody() { + return httpHealthCheckFailResponseBody; + } + + public long getDeltaCountersAggregationIntervalSeconds() { + return deltaCountersAggregationIntervalSeconds; + } + + public String getDeltaCountersAggregationListenerPorts() { + return deltaCountersAggregationListenerPorts; + } + + @JsonIgnore + public TimeProvider getTimeProvider() { + return timeProvider; + } + + public String getPrivateCertPath() { + return privateCertPath; + } + + public String getPrivateKeyPath() { + return privateKeyPath; + } + + public String getTlsPorts() { + return tlsPorts; + } + + public boolean isTrafficShaping() { + return trafficShaping; + } + + public int getTrafficShapingWindowSeconds() { + return trafficShapingWindowSeconds; + } + + public double getTrafficShapingHeadroom() { + return trafficShapingHeadroom; + } + + public List getCorsEnabledPorts() { + return Splitter.on(",").trimResults().omitEmptyStrings().splitToList(corsEnabledPorts); + } + + public List getCorsOrigin() { + return Splitter.on(",").trimResults().omitEmptyStrings().splitToList(corsOrigin); + } + + public boolean isCorsAllowNullOrigin() { + return corsAllowNullOrigin; + } + + public String getLogServerIngestionToken() { + return logServerIngestionToken; + } + + public String getLogServerIngestionURL() { + return logServerIngestionURL; + } + + public boolean enableHyperlogsConvergedCsp() { + return enableHyperlogsConvergedCsp; + } + + public void setEnableHyperlogsConvergedCsp(boolean enableHyperlogsConvergedCsp) { + this.enableHyperlogsConvergedCsp = enableHyperlogsConvergedCsp; + } + + public boolean receivedLogServerDetails() { + return receivedLogServerDetails; + } + + public void setReceivedLogServerDetails(boolean receivedLogServerDetails) { + this.receivedLogServerDetails = receivedLogServerDetails; + } + + @Override + public void verifyAndInit() { + throw new UnsupportedOperationException("not implemented"); + } + + // TODO: review this options that are only available on the config file. + private void configFileExtraArguments(ReportableConfig config) { + // Multicasting configurations + int multicastingTenants = Integer.parseInt(config.getProperty("multicastingTenants", "0")); + for (int i = 1; i <= multicastingTenants; i++) { + String tenantName = config.getProperty(String.format("multicastingTenantName_%d", i), ""); + if (tenantName.equals(APIContainer.CENTRAL_TENANT_NAME)) { + throw new IllegalArgumentException( + "Error in multicasting endpoints initiation: " + + "\"central\" is the reserved tenant name."); + } + String tenantServer = config.getProperty(String.format("multicastingServer_%d", i), ""); + String tenantToken = config.getProperty(String.format("multicastingToken_%d", i), ""); + String tenantCSPAppId = config.getProperty(String.format("multicastingCSPAppId_%d", i), ""); + String tenantCSPAppSecret = + config.getProperty(String.format("multicastingCSPAppSecret_%d", i), ""); + String tenantCSPOrgId = config.getProperty(String.format("multicastingCSPOrgId_%d", i), ""); + String tenantCSPAPIToken = + config.getProperty(String.format("multicastingCSPAPIToken_%d", i), ""); + + // Based on the setup parameters, the pertinent tenant information object will be produced + // using the proper proxy + // authentication technique. + constructTenantInfoObject( + tenantCSPAppId, + tenantCSPAppSecret, + tenantCSPOrgId, + tenantCSPAPIToken, + tenantToken, + tenantServer, + tenantName); + } + + if (config.isDefined("avgHistogramKeyBytes")) { + histogramMinuteAvgKeyBytes = + histogramHourAvgKeyBytes = + histogramDayAvgKeyBytes = + histogramDistAvgKeyBytes = config.getInteger("avgHistogramKeyBytes", 150); + } + + if (config.isDefined("avgHistogramDigestBytes")) { + histogramMinuteAvgDigestBytes = + histogramHourAvgDigestBytes = + histogramDayAvgDigestBytes = + histogramDistAvgDigestBytes = config.getInteger("avgHistogramDigestBytes", 500); + } + if (config.isDefined("histogramAccumulatorSize")) { + histogramMinuteAccumulatorSize = + histogramHourAccumulatorSize = + histogramDayAccumulatorSize = + histogramDistAccumulatorSize = config.getLong("histogramAccumulatorSize", 100000); + } + if (config.isDefined("histogramCompression")) { + histogramMinuteCompression = + histogramHourCompression = + histogramDayCompression = + histogramDistCompression = + config.getNumber("histogramCompression", null, 20, 1000).shortValue(); + } + if (config.isDefined("persistAccumulator")) { + histogramMinuteAccumulatorPersisted = + histogramHourAccumulatorPersisted = + histogramDayAccumulatorPersisted = + histogramDistAccumulatorPersisted = + config.getBoolean("persistAccumulator", false); + } + + histogramMinuteCompression = + config + .getNumber("histogramMinuteCompression", histogramMinuteCompression, 20, 1000) + .shortValue(); + histogramMinuteAvgDigestBytes = 32 + histogramMinuteCompression * 7; + + histogramHourCompression = + config + .getNumber("histogramHourCompression", histogramHourCompression, 20, 1000) + .shortValue(); + histogramHourAvgDigestBytes = 32 + histogramHourCompression * 7; + + histogramDayCompression = + config.getNumber("histogramDayCompression", histogramDayCompression, 20, 1000).shortValue(); + histogramDayAvgDigestBytes = 32 + histogramDayCompression * 7; + + histogramDistCompression = + config + .getNumber("histogramDistCompression", histogramDistCompression, 20, 1000) + .shortValue(); + histogramDistAvgDigestBytes = 32 + histogramDistCompression * 7; + + proxyPassword = config.getString("proxyPassword", proxyPassword, s -> ""); + httpMaxConnTotal = Math.min(200, config.getInteger("httpMaxConnTotal", httpMaxConnTotal)); + httpMaxConnPerRoute = + Math.min(100, config.getInteger("httpMaxConnPerRoute", httpMaxConnPerRoute)); + gzipCompressionLevel = + config.getNumber("gzipCompressionLevel", gzipCompressionLevel, 1, 9).intValue(); + + // clamp values for pushFlushMaxPoints/etc between min split size + // (or 1 in case of source tags and events) and default batch size. + // also make sure it is never higher than the configured rate limit. + pushFlushMaxPoints = + Math.max( + Math.min( + Math.min( + config.getInteger("pushFlushMaxPoints", pushFlushMaxPoints), + DEFAULT_BATCH_SIZE), + (int) pushRateLimit), + DEFAULT_MIN_SPLIT_BATCH_SIZE); + pushFlushMaxHistograms = + Math.max( + Math.min( + Math.min( + config.getInteger("pushFlushMaxHistograms", pushFlushMaxHistograms), + DEFAULT_BATCH_SIZE_HISTOGRAMS), + (int) pushRateLimitHistograms), + DEFAULT_MIN_SPLIT_BATCH_SIZE); + pushFlushMaxSourceTags = + Math.max( + Math.min( + Math.min( + config.getInteger("pushFlushMaxSourceTags", pushFlushMaxSourceTags), + DEFAULT_BATCH_SIZE_SOURCE_TAGS), + (int) pushRateLimitSourceTags), + 1); + pushFlushMaxSpans = + Math.max( + Math.min( + Math.min( + config.getInteger("pushFlushMaxSpans", pushFlushMaxSpans), + DEFAULT_BATCH_SIZE_SPANS), + (int) pushRateLimitSpans), + DEFAULT_MIN_SPLIT_BATCH_SIZE); + pushFlushMaxSpanLogs = + Math.max( + Math.min( + Math.min( + config.getInteger("pushFlushMaxSpanLogs", pushFlushMaxSpanLogs), + DEFAULT_BATCH_SIZE_SPAN_LOGS), + (int) pushRateLimitSpanLogs), + DEFAULT_MIN_SPLIT_BATCH_SIZE); + pushFlushMaxEvents = + Math.min( + Math.min( + Math.max(config.getInteger("pushFlushMaxEvents", pushFlushMaxEvents), 1), + DEFAULT_BATCH_SIZE_EVENTS), + (int) (pushRateLimitEvents + 1)); + + pushFlushMaxLogs = + Math.max( + Math.min( + Math.min( + config.getInteger("pushFlushMaxLogs", pushFlushMaxLogs), + MAX_BATCH_SIZE_LOGS_PAYLOAD), + (int) pushRateLimitLogs), + DEFAULT_MIN_SPLIT_BATCH_SIZE_LOGS_PAYLOAD); + pushMemoryBufferLimitLogs = + Math.max( + config.getInteger("pushMemoryBufferLimitLogs", pushMemoryBufferLimitLogs), + pushFlushMaxLogs); + + pushMemoryBufferLimit = + Math.max( + config.getInteger("pushMemoryBufferLimit", pushMemoryBufferLimit), pushFlushMaxPoints); + retryBackoffBaseSeconds = + Math.max( + Math.min( + config.getDouble("retryBackoffBaseSeconds", retryBackoffBaseSeconds), + MAX_RETRY_BACKOFF_BASE_SECONDS), + 1.0); + } + + /** + * Parse commandline arguments into {@link ProxyConfig} object. + * + * @param args arguments to parse + * @param programName program name (to display help) + * @return true if proxy should continue, false if proxy should terminate. + * @throws ParameterException if configuration parsing failed + */ + public boolean parseArguments(String[] args, String programName) throws ParameterException { + JCommander jc = + JCommander.newBuilder() + .programName(programName) + .addObject(this) + .allowParameterOverwriting(true) + .acceptUnknownOptions(true) + .build(); + + // Command line arguments + jc.parse(args); + + if (this.isVersion()) { + return false; + } + if (this.isHelp()) { + jc.usage(); + return false; + } + + detectModifiedOptions(Arrays.stream(args).filter(s -> s.startsWith("-")), modifyByArgs); + String argsStr = + modifyByArgs.stream().map(field -> field.getName()).collect(Collectors.joining(", ")); + logger.info("modifyByArgs: " + argsStr); + + // Config file + if (pushConfigFile != null) { + ReportableConfig confFile = new ReportableConfig(); + List fileArgs = new ArrayList<>(); + try { + confFile.load(Files.newInputStream(Paths.get(pushConfigFile))); + } catch (Throwable exception) { + logger.severe("Could not load configuration file " + pushConfigFile); + throw new RuntimeException(exception.getMessage()); + } + + confFile.entrySet().stream() + .filter(entry -> !entry.getKey().toString().startsWith("multicasting")) + .forEach( + entry -> { + fileArgs.add("--" + entry.getKey().toString()); + fileArgs.add(entry.getValue().toString()); + }); + + jc.parse(fileArgs.toArray(new String[0])); + detectModifiedOptions(fileArgs.stream().filter(s -> s.startsWith("-")), modifyByFile); + String fileStr = + modifyByFile.stream().map(field -> field.getName()).collect(Collectors.joining(", ")); + logger.info("modifyByFile: " + fileStr); + modifyByArgs.removeAll(modifyByFile); // argument are override by the config file + configFileExtraArguments(confFile); + } + + constructTenantInfoObject( + cspAppId, + cspAppSecret, + cspOrgId, + cspAPIToken, + token, + server, + APIContainer.CENTRAL_TENANT_NAME); + + logger.info("Unparsed arguments: " + Joiner.on(", ").join(jc.getUnknownOptions())); + + String FQDN = getLocalHostName(); + if (!hostname.equals(FQDN)) { + logger.warning( + "Deprecated field hostname specified in config setting. Please use " + + "proxyname config field to set proxy name."); + if (proxyname.equals(FQDN)) proxyname = hostname; + } + logger.info("Using proxyname:'" + proxyname + "' hostname:'" + hostname + "'"); + + if (httpUserAgent == null) { + httpUserAgent = "Wavefront-Proxy/" + getBuildVersion(); + } + + // TODO: deprecate this + createConfigMetrics(); + + List cfgStrs = new ArrayList<>(); + List cfg = new ArrayList<>(); + cfg.addAll(modifyByArgs); + cfg.addAll(modifyByFile); + cfg.stream() + .forEach( + field -> { + Optional option = + Arrays.stream(field.getAnnotationsByType(ProxyConfigOption.class)).findFirst(); + boolean hide = option.isPresent() && option.get().hide(); + if (!hide) { + boolean secret = option.isPresent() && option.get().secret(); + try { + boolean arg = !modifyByFile.contains(field); + cfgStrs.add( + "\t" + + (arg ? "* " : " ") + + field.getName() + + " = " + + (secret ? "******" : field.get(this))); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + }); + logger.info("Config: (* command line argument)"); + for (String cfgStr : cfgStrs) { + logger.info(cfgStr); + } + + return true; + } + + private void createConfigMetrics() { + Field[] fields = this.getClass().getDeclaredFields(); + for (Field field : fields) { + Optional parameter = + Arrays.stream(field.getAnnotationsByType(Parameter.class)).findFirst(); + Optional option = + Arrays.stream(field.getAnnotationsByType(ProxyConfigOption.class)).findFirst(); + boolean hide = option.isPresent() && option.get().hide(); + if (parameter.isPresent() && !hide) { + MetricName name = new MetricName("config", "", field.getName()); + try { + Class type = (Class) field.getGenericType(); + if (type.isAssignableFrom(String.class)) { + String val = (String) field.get(this); + if (StringUtils.isNotBlank(val)) { + name = new TaggedMetricName(name.getGroup(), name.getName(), "value", val); + reportGauge(1, name); + } else { + reportGauge(0, name); + } + } else if (type.isEnum()) { + String val = field.get(this).toString(); + name = new TaggedMetricName(name.getGroup(), name.getName(), "value", val); + reportGauge(1, name); + } else if (type.isAssignableFrom(boolean.class)) { + Boolean val = (Boolean) field.get(this); + reportGauge(val.booleanValue() ? 1 : 0, name); + } else if (type.isAssignableFrom(int.class)) { + reportGauge((int) field.get(this), name); + } else if (type.isAssignableFrom(double.class)) { + reportGauge((double) field.get(this), name); + } else if (type.isAssignableFrom(long.class)) { + reportGauge((long) field.get(this), name); + } else if (type.isAssignableFrom(short.class)) { + reportGauge((short) field.get(this), name); + } else { + throw new RuntimeException("--- " + field.getType()); + } + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + } + } + + private void detectModifiedOptions(Stream args, List list) { + args.forEach( + arg -> { + Field[] fields = this.getClass().getSuperclass().getDeclaredFields(); + list.addAll( + Arrays.stream(fields) + .filter( + field -> { + Optional parameter = + Arrays.stream(field.getAnnotationsByType(Parameter.class)).findFirst(); + if (parameter.isPresent()) { + String[] names = parameter.get().names(); + if (Arrays.asList(names).contains(arg)) { + return true; + } + } + return false; + }) + .collect(Collectors.toList())); + }); + } + + @JsonIgnore + public JsonNode getJsonConfig() { + Map>> cfg = + new TreeMap<>(Comparator.comparingInt(Categories::getOrder)); + for (Field field : this.getClass().getSuperclass().getDeclaredFields()) { + Optional option = + Arrays.stream(field.getAnnotationsByType(ProxyConfigOption.class)).findFirst(); + Optional parameter = + Arrays.stream(field.getAnnotationsByType(Parameter.class)).findFirst(); + if (parameter.isPresent()) { + ProxyConfigOptionDescriptor data = new ProxyConfigOptionDescriptor(); + data.name = + Arrays.stream(parameter.get().names()) + .max(Comparator.comparingInt(String::length)) + .orElseGet(() -> field.getName()) + .replaceAll("--", ""); + data.description = parameter.get().description(); + data.order = parameter.get().order() == -1 ? 99999 : parameter.get().order(); + try { + Object val = field.get(this); + if ((data.name.equals("token") + || data.name.equals("cspAPIToken") + || data.name.equals("cspAppId") + || data.name.equals("cspAppSecret")) + && val != null) { + String value = val.toString(); + data.value = + StringUtils.repeat("*", value.length() - NUMBER_OF_VISIBLE_DIGITS) + + value.substring(value.length() - NUMBER_OF_VISIBLE_DIGITS); + } else { + data.value = val != null ? val.toString() : "null"; + } + } catch (IllegalAccessException e) { + logger.severe(e.toString()); + } + + if (modifyByArgs.contains(field)) { + data.modifyBy = "Argument"; + } else if (modifyByFile.contains(field)) { + data.modifyBy = "Config file"; + } + + if (option.isPresent()) { + Categories category = option.get().category(); + SubCategories subCategory = option.get().subCategory(); + if (!option.get().hide()) { + Set options = + cfg.computeIfAbsent( + category, + s -> new TreeMap<>(Comparator.comparingInt(SubCategories::getOrder))) + .computeIfAbsent(subCategory, s -> new TreeSet<>()); + options.add(data); + } + } else { + throw new RuntimeException( + "All options need 'ProxyConfigOption' annotation (" + data.name + ") !!"); + } + } + } + ObjectMapper mapper = new ObjectMapper(); + JsonNode node = mapper.convertValue(cfg, JsonNode.class); + return node; + } + + public static class TokenValidationMethodConverter + implements IStringConverter { + @Override + public TokenValidationMethod convert(String value) { + TokenValidationMethod convertedValue = TokenValidationMethod.fromString(value); + if (convertedValue == null) { + throw new ParameterException("Unknown token validation method value: " + value); + } + return convertedValue; + } + } + + public static class TaskQueueLevelConverter implements IStringConverter { + @Override + public TaskQueueLevel convert(String value) { + TaskQueueLevel convertedValue = TaskQueueLevel.fromString(value); + if (convertedValue == null) { + throw new ParameterException("Unknown task queue level: " + value); + } + return convertedValue; + } + } + + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public static class ProxyConfigOptionDescriptor implements Comparable { + public String name, description, value, modifyBy; + public int order = 0; + + @Override + public int compareTo(@NotNull Object o) { + ProxyConfigOptionDescriptor other = (ProxyConfigOptionDescriptor) o; + if (this.order == other.order) { + return this.name.compareTo(other.name); + } + return Integer.compare(this.order, other.order); + } + } + + /** + * Helper function to construct tenant info {@link TokenWorkerCSP} object based on input + * parameters. + * + * @param appId the CSP OAuth server to server app id. + * @param appSecret the CSP OAuth server to server app secret. + * @param cspOrgId the CSP organisation id. + * @param cspAPIToken the CSP API wfToken. + * @param wfToken the Wavefront API wfToken. + * @param server the server url. + * @param tenantName the name of the tenant. + * @throws IllegalArgumentException for invalid arguments. + */ + public void constructTenantInfoObject( + @Nullable final String appId, + @Nullable final String appSecret, + @Nullable final String cspOrgId, + @Nullable final String cspAPIToken, + @Nonnull final String wfToken, + @Nonnull final String server, + @Nonnull final String tenantName) { + + final String BAD_CONFIG = + "incorrect configuration, one (and only one) of these options are required: `token`, `cspAPIToken` or `cspAppId, cspAppSecret`" + + (CENTRAL_TENANT_NAME.equals(tenantName) ? "" : " for tenant `" + tenantName + "`"); + + boolean isOAuthApp = StringUtils.isNotBlank(appId) || StringUtils.isNotBlank(appSecret); + boolean isCSPAPIToken = StringUtils.isNotBlank(cspAPIToken); + boolean isWFToken = StringUtils.isNotBlank(wfToken); + + if (Stream.of(isOAuthApp, isCSPAPIToken, isWFToken).filter(auth -> auth).count() != 1) { + throw new IllegalArgumentException(BAD_CONFIG); + } + + TenantInfo tokenWorker; + if (isOAuthApp) { + if (StringUtils.isNotBlank(appId) && StringUtils.isNotBlank(appSecret)) { + logger.info( + "TCSP OAuth server to server app credentials for further authentication. For the server " + + server); + tokenWorker = new TokenWorkerCSP(appId, appSecret, cspOrgId, server); + } else { + throw new IllegalArgumentException( + "To use server to server oauth, both `cspAppId` and `cspAppSecret` are required."); + } + } else if (isCSPAPIToken) { + logger.info("CSP api token for further authentication. For the server " + server); + tokenWorker = new TokenWorkerCSP(cspAPIToken, server); + } else { // isWFToken + logger.info("Wavefront api token for further authentication. For the server " + server); + tokenWorker = new TokenWorkerWF(wfToken, server); + } + + TokenManager.addTenant(tenantName, tokenWorker); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/ProxyConfigDef.java b/proxy/src/main/java/com/wavefront/agent/ProxyConfigDef.java new file mode 100644 index 000000000..8f5cdd1f0 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/ProxyConfigDef.java @@ -0,0 +1,1553 @@ +package com.wavefront.agent; + +import static com.wavefront.agent.ProxyConfig.GRAPHITE_LISTENING_PORT; +import static com.wavefront.agent.data.EntityProperties.*; +import static com.wavefront.common.Utils.getLocalHostName; + +import com.beust.jcommander.Parameter; +import com.wavefront.agent.auth.TokenValidationMethod; +import com.wavefront.agent.config.Categories; +import com.wavefront.agent.config.Configuration; +import com.wavefront.agent.config.ProxyConfigOption; +import com.wavefront.agent.config.SubCategories; +import com.wavefront.agent.data.TaskQueueLevel; + +/** Proxy configuration (refactored from {@link AbstractAgent}). */ +public abstract class ProxyConfigDef extends Configuration { + @Parameter( + names = {"--privateCertPath"}, + description = + "TLS certificate path to use for securing all the ports. " + + "X.509 certificate chain file in PEM format.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.TLS) + protected String privateCertPath = ""; + + @Parameter( + names = {"--privateKeyPath"}, + description = + "TLS private key path to use for securing all the ports. " + + "PKCS#8 private key file in PEM format.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.TLS) + protected String privateKeyPath = ""; + + @Parameter( + names = {"--tlsPorts"}, + description = + "Comma-separated list of ports to be secured using TLS. " + + "All ports will be secured when * specified.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.TLS) + protected String tlsPorts = ""; + + @Parameter( + names = {"--trafficShaping"}, + description = + "Enables intelligent traffic shaping " + + "based on received rate over last 5 minutes. Default: disabled", + arity = 1) + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + protected boolean trafficShaping = false; + + @Parameter( + names = {"--trafficShapingWindowSeconds"}, + description = + "Sets the width " + + "(in seconds) for the sliding time window which would be used to calculate received " + + "traffic rate. Default: 600 (10 minutes)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + protected int trafficShapingWindowSeconds = 600; + + @Parameter( + names = {"--trafficShapingHeadroom"}, + description = + "Sets the headroom multiplier " + + " to use for traffic shaping when there's backlog. Default: 1.15 (15% headroom)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + protected double trafficShapingHeadroom = 1.15; + + @Parameter( + names = {"--corsEnabledPorts"}, + description = + "Enables CORS for specified " + + "comma-delimited list of listening ports. Default: none (CORS disabled)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CORS) + protected String corsEnabledPorts = ""; + + @Parameter( + names = {"--corsOrigin"}, + description = + "Allowed origin for CORS requests, " + "or '*' to allow everything. Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CORS) + protected String corsOrigin = ""; + + @Parameter( + names = {"--corsAllowNullOrigin"}, + description = "Allow 'null' origin for CORS " + "requests. Default: false") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CORS) + protected boolean corsAllowNullOrigin = false; + + @Parameter( + names = {"--help"}, + help = true) + @ProxyConfigOption(hide = true) + boolean help = false; + + @Parameter( + names = {"--version"}, + description = "Print version and exit.", + order = 0) + @ProxyConfigOption(hide = true) + boolean version = false; + + @Parameter( + names = {"-f", "--file"}, + description = "Proxy configuration file", + order = 2) + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String pushConfigFile = null; + + @Parameter( + names = {"-p", "--prefix"}, + description = "Prefix to prepend to all push metrics before reporting.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String prefix = null; + + @Parameter( + names = {"-t", "--token"}, + description = "Token to auto-register proxy with an account", + order = 1) + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF, secret = true) + String token = null; + + @Parameter( + names = {"--testLogs"}, + description = "Run interactive session for crafting logsIngestionConfig.yaml") + @ProxyConfigOption(hide = true) + boolean testLogs = false; + + @Parameter( + names = {"--testPreprocessorForPort"}, + description = + "Run interactive session for " + "testing preprocessor rules for specified port") + @ProxyConfigOption(hide = true) + String testPreprocessorForPort = null; + + @Parameter( + names = {"--testSpanPreprocessorForPort"}, + description = + "Run interactive session " + "for testing preprocessor span rules for specifierd port") + @ProxyConfigOption(hide = true) + String testSpanPreprocessorForPort = null; + + @Parameter( + names = {"--server", "-h", "--host"}, + description = "Server URL", + order = 0) + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String server = "http://localhost:8080/api/"; + + @Parameter( + names = {"--buffer"}, + description = + "File name prefix to use for buffering " + + "transmissions to be retried. Defaults to /var/spool/wavefront-proxy/buffer.", + order = 4) + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.DISK) + String bufferFile = "/var/spool/wavefront-proxy/buffer"; + + @Parameter( + names = {"--bufferShardSize"}, + description = + "Buffer file partition size, in MB. " + + "Setting this value too low may reduce the efficiency of disk space utilization, " + + "while setting this value too high will allocate disk space in larger increments. " + + "Default: 128") + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.DISK) + int bufferShardSize = 128; + + @Parameter( + names = {"--disableBufferSharding"}, + description = "Use single-file buffer " + "(legacy functionality). Default: false", + arity = 1) + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.DISK) + boolean disableBufferSharding = false; + + @Parameter( + names = {"--sqsBuffer"}, + description = + "Use AWS SQS Based for buffering transmissions " + "to be retried. Defaults to False", + arity = 1) + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.SQS) + boolean sqsQueueBuffer = false; + + @Parameter( + names = {"--sqsQueueNameTemplate"}, + description = + "The replacement pattern to use for naming the " + + "sqs queues. e.g. wf-proxy-{{id}}-{{entity}}-{{port}} would result in a queue named wf-proxy-id-points-2878") + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.SQS) + String sqsQueueNameTemplate = "wf-proxy-{{id}}-{{entity}}-{{port}}"; + + @Parameter( + names = {"--sqsQueueIdentifier"}, + description = "An identifier for identifying these proxies in SQS") + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.SQS) + String sqsQueueIdentifier = null; + + @Parameter( + names = {"--sqsQueueRegion"}, + description = "The AWS Region name the queue will live in.") + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.SQS) + String sqsQueueRegion = "us-west-2"; + + @Parameter( + names = {"--taskQueueLevel"}, + converter = ProxyConfig.TaskQueueLevelConverter.class, + description = + "Sets queueing strategy. Allowed values: MEMORY, PUSHBACK, ANY_ERROR. " + + "Default: ANY_ERROR") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + TaskQueueLevel taskQueueLevel = TaskQueueLevel.ANY_ERROR; + + @Parameter( + names = {"--exportQueuePorts"}, + description = + "Export queued data in plaintext " + + "format for specified ports (comma-delimited list) and exit. Set to 'all' to export " + + "everything. Default: none") + @ProxyConfigOption(hide = true) + String exportQueuePorts = null; + + @Parameter( + names = {"--exportQueueOutputFile"}, + description = + "Export queued data in plaintext " + + "format for specified ports (comma-delimited list) and exit. Default: none") + @ProxyConfigOption(hide = true) + String exportQueueOutputFile = null; + + @Parameter( + names = {"--exportQueueRetainData"}, + description = "Whether to retain data in the " + "queue during export. Defaults to true.", + arity = 1) + @ProxyConfigOption(hide = true) + boolean exportQueueRetainData = true; + + @Parameter( + names = {"--useNoopSender"}, + description = + "Run proxy in debug/performance test " + + "mode and discard all received data. Default: false", + arity = 1) + @ProxyConfigOption(hide = true) + boolean useNoopSender = false; + + @Parameter( + names = {"--flushThreads"}, + description = + "Number of threads that flush data to the server. Defaults to" + + "the number of processors (min. 4). Setting this value too large will result in sending batches that are too " + + "small to the server and wasting connections. This setting is per listening port.") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.METRICS) + int flushThreads = Math.min(16, Math.max(4, Runtime.getRuntime().availableProcessors())); + + @Parameter( + names = {"--flushThreadsSourceTags"}, + description = "Number of threads that send " + "source tags data to the server. Default: 2") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.SOURCETAGS) + int flushThreadsSourceTags = DEFAULT_FLUSH_THREADS_SOURCE_TAGS; + + @Parameter( + names = {"--flushThreadsEvents"}, + description = "Number of threads that send " + "event data to the server. Default: 2") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.EVENTS) + int flushThreadsEvents = DEFAULT_FLUSH_THREADS_EVENTS; + + @Parameter( + names = {"--flushThreadsLogs"}, + description = + "Number of threads that flush data to " + + "the server. Defaults to the number of processors (min. 4). Setting this value too large " + + "will result in sending batches that are too small to the server and wasting connections. This setting is per listening port.", + order = 5) + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.LOGS) + int flushThreadsLogs = Math.min(16, Math.max(4, Runtime.getRuntime().availableProcessors())); + + @Parameter( + names = {"--purgeBuffer"}, + description = "Whether to purge the retry buffer on start-up. Defaults to " + "false.", + arity = 1) + @ProxyConfigOption(hide = true) + boolean purgeBuffer = false; + + @Parameter( + names = {"--pushFlushInterval"}, + description = "Milliseconds between batches. " + "Defaults to 1000 ms") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.METRICS) + int pushFlushInterval = DEFAULT_FLUSH_INTERVAL; + + @Parameter( + names = {"--pushFlushIntervalLogs"}, + description = "Milliseconds between batches. Defaults to 1000 ms") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.LOGS) + int pushFlushIntervalLogs = DEFAULT_FLUSH_INTERVAL; + + @Parameter( + names = {"--pushFlushMaxPoints"}, + description = "Maximum allowed points " + "in a single flush. Defaults: 40000") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.METRICS) + int pushFlushMaxPoints = DEFAULT_BATCH_SIZE; + + @Parameter( + names = {"--pushFlushMaxHistograms"}, + description = "Maximum allowed histograms " + "in a single flush. Default: 10000") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.HISTO) + int pushFlushMaxHistograms = DEFAULT_BATCH_SIZE_HISTOGRAMS; + + @Parameter( + names = {"--pushFlushMaxSourceTags"}, + description = "Maximum allowed source tags " + "in a single flush. Default: 50") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.SOURCETAGS) + int pushFlushMaxSourceTags = DEFAULT_BATCH_SIZE_SOURCE_TAGS; + + @Parameter( + names = {"--pushFlushMaxSpans"}, + description = "Maximum allowed spans " + "in a single flush. Default: 5000") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.TRACES) + int pushFlushMaxSpans = DEFAULT_BATCH_SIZE_SPANS; + + @Parameter( + names = {"--pushFlushMaxSpanLogs"}, + description = "Maximum allowed span logs " + "in a single flush. Default: 1000") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.TRACES) + int pushFlushMaxSpanLogs = DEFAULT_BATCH_SIZE_SPAN_LOGS; + + @Parameter( + names = {"--pushFlushMaxEvents"}, + description = "Maximum allowed events " + "in a single flush. Default: 50") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.EVENTS) + int pushFlushMaxEvents = DEFAULT_BATCH_SIZE_EVENTS; + + @Parameter( + names = {"--pushFlushMaxLogs"}, + description = + "Maximum size of a log payload " + + "in a single flush in bytes between 1mb (1048576) and 5mb (5242880). Default: 4mb (4194304)") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.LOGS) + int pushFlushMaxLogs = DEFAULT_BATCH_SIZE_LOGS_PAYLOAD; + + @Parameter( + names = {"--pushRateLimit"}, + description = "Limit the outgoing point rate at the proxy. Default: " + "do not throttle.") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.METRICS) + double pushRateLimit = NO_RATE_LIMIT; + + @Parameter( + names = {"--pushRateLimitHistograms"}, + description = + "Limit the outgoing histogram " + "rate at the proxy. Default: do not throttle.") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.HISTO) + double pushRateLimitHistograms = NO_RATE_LIMIT; + + @Parameter( + names = {"--pushRateLimitSourceTags"}, + description = "Limit the outgoing rate " + "for source tags at the proxy. Default: 5 op/s") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.SOURCETAGS) + double pushRateLimitSourceTags = 5.0d; + + @Parameter( + names = {"--pushRateLimitSpans"}, + description = + "Limit the outgoing tracing spans " + "rate at the proxy. Default: do not throttle.") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.TRACES) + double pushRateLimitSpans = NO_RATE_LIMIT; + + @Parameter( + names = {"--pushRateLimitSpanLogs"}, + description = + "Limit the outgoing span logs " + "rate at the proxy. Default: do not throttle.") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.TRACES) + double pushRateLimitSpanLogs = NO_RATE_LIMIT; + + @Parameter( + names = {"--pushRateLimitEvents"}, + description = "Limit the outgoing rate " + "for events at the proxy. Default: 5 events/s") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.EVENTS) + double pushRateLimitEvents = 5.0d; + + @Parameter( + names = {"--pushRateLimitLogs"}, + description = + "Limit the outgoing logs " + "data rate at the proxy. Default: do not throttle.") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.LOGS) + double pushRateLimitLogs = NO_RATE_LIMIT_BYTES; + + @Parameter( + names = {"--pushRateLimitMaxBurstSeconds"}, + description = + "Max number of burst seconds to allow " + + "when rate limiting to smooth out uneven traffic. Set to 1 when doing data backfills. Default: 10") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.METRICS) + int pushRateLimitMaxBurstSeconds = 10; + + @Parameter( + names = {"--pushMemoryBufferLimit"}, + description = + "Max number of points that can stay in memory buffers" + + " before spooling to disk. Defaults to 16 * pushFlushMaxPoints, minimum size: pushFlushMaxPoints. Setting this " + + " value lower than default reduces memory usage but will force the proxy to spool to disk more frequently if " + + " you have points arriving at the proxy in short bursts") + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.METRICS) + int pushMemoryBufferLimit = 16 * pushFlushMaxPoints; + + @Parameter( + names = {"--pushMemoryBufferLimitLogs"}, + description = + "Max number of logs that " + + "can stay in memory buffers before spooling to disk. Defaults to 16 * pushFlushMaxLogs, " + + "minimum size: pushFlushMaxLogs. Setting this value lower than default reduces memory usage " + + "but will force the proxy to spool to disk more frequently if you have points arriving at the " + + "proxy in short bursts") + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.LOGS) + int pushMemoryBufferLimitLogs = 16 * pushFlushMaxLogs; + + @Parameter( + names = {"--pushBlockedSamples"}, + description = "Max number of blocked samples to print to log. Defaults" + " to 5.") + @ProxyConfigOption(category = Categories.TRACE, subCategory = SubCategories.METRICS) + int pushBlockedSamples = 5; + + @Parameter( + names = {"--blockedPointsLoggerName"}, + description = "Logger Name for blocked " + "points. " + "Default: RawBlockedPoints") + @ProxyConfigOption(category = Categories.TRACE, subCategory = SubCategories.METRICS) + String blockedPointsLoggerName = "RawBlockedPoints"; + + @Parameter( + names = {"--blockedHistogramsLoggerName"}, + description = "Logger Name for blocked " + "histograms" + "Default: RawBlockedPoints") + @ProxyConfigOption(category = Categories.TRACE, subCategory = SubCategories.HISTO) + String blockedHistogramsLoggerName = "RawBlockedPoints"; + + @Parameter( + names = {"--blockedSpansLoggerName"}, + description = "Logger Name for blocked spans" + "Default: RawBlockedPoints") + @ProxyConfigOption(category = Categories.TRACE, subCategory = SubCategories.TRACES) + String blockedSpansLoggerName = "RawBlockedPoints"; + + @Parameter( + names = {"--blockedLogsLoggerName"}, + description = "Logger Name for blocked logs" + "Default: RawBlockedLogs") + @ProxyConfigOption(category = Categories.TRACE, subCategory = SubCategories.LOGS) + String blockedLogsLoggerName = "RawBlockedLogs"; + + @Parameter( + names = {"--pushListenerPorts"}, + description = "Comma-separated list of ports to listen on. Defaults to " + "2878.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.METRICS) + String pushListenerPorts = "" + GRAPHITE_LISTENING_PORT; + + @Parameter( + names = {"--pushListenerMaxReceivedLength"}, + description = + "Maximum line length for received points in" + + " plaintext format on Wavefront/OpenTSDB/Graphite ports. Default: 32768 (32KB)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.METRICS) + int pushListenerMaxReceivedLength = 32768; + + @Parameter( + names = {"--pushListenerHttpBufferSize"}, + description = + "Maximum allowed request size (in bytes) for" + + " incoming HTTP requests on Wavefront/OpenTSDB/Graphite ports (Default: 16MB)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.METRICS) + int pushListenerHttpBufferSize = 16 * 1024 * 1024; + + @Parameter( + names = {"--traceListenerMaxReceivedLength"}, + description = "Maximum line length for received spans and" + " span logs (Default: 1MB)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES) + int traceListenerMaxReceivedLength = 1024 * 1024; + + @Parameter( + names = {"--traceListenerHttpBufferSize"}, + description = + "Maximum allowed request size (in bytes) for" + + " incoming HTTP requests on tracing ports (Default: 16MB)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES) + int traceListenerHttpBufferSize = 16 * 1024 * 1024; + + @Parameter( + names = {"--listenerIdleConnectionTimeout"}, + description = + "Close idle inbound connections after " + " specified time in seconds. Default: 300") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.OTHER) + int listenerIdleConnectionTimeout = 300; + + @Parameter( + names = {"--memGuardFlushThreshold"}, + description = + "If heap usage exceeds this threshold (in percent), " + + "flush pending points to disk as an additional OoM protection measure. Set to 0 to disable. Default: 99") + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.OTHER) + int memGuardFlushThreshold = 98; + + @Parameter( + names = {"--histogramPassthroughRecompression"}, + description = + "Whether we should recompress histograms received on pushListenerPorts. " + + "Default: true", + arity = 1) + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + boolean histogramPassthroughRecompression = true; + + @Parameter( + names = {"--histogramStateDirectory"}, + description = "Directory for persistent proxy state, must be writable.") + @ProxyConfigOption(category = Categories.BUFFER, subCategory = SubCategories.DISK) + String histogramStateDirectory = "/var/spool/wavefront-proxy"; + + @Parameter( + names = {"--histogramAccumulatorResolveInterval"}, + description = + "Interval to write-back accumulation changes from memory cache to disk in " + + "millis (only applicable when memory cache is enabled") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + long histogramAccumulatorResolveInterval = 5000L; + + @Parameter( + names = {"--histogramAccumulatorFlushInterval"}, + description = + "Interval to check for histograms to send to Wavefront in millis. " + "(Default: 10000)") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.HISTO) + long histogramAccumulatorFlushInterval = 10000L; + + @Parameter( + names = {"--histogramAccumulatorFlushMaxBatchSize"}, + description = + "Max number of histograms to send to Wavefront in one flush " + "(Default: no limit)") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.HISTO) + int histogramAccumulatorFlushMaxBatchSize = -1; + + @Parameter( + names = {"--histogramMaxReceivedLength"}, + description = "Maximum line length for received histogram data (Default: 65536)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramMaxReceivedLength = 64 * 1024; + + @Parameter( + names = {"--histogramHttpBufferSize"}, + description = + "Maximum allowed request size (in bytes) for incoming HTTP requests on " + + "histogram ports (Default: 16MB)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramHttpBufferSize = 16 * 1024 * 1024; + + @Parameter( + names = {"--histogramMinuteListenerPorts"}, + description = "Comma-separated list of ports to listen on. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + String histogramMinuteListenerPorts = ""; + + @Parameter( + names = {"--histogramMinuteFlushSecs"}, + description = + "Number of seconds to keep a minute granularity accumulator open for " + "new samples.") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.HISTO) + int histogramMinuteFlushSecs = 70; + + @Parameter( + names = {"--histogramMinuteCompression"}, + description = "Controls allowable number of centroids per histogram. Must be in [20;1000]") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + short histogramMinuteCompression = 32; + + @Parameter( + names = {"--histogramMinuteAvgKeyBytes"}, + description = + "Average number of bytes in a [UTF-8] encoded histogram key. Generally " + + "corresponds to a metric, source and tags concatenation.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramMinuteAvgKeyBytes = 150; + + @Parameter( + names = {"--histogramMinuteAvgDigestBytes"}, + description = "Average number of bytes in a encoded histogram.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramMinuteAvgDigestBytes = 500; + + @Parameter( + names = {"--histogramMinuteAccumulatorSize"}, + description = + "Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel " + + "reporting bins") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + long histogramMinuteAccumulatorSize = 100000L; + + @Parameter( + names = {"--histogramMinuteAccumulatorPersisted"}, + arity = 1, + description = "Whether the accumulator should persist to disk") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + boolean histogramMinuteAccumulatorPersisted = false; + + @Parameter( + names = {"--histogramMinuteMemoryCache"}, + arity = 1, + description = + "Enabling memory cache reduces I/O load with fewer time series and higher " + + "frequency data (more than 1 point per second per time series). Default: false") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + boolean histogramMinuteMemoryCache = false; + + @Parameter( + names = {"--histogramHourListenerPorts"}, + description = "Comma-separated list of ports to listen on. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + String histogramHourListenerPorts = ""; + + @Parameter( + names = {"--histogramHourFlushSecs"}, + description = + "Number of seconds to keep an hour granularity accumulator open for " + "new samples.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramHourFlushSecs = 4200; + + @Parameter( + names = {"--histogramHourCompression"}, + description = "Controls allowable number of centroids per histogram. Must be in [20;1000]") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + short histogramHourCompression = 32; + + @Parameter( + names = {"--histogramHourAvgKeyBytes"}, + description = + "Average number of bytes in a [UTF-8] encoded histogram key. Generally " + + " corresponds to a metric, source and tags concatenation.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramHourAvgKeyBytes = 150; + + @Parameter( + names = {"--histogramHourAvgDigestBytes"}, + description = "Average number of bytes in a encoded histogram.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramHourAvgDigestBytes = 500; + + @Parameter( + names = {"--histogramHourAccumulatorSize"}, + description = + "Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel " + + "reporting bins") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + long histogramHourAccumulatorSize = 100000L; + + @Parameter( + names = {"--histogramHourAccumulatorPersisted"}, + arity = 1, + description = "Whether the accumulator should persist to disk") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + boolean histogramHourAccumulatorPersisted = false; + + @Parameter( + names = {"--histogramHourMemoryCache"}, + arity = 1, + description = + "Enabling memory cache reduces I/O load with fewer time series and higher " + + "frequency data (more than 1 point per second per time series). Default: false") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + boolean histogramHourMemoryCache = false; + + @Parameter( + names = {"--histogramDayListenerPorts"}, + description = "Comma-separated list of ports to listen on. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + String histogramDayListenerPorts = ""; + + @Parameter( + names = {"--histogramDayFlushSecs"}, + description = "Number of seconds to keep a day granularity accumulator open for new samples.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramDayFlushSecs = 18000; + + @Parameter( + names = {"--histogramDayCompression"}, + description = "Controls allowable number of centroids per histogram. Must be in [20;1000]") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + short histogramDayCompression = 32; + + @Parameter( + names = {"--histogramDayAvgKeyBytes"}, + description = + "Average number of bytes in a [UTF-8] encoded histogram key. Generally " + + "corresponds to a metric, source and tags concatenation.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramDayAvgKeyBytes = 150; + + @Parameter( + names = {"--histogramDayAvgHistogramDigestBytes"}, + description = "Average number of bytes in a encoded histogram.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramDayAvgDigestBytes = 500; + + @Parameter( + names = {"--histogramDayAccumulatorSize"}, + description = + "Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel " + + "reporting bins") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + long histogramDayAccumulatorSize = 100000L; + + @Parameter( + names = {"--histogramDayAccumulatorPersisted"}, + arity = 1, + description = "Whether the accumulator should persist to disk") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + boolean histogramDayAccumulatorPersisted = false; + + @Parameter( + names = {"--histogramDayMemoryCache"}, + arity = 1, + description = + "Enabling memory cache reduces I/O load with fewer time series and higher " + + "frequency data (more than 1 point per second per time series). Default: false") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + boolean histogramDayMemoryCache = false; + + @Parameter( + names = {"--histogramDistListenerPorts"}, + description = "Comma-separated list of ports to listen on. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + String histogramDistListenerPorts = ""; + + @Parameter( + names = {"--histogramDistFlushSecs"}, + description = "Number of seconds to keep a new distribution bin open for new samples.") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.HISTO) + int histogramDistFlushSecs = 70; + + @Parameter( + names = {"--histogramDistCompression"}, + description = "Controls allowable number of centroids per histogram. Must be in [20;1000]") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + short histogramDistCompression = 32; + + @Parameter( + names = {"--histogramDistAvgKeyBytes"}, + description = + "Average number of bytes in a [UTF-8] encoded histogram key. Generally " + + "corresponds to a metric, source and tags concatenation.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramDistAvgKeyBytes = 150; + + @Parameter( + names = {"--histogramDistAvgDigestBytes"}, + description = "Average number of bytes in a encoded histogram.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int histogramDistAvgDigestBytes = 500; + + @Parameter( + names = {"--histogramDistAccumulatorSize"}, + description = + "Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel " + + "reporting bins") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + long histogramDistAccumulatorSize = 100000L; + + @Parameter( + names = {"--histogramDistAccumulatorPersisted"}, + arity = 1, + description = "Whether the accumulator should persist to disk") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + boolean histogramDistAccumulatorPersisted = false; + + @Parameter( + names = {"--histogramDistMemoryCache"}, + arity = 1, + description = + "Enabling memory cache reduces I/O load with fewer time series and higher " + + "frequency data (more than 1 point per second per time series). Default: false") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + boolean histogramDistMemoryCache = false; + + @Parameter( + names = {"--graphitePorts"}, + description = + "Comma-separated list of ports to listen on for graphite " + + "data. Defaults to empty list.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.GRAPHITE) + String graphitePorts = ""; + + @Parameter( + names = {"--graphiteFormat"}, + description = + "Comma-separated list of metric segments to extract and " + + "reassemble as the hostname (1-based).") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.GRAPHITE) + String graphiteFormat = ""; + + @Parameter( + names = {"--graphiteDelimiters"}, + description = + "Concatenated delimiters that should be replaced in the " + + "extracted hostname with dots. Defaults to underscores (_).") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.GRAPHITE) + String graphiteDelimiters = "_"; + + @Parameter( + names = {"--graphiteFieldsToRemove"}, + description = "Comma-separated list of metric segments to remove (1-based)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.GRAPHITE) + String graphiteFieldsToRemove; + + @Parameter( + names = {"--jsonListenerPorts", "--httpJsonPorts"}, + description = + "Comma-separated list of ports to " + + "listen on for json metrics data. Binds, by default, to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.JSON) + String jsonListenerPorts = ""; + + @Parameter( + names = {"--dataDogJsonPorts"}, + description = + "Comma-separated list of ports to listen on for JSON " + + "metrics data in DataDog format. Binds, by default, to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.DDOG) + String dataDogJsonPorts = ""; + + @Parameter( + names = {"--dataDogRequestRelayTarget"}, + description = + "HTTP/HTTPS target for relaying all incoming " + + "requests on dataDogJsonPorts to. Defaults to none (do not relay incoming requests)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.DDOG) + String dataDogRequestRelayTarget = null; + + @Parameter( + names = {"--dataDogRequestRelayAsyncThreads"}, + description = + "Max number of " + + "in-flight HTTP requests being relayed to dataDogRequestRelayTarget. Default: 32") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.DDOG) + int dataDogRequestRelayAsyncThreads = 32; + + @Parameter( + names = {"--dataDogRequestRelaySyncMode"}, + description = + "Whether we should wait " + + "until request is relayed successfully before processing metrics. Default: false") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.DDOG) + boolean dataDogRequestRelaySyncMode = false; + + @Parameter( + names = {"--dataDogProcessSystemMetrics"}, + description = + "If true, handle system metrics as reported by " + + "DataDog collection agent. Defaults to false.", + arity = 1) + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.DDOG) + boolean dataDogProcessSystemMetrics = false; + + @Parameter( + names = {"--dataDogProcessServiceChecks"}, + description = "If true, convert service checks to metrics. " + "Defaults to true.", + arity = 1) + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.DDOG) + boolean dataDogProcessServiceChecks = true; + + @Parameter( + names = {"--writeHttpJsonListenerPorts", "--writeHttpJsonPorts"}, + description = + "Comma-separated list " + + "of ports to listen on for json metrics from collectd write_http json format data. Binds, by default, to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.JSON) + String writeHttpJsonListenerPorts = ""; + + @Parameter( + names = {"--otlpGrpcListenerPorts"}, + description = + "Comma-separated list of ports to" + + " listen on for OpenTelemetry/OTLP Protobuf formatted data over gRPC. Binds, by default, to" + + " none (4317 is recommended).") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.OPENTEL) + String otlpGrpcListenerPorts = ""; + + @Parameter( + names = {"--otlpHttpListenerPorts"}, + description = + "Comma-separated list of ports to" + + " listen on for OpenTelemetry/OTLP Protobuf formatted data over HTTP. Binds, by default, to" + + " none (4318 is recommended).") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.OPENTEL) + String otlpHttpListenerPorts = ""; + + @Parameter( + names = {"--otlpResourceAttrsOnMetricsIncluded"}, + arity = 1, + description = "If true, includes OTLP resource attributes on metrics (Default: false)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.OPENTEL) + boolean otlpResourceAttrsOnMetricsIncluded = false; + + @Parameter( + names = {"--otlpAppTagsOnMetricsIncluded"}, + arity = 1, + description = + "If true, includes the following application-related resource attributes on " + + "metrics: application, service.name, shard, cluster (Default: true)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.OPENTEL) + boolean otlpAppTagsOnMetricsIncluded = true; + + // logs ingestion + @Parameter( + names = {"--filebeatPort"}, + description = "Port on which to listen for filebeat data.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.FILEB) + int filebeatPort = 0; + + @Parameter( + names = {"--rawLogsPort"}, + description = "Port on which to listen for raw logs data.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.RAWLOGS) + int rawLogsPort = 0; + + @Parameter( + names = {"--rawLogsMaxReceivedLength"}, + description = "Maximum line length for received raw logs (Default: 4096)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.RAWLOGS) + int rawLogsMaxReceivedLength = 4096; + + @Parameter( + names = {"--rawLogsHttpBufferSize"}, + description = + "Maximum allowed request size (in bytes) for" + + " incoming HTTP requests with raw logs (Default: 16MB)") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.RAWLOGS) + int rawLogsHttpBufferSize = 16 * 1024 * 1024; + + @Parameter( + names = {"--logsIngestionConfigFile"}, + description = "Location of logs ingestions config yaml file.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.RAWLOGS) + String logsIngestionConfigFile = "/etc/wavefront/wavefront-proxy/logsingestion.yaml"; + + /** + * Deprecated property, please use proxyname config field to set proxy name. Default hostname to + * FQDN of machine. Sent as internal metric tag with checkin. + */ + @Parameter( + names = {"--hostname"}, + description = "Hostname for the proxy. Defaults to FQDN of machine.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + @Deprecated + String hostname = getLocalHostName(); + + /** This property holds the proxy name. Default proxyname to FQDN of machine. */ + @Parameter( + names = {"--proxyname"}, + description = "Name for the proxy. Defaults to hostname.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String proxyname = getLocalHostName(); + + @Parameter( + names = {"--idFile"}, + description = + "File to read proxy id from. Defaults to ~/.dshell/id." + + "This property is ignored if ephemeral=true.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String idFile = null; + + @Parameter( + names = {"--allowRegex", "--whitelistRegex"}, + description = + "Regex pattern (java" + + ".util.regex) that graphite input lines must match to be accepted") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.METRICS) + String allowRegex; + + @Parameter( + names = {"--blockRegex", "--blacklistRegex"}, + description = + "Regex pattern (java" + + ".util.regex) that graphite input lines must NOT match to be accepted") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.METRICS) + String blockRegex; + + @Parameter( + names = {"--opentsdbPorts"}, + description = + "Comma-separated list of ports to listen on for opentsdb data. " + + "Binds, by default, to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TSDB) + String opentsdbPorts = ""; + + @Parameter( + names = {"--opentsdbAllowRegex", "--opentsdbWhitelistRegex"}, + description = + "Regex " + + "pattern (java.util.regex) that opentsdb input lines must match to be accepted") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TSDB) + String opentsdbAllowRegex; + + @Parameter( + names = {"--opentsdbBlockRegex", "--opentsdbBlacklistRegex"}, + description = + "Regex " + + "pattern (java.util.regex) that opentsdb input lines must NOT match to be accepted") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TSDB) + String opentsdbBlockRegex; + + @Parameter( + names = {"--picklePorts"}, + description = + "Comma-separated list of ports to listen on for pickle protocol " + + "data. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.OTHER) + String picklePorts; + + @Parameter( + names = {"--traceListenerPorts"}, + description = + "Comma-separated list of ports to listen on for trace " + "data. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES) + String traceListenerPorts; + + @Parameter( + names = {"--traceJaegerListenerPorts"}, + description = + "Comma-separated list of ports on which to listen " + + "on for jaeger thrift formatted data over TChannel protocol. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES_JAEGER) + String traceJaegerListenerPorts; + + @Parameter( + names = {"--traceJaegerHttpListenerPorts"}, + description = + "Comma-separated list of ports on which to listen " + + "on for jaeger thrift formatted data over HTTP. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES_JAEGER) + String traceJaegerHttpListenerPorts; + + @Parameter( + names = {"--traceJaegerGrpcListenerPorts"}, + description = + "Comma-separated list of ports on which to listen " + + "on for jaeger Protobuf formatted data over gRPC. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES_JAEGER) + String traceJaegerGrpcListenerPorts; + + @Parameter( + names = {"--traceJaegerApplicationName"}, + description = "Application name for Jaeger. Defaults to Jaeger.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES_JAEGER) + String traceJaegerApplicationName; + + @Parameter( + names = {"--traceZipkinListenerPorts"}, + description = + "Comma-separated list of ports on which to listen " + + "on for zipkin trace data over HTTP. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES_JAEGER) + String traceZipkinListenerPorts; + + @Parameter( + names = {"--traceZipkinApplicationName"}, + description = "Application name for Zipkin. Defaults to Zipkin.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES_ZIPKIN) + String traceZipkinApplicationName; + + @Parameter( + names = {"--customTracingListenerPorts"}, + description = + "Comma-separated list of ports to listen on spans from level 1 SDK. Helps " + + "derive RED metrics and for the span and heartbeat for corresponding application at " + + "proxy. Defaults: none") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES) + String customTracingListenerPorts = ""; + + @Parameter( + names = {"--customTracingApplicationName"}, + description = + "Application name to use " + + "for spans sent to customTracingListenerPorts when span doesn't have application tag. " + + "Defaults to defaultApp.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES) + String customTracingApplicationName; + + @Parameter( + names = {"--customTracingServiceName"}, + description = + "Service name to use for spans" + + " sent to customTracingListenerPorts when span doesn't have service tag. " + + "Defaults to defaultService.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES) + String customTracingServiceName; + + @Parameter( + names = {"--traceSamplingRate"}, + description = "Value between 0.0 and 1.0. " + "Defaults to 1.0 (allow all spans).") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES) + double traceSamplingRate = 1.0d; + + @Parameter( + names = {"--traceSamplingDuration"}, + description = + "Sample spans by duration in " + + "milliseconds. " + + "Defaults to 0 (ignore duration based sampling).") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES) + int traceSamplingDuration = 0; + + @Parameter( + names = {"--traceDerivedCustomTagKeys"}, + description = "Comma-separated " + "list of custom tag keys for trace derived RED metrics.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES) + String traceDerivedCustomTagKeys; + + @Parameter( + names = {"--backendSpanHeadSamplingPercentIgnored"}, + description = "Ignore " + "spanHeadSamplingPercent config in backend CustomerSettings") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.TRACES) + boolean backendSpanHeadSamplingPercentIgnored = false; + + @Parameter( + names = {"--pushRelayListenerPorts"}, + description = + "Comma-separated list of ports on which to listen " + + "on for proxy chaining data. For internal use. Defaults to none.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.METRICS) + String pushRelayListenerPorts; + + @Parameter( + names = {"--pushRelayHistogramAggregator"}, + description = + "If true, aggregate " + + "histogram distributions received on the relay port. Default: false", + arity = 1) + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + boolean pushRelayHistogramAggregator = false; + + @Parameter( + names = {"--pushRelayHistogramAggregatorAccumulatorSize"}, + description = + "Expected upper bound of concurrent accumulations, ~ #timeseries * #parallel " + + "reporting bins") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + long pushRelayHistogramAggregatorAccumulatorSize = 100000L; + + @Parameter( + names = {"--pushRelayHistogramAggregatorFlushSecs"}, + description = "Number of seconds to keep accumulator open for new samples.") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + int pushRelayHistogramAggregatorFlushSecs = 70; + + @Parameter( + names = {"--pushRelayHistogramAggregatorCompression"}, + description = + "Controls allowable number of centroids per histogram. Must be in [20;1000] " + + "range. Default: 32") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.HISTO) + short pushRelayHistogramAggregatorCompression = 32; + + @Parameter( + names = {"--splitPushWhenRateLimited"}, + description = + "Whether to split the push " + + "batch size when the push is rejected by Wavefront due to rate limit. Default false.", + arity = 1) + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.OTHER) + boolean splitPushWhenRateLimited = DEFAULT_SPLIT_PUSH_WHEN_RATE_LIMITED; + + @Parameter( + names = {"--retryBackoffBaseSeconds"}, + description = + "For exponential backoff " + + "when retry threads are throttled, the base (a in a^b) in seconds. Default 2.0") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.OTHER) + double retryBackoffBaseSeconds = DEFAULT_RETRY_BACKOFF_BASE_SECONDS; + + @Parameter( + names = {"--customSourceTags"}, + description = + "Comma separated list of point tag " + + "keys that should be treated as the source in Wavefront in the absence of a tag named " + + "`source` or `host`. Default: fqdn") + @ProxyConfigOption(category = Categories.INPUT, subCategory = SubCategories.SOURCETAGS) + String customSourceTags = "fqdn"; + + @Parameter( + names = {"--agentMetricsPointTags"}, + description = + "Additional point tags and their " + + " respective values to be included into internal agent's metrics " + + "(comma-separated list, ex: dc=west,env=prod). Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String agentMetricsPointTags = null; + + @Parameter( + names = {"--ephemeral"}, + arity = 1, + description = + "If true, this proxy is removed " + + "from Wavefront after 24 hours of inactivity. Default: true") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + boolean ephemeral = true; + + @Parameter( + names = {"--disableRdnsLookup"}, + arity = 1, + description = + "When receiving" + + " Wavefront-formatted data without source/host specified, use remote IP address as source " + + "instead of trying to resolve the DNS name. Default false.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + boolean disableRdnsLookup = false; + + @Parameter( + names = {"--gzipCompression"}, + arity = 1, + description = + "If true, enables gzip " + "compression for traffic sent to Wavefront (Default: true)") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.CONF) + boolean gzipCompression = true; + + @Parameter( + names = {"--gzipCompressionLevel"}, + description = + "If gzipCompression is enabled, " + + "sets compression level (1-9). Higher compression levels use more CPU. Default: 4") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.CONF) + int gzipCompressionLevel = 4; + + @Parameter( + names = {"--soLingerTime"}, + description = + "If provided, enables SO_LINGER with the specified linger time in seconds (default: SO_LINGER disabled)") + @ProxyConfigOption(category = Categories.OUTPUT, subCategory = SubCategories.CONF) + int soLingerTime = -1; + + @Parameter( + names = {"--proxyHost"}, + description = "Proxy host for routing traffic through a http proxy") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.HTTPPROXY) + String proxyHost = null; + + @Parameter( + names = {"--proxyPort"}, + description = "Proxy port for routing traffic through a http proxy") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.HTTPPROXY) + int proxyPort = 0; + + @Parameter( + names = {"--proxyUser"}, + description = + "If proxy authentication is necessary, this is the username that will be passed along") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.HTTPPROXY) + String proxyUser = null; + + @Parameter( + names = {"--proxyPassword"}, + description = + "If proxy authentication is necessary, this is the password that will be passed along") + @ProxyConfigOption( + category = Categories.GENERAL, + subCategory = SubCategories.HTTPPROXY, + secret = true) + String proxyPassword = null; + + @Parameter( + names = {"--httpUserAgent"}, + description = "Override User-Agent in request headers") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String httpUserAgent = null; + + @Parameter( + names = {"--httpConnectTimeout"}, + description = "Connect timeout in milliseconds (default: 5000)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int httpConnectTimeout = 5000; + + @Parameter( + names = {"--httpRequestTimeout"}, + description = "Request timeout in milliseconds (default: 10000)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int httpRequestTimeout = 10000; + + @Parameter( + names = {"--httpMaxConnTotal"}, + description = "Max connections to keep open (default: 200)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int httpMaxConnTotal = 200; + + @Parameter( + names = {"--httpMaxConnPerRoute"}, + description = "Max connections per route to keep open (default: 100)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int httpMaxConnPerRoute = 100; + + @Parameter( + names = {"--httpAutoRetries"}, + description = + "Number of times to retry http requests before queueing, set to 0 to disable (default: 3)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int httpAutoRetries = 3; + + @Parameter( + names = {"--preprocessorConfigFile"}, + description = + "Optional YAML file with additional configuration options for filtering and pre-processing points") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String preprocessorConfigFile = null; + + @Parameter( + names = {"--dataBackfillCutoffHours"}, + description = + "The cut-off point for what is considered a valid timestamp for back-dated points. Default is 8760 (1 year)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int dataBackfillCutoffHours = 8760; + + @Parameter( + names = {"--dataPrefillCutoffHours"}, + description = + "The cut-off point for what is considered a valid timestamp for pre-dated points. Default is 24 (1 day)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int dataPrefillCutoffHours = 24; + + @Parameter( + names = {"--authMethod"}, + converter = ProxyConfig.TokenValidationMethodConverter.class, + description = + "Authenticate all incoming HTTP requests and disables TCP streams when set to a value " + + "other than NONE. Allowed values are: NONE, STATIC_TOKEN, HTTP_GET, OAUTH2. Default: NONE") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + TokenValidationMethod authMethod = TokenValidationMethod.NONE; + + @Parameter( + names = {"--authTokenIntrospectionServiceUrl"}, + description = + "URL for the token introspection endpoint " + + "used to validate tokens for incoming HTTP requests. Required for authMethod = OAUTH2 (endpoint must be " + + "RFC7662-compliant) and authMethod = HTTP_GET (use {{token}} placeholder in the URL to pass token to the " + + "service, endpoint must return any 2xx status for valid tokens, any other response code is a fail)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String authTokenIntrospectionServiceUrl = null; + + @Parameter( + names = {"--authTokenIntrospectionAuthorizationHeader"}, + description = "Optional credentials for use " + "with the token introspection endpoint.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String authTokenIntrospectionAuthorizationHeader = null; + + @Parameter( + names = {"--authResponseRefreshInterval"}, + description = + "Cache TTL (in seconds) for token validation " + + "results (re-authenticate when expired). Default: 600 seconds") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int authResponseRefreshInterval = 600; + + @Parameter( + names = {"--authResponseMaxTtl"}, + description = + "Maximum allowed cache TTL (in seconds) for token " + + "validation results when token introspection service is unavailable. Default: 86400 seconds (1 day)") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int authResponseMaxTtl = 86400; + + @Parameter( + names = {"--authStaticToken"}, + description = + "Static token that is considered valid " + + "for all incoming HTTP requests. Required when authMethod = STATIC_TOKEN.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF, secret = true) + String authStaticToken = null; + + @Parameter( + names = {"--adminApiListenerPort"}, + description = "Enables admin port to control " + "healthcheck status per port. Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int adminApiListenerPort = 0; + + @Parameter( + names = {"--adminApiRemoteIpAllowRegex"}, + description = "Remote IPs must match " + "this regex to access admin API") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String adminApiRemoteIpAllowRegex = null; + + @Parameter( + names = {"--httpHealthCheckPorts"}, + description = + "Comma-delimited list of ports " + + "to function as standalone healthchecks. May be used independently of " + + "--httpHealthCheckAllPorts parameter. Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String httpHealthCheckPorts = null; + + @Parameter( + names = {"--httpHealthCheckAllPorts"}, + description = + "When true, all listeners that " + + "support HTTP protocol also respond to healthcheck requests. May be used independently of " + + "--httpHealthCheckPorts parameter. Default: false", + arity = 1) + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + boolean httpHealthCheckAllPorts = false; + + @Parameter( + names = {"--httpHealthCheckPath"}, + description = "Healthcheck's path, for example, " + "'/health'. Default: '/'") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String httpHealthCheckPath = "/"; + + @Parameter( + names = {"--httpHealthCheckResponseContentType"}, + description = + "Optional " + + "Content-Type to use in healthcheck response, for example, 'application/json'. Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String httpHealthCheckResponseContentType = null; + + @Parameter( + names = {"--httpHealthCheckPassStatusCode"}, + description = "HTTP status code for " + "'pass' health checks. Default: 200") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int httpHealthCheckPassStatusCode = 200; + + @Parameter( + names = {"--httpHealthCheckPassResponseBody"}, + description = + "Optional response " + "body to return with 'pass' health checks. Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String httpHealthCheckPassResponseBody = null; + + @Parameter( + names = {"--httpHealthCheckFailStatusCode"}, + description = "HTTP status code for " + "'fail' health checks. Default: 503") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + int httpHealthCheckFailStatusCode = 503; + + @Parameter( + names = {"--httpHealthCheckFailResponseBody"}, + description = + "Optional response " + "body to return with 'fail' health checks. Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String httpHealthCheckFailResponseBody = null; + + @Parameter( + names = {"--deltaCountersAggregationIntervalSeconds"}, + description = "Delay time for delta counter reporter. Defaults to 30 seconds.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + long deltaCountersAggregationIntervalSeconds = 30; + + @Parameter( + names = {"--deltaCountersAggregationListenerPorts"}, + description = + "Comma-separated list of ports to listen on Wavefront-formatted delta " + + "counters. Helps reduce outbound point rate by pre-aggregating delta counters at proxy." + + " Defaults: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String deltaCountersAggregationListenerPorts = ""; + + @Parameter( + names = {"--customTimestampTags"}, + description = + "Comma separated list of log tag " + + "keys that should be treated as the timestamp in Wavefront in the absence of a tag named " + + "`timestamp` or `log_timestamp`. Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String customTimestampTags = ""; + + @Parameter( + names = {"--customMessageTags"}, + description = + "Comma separated list of log tag " + + "keys that should be treated as the source in Wavefront in the absence of a tag named " + + "`message` or `text`. Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String customMessageTags = ""; + + @Parameter( + names = {"--customApplicationTags"}, + description = + "Comma separated list of log tag " + + "keys that should be treated as the application in Wavefront in the absence of a tag named " + + "`application`. Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String customApplicationTags = ""; + + @Parameter( + names = {"--customServiceTags"}, + description = + "Comma separated list of log tag " + + "keys that should be treated as the service in Wavefront in the absence of a tag named " + + "`service`. Default: none") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String customServiceTags = ""; + + @Parameter( + names = {"--customExceptionTags"}, + description = + "Comma separated list of log tag " + + "keys that should be treated as the exception in Wavefront in the absence of a " + + "tag named `exception`. Default: exception, error_name") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String customExceptionTags = ""; + + @Parameter( + names = {"--customLevelTags"}, + description = + "Comma separated list of log tag " + + "keys that should be treated as the log level in Wavefront in the absence of a " + + "tag named `level`. Default: level, log_level") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF) + String customLevelTags = ""; + + @Parameter( + names = {"--logServerIngestionToken"}, + description = "Log insight ingestion token, required to ingest logs to the log server.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.LOGS, secret = true) + String logServerIngestionToken = null; + + @Parameter( + names = {"--logServerIngestionURL"}, + description = "Log insight ingestion URL, required to ingest logs to the log server.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.LOGS, secret = true) + String logServerIngestionURL = null; + + boolean enableHyperlogsConvergedCsp = false; + boolean receivedLogServerDetails = true; + + @Parameter( + names = {"--cspBaseUrl"}, + description = "The CSP base url. By default prod.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF, secret = true) + String cspBaseUrl = "https://console.cloud.vmware.com"; + + @Parameter( + names = {"--cspAPIToken"}, + description = "The CSP api token.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF, secret = true) + String cspAPIToken = null; + + /** + * A client is created for a service defined in CSP. Machine/Cluster can use this client id/secret + * to authenticate with CSP. + * + *

If this value is present, the csp authentication will kick in by default. + */ + @Parameter( + names = {"--cspAppId"}, + description = "A server-to-server OAuth app id.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF, secret = true) + String cspAppId = null; + + @Parameter( + names = {"--cspAppSecret"}, + description = "A server-to-server OAuth app secret.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF, secret = true) + String cspAppSecret = null; + + @Parameter( + names = {"--cspOrgId"}, + description = "The CSP organisation identifier.") + @ProxyConfigOption(category = Categories.GENERAL, subCategory = SubCategories.CONF, secret = true) + String cspOrgId = null; +} diff --git a/proxy/src/main/java/com/wavefront/agent/ProxyMemoryGuard.java b/proxy/src/main/java/com/wavefront/agent/ProxyMemoryGuard.java new file mode 100644 index 000000000..2579577bd --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/ProxyMemoryGuard.java @@ -0,0 +1,68 @@ +package com.wavefront.agent; + +import static com.wavefront.common.Utils.lazySupplier; + +import com.google.common.base.Preconditions; +import com.wavefront.common.TaggedMetricName; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Counter; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryNotificationInfo; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryType; +import java.util.function.Supplier; +import java.util.logging.Logger; +import javax.annotation.Nonnull; +import javax.management.NotificationEmitter; + +/** + * Logic around OoM protection logic that drains memory buffers on MEMORY_THRESHOLD_EXCEEDED + * notifications, extracted from AbstractAgent. + * + * @author vasily@wavefront.com + */ +public class ProxyMemoryGuard { + private static final Logger logger = Logger.getLogger(ProxyMemoryGuard.class.getCanonicalName()); + + private final Supplier drainBuffersCount = + lazySupplier( + () -> + Metrics.newCounter( + new TaggedMetricName("buffer", "flush-count", "reason", "heapUsageThreshold"))); + + /** + * Set up the memory guard. + * + * @param flushTask runnable to invoke when in-memory buffers need to be drained to disk + * @param threshold memory usage threshold that is considered critical, 0 < threshold <= 1. + */ + public ProxyMemoryGuard(@Nonnull final Runnable flushTask, double threshold) { + Preconditions.checkArgument(threshold > 0, "ProxyMemoryGuard threshold must be > 0!"); + Preconditions.checkArgument(threshold <= 1, "ProxyMemoryGuard threshold must be <= 1!"); + MemoryPoolMXBean tenuredGenPool = getTenuredGenPool(); + if (tenuredGenPool == null) return; + tenuredGenPool.setUsageThreshold((long) (tenuredGenPool.getUsage().getMax() * threshold)); + + NotificationEmitter emitter = (NotificationEmitter) ManagementFactory.getMemoryMXBean(); + emitter.addNotificationListener( + (notification, obj) -> { + if (notification.getType().equals(MemoryNotificationInfo.MEMORY_THRESHOLD_EXCEEDED)) { + logger.warning("Heap usage threshold exceeded - draining buffers to disk!"); + drainBuffersCount.get().inc(); + flushTask.run(); + logger.info("Draining buffers to disk: finished"); + } + }, + null, + null); + } + + private MemoryPoolMXBean getTenuredGenPool() { + for (MemoryPoolMXBean pool : ManagementFactory.getMemoryPoolMXBeans()) { + if (pool.getType() == MemoryType.HEAP && pool.isUsageThresholdSupported()) { + return pool; + } + } + return null; + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/ProxySendConfigScheduler.java b/proxy/src/main/java/com/wavefront/agent/ProxySendConfigScheduler.java new file mode 100644 index 000000000..0e5229b0e --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/ProxySendConfigScheduler.java @@ -0,0 +1,48 @@ +package com.wavefront.agent; + +import com.wavefront.agent.api.APIContainer; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class ProxySendConfigScheduler { + private static final Logger logger = + Logger.getLogger(ProxySendConfigScheduler.class.getCanonicalName()); + private boolean successful = false; + private final ScheduledExecutorService executor; + private final Runnable task; + + public ProxySendConfigScheduler( + APIContainer apiContainer, UUID proxyId, ProxyConfig proxyConfig) { + executor = Executors.newScheduledThreadPool(1); + task = + () -> { + try { + apiContainer + .getProxyV2APIForTenant(APIContainer.CENTRAL_TENANT_NAME) + .proxySaveConfig(proxyId, proxyConfig.getJsonConfig()); + successful = true; + logger.info("Configuration sent to the server successfully."); + } catch (javax.ws.rs.NotFoundException ex) { + logger.log(Level.FINE, "'proxySaveConfig' api end point not found", ex); + successful = true; + } catch (Throwable e) { + logger.severe( + "Can't send the Proxy configuration to the server, retrying in 60 seconds. " + + e.getMessage()); + logger.log(Level.FINE, "Exception: ", e); + } + + if (successful) { + executor.shutdown(); + } + }; + } + + public void start() { + executor.scheduleAtFixedRate(task, 0, 60, TimeUnit.SECONDS); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/ProxyUtil.java b/proxy/src/main/java/com/wavefront/agent/ProxyUtil.java new file mode 100644 index 000000000..3e6f35204 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/ProxyUtil.java @@ -0,0 +1,183 @@ +package com.wavefront.agent; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; +import com.wavefront.agent.channel.ConnectionTrackingHandler; +import com.wavefront.agent.channel.IdleStateEventHandler; +import com.wavefront.agent.channel.PlainTextOrHttpFrameDecoder; +import com.wavefront.common.TaggedMetricName; +import com.yammer.metrics.Metrics; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.codec.http.cors.CorsConfig; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.timeout.IdleStateHandler; +import java.io.File; +import java.io.IOException; +import java.util.Objects; +import java.util.UUID; +import java.util.function.Supplier; +import java.util.logging.Logger; +import javax.annotation.Nullable; + +/** + * Miscellaneous support methods for running Wavefront proxy. + * + * @author vasily@wavefront.com + */ +abstract class ProxyUtil { + protected static final Logger logger = Logger.getLogger("proxy"); + + private ProxyUtil() {} + + /** + * Gets or creates proxy id for this machine. + * + * @param proxyConfig proxy configuration + * @return proxy ID + */ + static UUID getOrCreateProxyId(ProxyConfig proxyConfig) { + if (proxyConfig.isEphemeral()) { + UUID proxyId = UUID.randomUUID(); // don't need to store one + logger.info("Ephemeral proxy id created: " + proxyId); + return proxyId; + } else { + return getOrCreateProxyIdFromFile(proxyConfig.getIdFile()); + } + } + + /** + * Read or create proxy id for this machine. Reads the UUID from specified file, or from + * ~/.dshell/id if idFileName is null. + * + * @param idFileName file name to read proxy ID from. + * @return proxy id + */ + static UUID getOrCreateProxyIdFromFile(@Nullable String idFileName) { + File proxyIdFile; + UUID proxyId = UUID.randomUUID(); + if (idFileName != null) { + proxyIdFile = new File(idFileName); + } else { + File userHome = new File(System.getProperty("user.home")); + if (!userHome.exists() || !userHome.isDirectory()) { + throw new RuntimeException("Cannot read from user.home, quitting"); + } + File configDirectory = new File(userHome, ".dshell"); + if (configDirectory.exists()) { + if (!configDirectory.isDirectory()) { + throw new RuntimeException(configDirectory + " must be a directory!"); + } + } else { + if (!configDirectory.mkdir()) { + throw new RuntimeException("Cannot create .dshell directory under " + userHome); + } + } + proxyIdFile = new File(configDirectory, "id"); + } + if (proxyIdFile.exists()) { + if (proxyIdFile.isFile()) { + try { + proxyId = + UUID.fromString( + Objects.requireNonNull( + Files.asCharSource(proxyIdFile, Charsets.UTF_8).readFirstLine())); + logger.info("Proxy Id read from file: " + proxyId); + } catch (IllegalArgumentException ex) { + throw new RuntimeException( + "Cannot read proxy id from " + proxyIdFile + ", content is malformed"); + } catch (IOException e) { + throw new RuntimeException("Cannot read from " + proxyIdFile, e); + } + } else { + throw new RuntimeException(proxyIdFile + " is not a file!"); + } + } else { + logger.info("Proxy Id created: " + proxyId); + try { + Files.asCharSink(proxyIdFile, Charsets.UTF_8).write(proxyId.toString()); + } catch (IOException e) { + throw new RuntimeException("Cannot write to " + proxyIdFile); + } + } + return proxyId; + } + + /** + * Create a {@link ChannelInitializer} with a single {@link ChannelHandler}, wrapped in {@link + * PlainTextOrHttpFrameDecoder}. + * + * @param channelHandler handler + * @param port port number. + * @param messageMaxLength maximum line length for line-based protocols. + * @param httpRequestBufferSize maximum request size for HTTP POST. + * @param idleTimeout idle timeout in seconds. + * @param sslContext SSL context. + * @param corsConfig enables CORS when {@link CorsConfig} is specified. + * @return channel initializer + */ + static ChannelInitializer createInitializer( + ChannelHandler channelHandler, + int port, + int messageMaxLength, + int httpRequestBufferSize, + int idleTimeout, + @Nullable SslContext sslContext, + @Nullable CorsConfig corsConfig) { + return createInitializer( + ImmutableList.of( + () -> + new PlainTextOrHttpFrameDecoder( + channelHandler, corsConfig, messageMaxLength, httpRequestBufferSize)), + port, + idleTimeout, + sslContext); + } + + /** + * Create a {@link ChannelInitializer} with multiple dynamically created {@link ChannelHandler} + * objects. + * + * @param channelHandlerSuppliers Suppliers of ChannelHandlers. + * @param port port number. + * @param idleTimeout idle timeout in seconds. + * @param sslContext SSL context. + * @return channel initializer + */ + static ChannelInitializer createInitializer( + Iterable> channelHandlerSuppliers, + int port, + int idleTimeout, + @Nullable SslContext sslContext) { + String strPort = String.valueOf(port); + ChannelHandler idleStateEventHandler = + new IdleStateEventHandler( + Metrics.newCounter( + new TaggedMetricName("listeners", "connections.idle.closed", "port", strPort))); + ChannelHandler connectionTracker = + new ConnectionTrackingHandler( + Metrics.newCounter( + new TaggedMetricName("listeners", "connections.accepted", "port", strPort)), + Metrics.newCounter( + new TaggedMetricName("listeners", "connections.active", "port", strPort))); + if (sslContext != null) { + logger.info("TLS enabled on port: " + port); + } + return new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) { + ChannelPipeline pipeline = ch.pipeline(); + if (sslContext != null) { + pipeline.addLast(sslContext.newHandler(ch.alloc())); + } + pipeline.addFirst("idlehandler", new IdleStateHandler(idleTimeout, 0, 0)); + pipeline.addLast("idlestateeventhandler", idleStateEventHandler); + pipeline.addLast("connectiontracker", connectionTracker); + channelHandlerSuppliers.forEach(x -> pipeline.addLast(x.get())); + } + }; + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/PushAgent.java b/proxy/src/main/java/com/wavefront/agent/PushAgent.java index c0ff5b784..13a80d03c 100644 --- a/proxy/src/main/java/com/wavefront/agent/PushAgent.java +++ b/proxy/src/main/java/com/wavefront/agent/PushAgent.java @@ -1,123 +1,105 @@ package com.wavefront.agent; +import static com.google.common.base.Preconditions.checkArgument; +import static com.wavefront.agent.ProxyUtil.createInitializer; +import static com.wavefront.agent.api.APIContainer.CENTRAL_TENANT_NAME; +import static com.wavefront.agent.data.EntityProperties.NO_RATE_LIMIT; +import static com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl.VALID_HISTOGRAMS_LOGGER; +import static com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl.VALID_POINTS_LOGGER; +import static com.wavefront.common.Utils.*; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; -import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; - -import com.squareup.tape.ObjectQueue; +import com.google.common.util.concurrent.RecyclableRateLimiter; import com.tdunning.math.stats.AgentDigest; import com.tdunning.math.stats.AgentDigest.AgentDigestMarshaller; import com.uber.tchannel.api.TChannel; import com.uber.tchannel.channels.Connection; -import com.wavefront.agent.channel.CachingGraphiteHostAnnotator; -import com.wavefront.agent.channel.ConnectionTrackingHandler; -import com.wavefront.agent.channel.IdleStateEventHandler; -import com.wavefront.agent.channel.PlainTextOrHttpFrameDecoder; +import com.wavefront.agent.auth.TokenAuthenticator; +import com.wavefront.agent.auth.TokenAuthenticatorBuilder; +import com.wavefront.agent.channel.CachingHostnameLookupResolver; +import com.wavefront.agent.channel.HealthCheckManager; +import com.wavefront.agent.channel.HealthCheckManagerImpl; +import com.wavefront.agent.channel.SharedGraphiteHostAnnotator; import com.wavefront.agent.config.ConfigurationException; +import com.wavefront.agent.data.EntityProperties; +import com.wavefront.agent.data.EntityPropertiesFactory; +import com.wavefront.agent.data.QueueingReason; import com.wavefront.agent.formatter.GraphiteFormatter; -import com.wavefront.agent.handlers.ReportableEntityHandlerFactory; -import com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl; -import com.wavefront.agent.handlers.SenderTaskFactory; -import com.wavefront.agent.handlers.SenderTaskFactoryImpl; -import com.wavefront.agent.histogram.HistogramLineIngester; -import com.wavefront.agent.histogram.MapLoader; -import com.wavefront.agent.histogram.PointHandlerDispatcher; -import com.wavefront.agent.histogram.QueuingChannelHandler; -import com.wavefront.agent.histogram.Utils; -import com.wavefront.agent.histogram.Utils.HistogramKey; -import com.wavefront.agent.histogram.Utils.HistogramKeyMarshaller; +import com.wavefront.agent.handlers.*; +import com.wavefront.agent.histogram.*; +import com.wavefront.agent.histogram.HistogramUtils.HistogramKeyMarshaller; import com.wavefront.agent.histogram.accumulator.AccumulationCache; -import com.wavefront.agent.histogram.accumulator.AccumulationTask; -import com.wavefront.agent.histogram.tape.TapeDeck; -import com.wavefront.agent.histogram.tape.TapeStringListConverter; -import com.wavefront.agent.logsharvesting.FilebeatIngester; -import com.wavefront.agent.logsharvesting.LogsIngester; -import com.wavefront.agent.logsharvesting.RawLogsIngester; -import com.wavefront.agent.listeners.ChannelByteArrayHandler; -import com.wavefront.agent.listeners.DataDogPortUnificationHandler; -import com.wavefront.agent.listeners.JaegerThriftCollectorHandler; -import com.wavefront.agent.listeners.JsonMetricsEndpoint; -import com.wavefront.agent.listeners.OpenTSDBPortUnificationHandler; -import com.wavefront.agent.listeners.RelayPortUnificationHandler; -import com.wavefront.agent.listeners.TracePortUnificationHandler; -import com.wavefront.agent.listeners.WavefrontPortUnificationHandler; -import com.wavefront.agent.listeners.WriteHttpJsonMetricsEndpoint; -import com.wavefront.agent.listeners.ZipkinPortUnificationHandler; +import com.wavefront.agent.histogram.accumulator.Accumulator; +import com.wavefront.agent.histogram.accumulator.AgentDigestFactory; +import com.wavefront.agent.listeners.*; +import com.wavefront.agent.listeners.otlp.OtlpGrpcMetricsHandler; +import com.wavefront.agent.listeners.otlp.OtlpGrpcTraceHandler; +import com.wavefront.agent.listeners.otlp.OtlpHttpHandler; +import com.wavefront.agent.listeners.tracing.*; import com.wavefront.agent.logsharvesting.FilebeatIngester; import com.wavefront.agent.logsharvesting.LogsIngester; -import com.wavefront.agent.logsharvesting.RawLogsIngester; -import com.wavefront.agent.channel.CachingGraphiteHostAnnotator; -import com.wavefront.agent.channel.ConnectionTrackingHandler; -import com.wavefront.agent.channel.IdleStateEventHandler; -import com.wavefront.agent.channel.PlainTextOrHttpFrameDecoder; -import com.wavefront.agent.preprocessor.ReportPointAddPrefixTransformer; -import com.wavefront.agent.preprocessor.ReportPointTimestampInRangeFilter; +import com.wavefront.api.agent.preprocessor.PreprocessorRuleMetrics; +import com.wavefront.api.agent.preprocessor.ReportPointAddPrefixTransformer; +import com.wavefront.api.agent.preprocessor.ReportPointTimestampInRangeFilter; +import com.wavefront.api.agent.preprocessor.SpanSanitizeTransformer; +import com.wavefront.agent.queueing.*; +import com.wavefront.agent.sampler.SpanSampler; import com.wavefront.agent.sampler.SpanSamplerUtils; import com.wavefront.api.agent.AgentConfiguration; -import com.wavefront.api.agent.Constants; import com.wavefront.common.NamedThreadFactory; import com.wavefront.common.TaggedMetricName; import com.wavefront.data.ReportableEntityType; -import com.wavefront.data.Validation; -import com.wavefront.ingester.Decoder; -import com.wavefront.ingester.GraphiteDecoder; -import com.wavefront.ingester.HistogramDecoder; -import com.wavefront.ingester.OpenTSDBDecoder; -import com.wavefront.ingester.PickleProtocolDecoder; -import com.wavefront.ingester.ReportPointDecoderWrapper; -import com.wavefront.ingester.ReportSourceTagDecoder; -import com.wavefront.ingester.ReportableEntityDecoder; -import com.wavefront.ingester.SpanDecoder; -import com.wavefront.ingester.StreamIngester; -import com.wavefront.ingester.TcpIngester; +import com.wavefront.ingester.*; +import com.wavefront.internal.reporter.WavefrontInternalReporter; import com.wavefront.metrics.ExpectedAgentMetric; +import com.wavefront.sdk.common.WavefrontSender; import com.wavefront.sdk.entities.tracing.sampling.CompositeSampler; +import com.wavefront.sdk.entities.tracing.sampling.RateSampler; import com.wavefront.sdk.entities.tracing.sampling.Sampler; import com.yammer.metrics.Metrics; import com.yammer.metrics.core.Counter; - -import net.openhft.chronicle.map.ChronicleMap; - -import org.apache.commons.lang.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.client.HttpClientBuilder; -import org.logstash.beats.Server; - +import io.grpc.netty.NettyServerBuilder; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelOption; +import io.netty.handler.codec.LengthFieldBasedFrameDecoder; +import io.netty.handler.codec.bytes.ByteArrayDecoder; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.cors.CorsConfig; +import io.netty.handler.codec.http.cors.CorsConfigBuilder; +import io.netty.handler.ssl.SslContext; import java.io.File; -import java.io.IOException; +import java.io.FileNotFoundException; import java.net.BindException; import java.net.InetAddress; import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.IdentityHashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.logging.Level; - +import java.util.logging.Logger; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; import javax.annotation.Nullable; - -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelInboundHandler; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.codec.LengthFieldBasedFrameDecoder; -import io.netty.handler.timeout.IdleStateHandler; +import net.openhft.chronicle.map.ChronicleMap; +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; +import org.apache.http.impl.client.HttpClientBuilder; +import org.logstash.beats.Server; +import wavefront.report.Histogram; import wavefront.report.ReportPoint; -import static com.google.common.base.Preconditions.checkArgument; - /** * Push-only Agent. * @@ -125,772 +107,2068 @@ */ public class PushAgent extends AbstractAgent { - protected final List managedThreads = new ArrayList<>(); - protected final IdentityHashMap, Object> childChannelOptions = new IdentityHashMap<>(); + protected final Map listeners = new HashMap<>(); + protected final IdentityHashMap, Object> childChannelOptions = + new IdentityHashMap<>(); protected ScheduledExecutorService histogramExecutor; - protected ScheduledExecutorService histogramScanExecutor; protected ScheduledExecutorService histogramFlushExecutor; - protected final Counter bindErrors = Metrics.newCounter(ExpectedAgentMetric.LISTENERS_BIND_ERRORS.metricName); - private volatile ReportableEntityDecoder wavefrontDecoder; - protected CachingGraphiteHostAnnotator remoteHostAnnotator; - protected SenderTaskFactory senderTaskFactory; - protected ReportableEntityHandlerFactory handlerFactory; - - public static void main(String[] args) throws IOException { + @VisibleForTesting protected List histogramFlushRunnables = new ArrayList<>(); + protected final Counter bindErrors = + Metrics.newCounter(ExpectedAgentMetric.LISTENERS_BIND_ERRORS.metricName); + protected TaskQueueFactory taskQueueFactory; + protected SharedGraphiteHostAnnotator remoteHostAnnotator; + protected Function hostnameResolver; + protected SenderTaskFactoryImpl senderTaskFactory; + protected QueueingFactory queueingFactory; + protected Function histogramRecompressor = null; + protected ReportableEntityHandlerFactoryImpl handlerFactory; + protected ReportableEntityHandlerFactory deltaCounterHandlerFactory; + protected HealthCheckManager healthCheckManager; + protected TokenAuthenticator tokenAuthenticator = TokenAuthenticator.DUMMY_AUTHENTICATOR; + protected final Supplier>> + decoderSupplier = + lazySupplier( + () -> + ImmutableMap.>builder() + .put( + ReportableEntityType.POINT, + new ReportPointDecoder( + () -> "unknown", proxyConfig.getCustomSourceTags())) + .put(ReportableEntityType.SOURCE_TAG, new ReportSourceTagDecoder()) + .put( + ReportableEntityType.HISTOGRAM, + new ReportPointDecoderWrapper(new HistogramDecoder("unknown"))) + .put(ReportableEntityType.TRACE, new SpanDecoder("unknown")) + .put(ReportableEntityType.TRACE_SPAN_LOGS, new SpanLogsDecoder()) + .put(ReportableEntityType.EVENT, new EventDecoder()) + .put( + ReportableEntityType.LOGS, + new ReportLogDecoder( + () -> "unknown", + proxyConfig.getCustomSourceTags(), + proxyConfig.getCustomTimestampTags(), + proxyConfig.getCustomMessageTags(), + proxyConfig.getCustomApplicationTags(), + proxyConfig.getCustomServiceTags(), + proxyConfig.getCustomLevelTags(), + proxyConfig.getCustomExceptionTags())) + .build()); + // default rate sampler which always samples. + protected final RateSampler rateSampler = new RateSampler(1.0d); + private Logger blockedPointsLogger; + private Logger blockedHistogramsLogger; + private Logger blockedSpansLogger; + private Logger blockedLogsLogger; + private AtomicBoolean usingLocalFileRules = new AtomicBoolean(true); + + public static void main(String[] args) { // Start the ssh daemon + String versionStr = + "Wavefront Proxy version " + + getBuildVersion() + + " (pkg:" + + getPackage() + + ")" + + ", runtime: " + + getJavaVersion(); + logger.info(versionStr); new PushAgent().start(args); } - public PushAgent() { - super(false, true); - } - - @Deprecated - protected PushAgent(boolean reportAsPushAgent) { - super(false, reportAsPushAgent); - } - - @VisibleForTesting - protected ReportableEntityDecoder getDecoderInstance() { - synchronized(PushAgent.class) { - if (wavefrontDecoder == null) { - wavefrontDecoder = new ReportPointDecoderWrapper(new GraphiteDecoder("unknown", customSourceTags)); - } - return wavefrontDecoder; + protected void setupMemoryGuard() { + if (proxyConfig.getMemGuardFlushThreshold() > 0) { + float threshold = ((float) proxyConfig.getMemGuardFlushThreshold() / 100); + new ProxyMemoryGuard( + () -> senderTaskFactory.drainBuffersToQueue(QueueingReason.MEMORY_PRESSURE), threshold); } } @Override - protected void startListeners() { - if (soLingerTime >= 0) { - childChannelOptions.put(ChannelOption.SO_LINGER, soLingerTime); + protected void startListeners() throws Exception { + blockedPointsLogger = Logger.getLogger(proxyConfig.getBlockedPointsLoggerName()); + blockedHistogramsLogger = Logger.getLogger(proxyConfig.getBlockedHistogramsLoggerName()); + blockedSpansLogger = Logger.getLogger(proxyConfig.getBlockedSpansLoggerName()); + blockedLogsLogger = Logger.getLogger(proxyConfig.getBlockedLogsLoggerName()); + + if (proxyConfig.getSoLingerTime() >= 0) { + childChannelOptions.put(ChannelOption.SO_LINGER, proxyConfig.getSoLingerTime()); } - remoteHostAnnotator = new CachingGraphiteHostAnnotator(customSourceTags, disableRdnsLookup); - senderTaskFactory = new SenderTaskFactoryImpl(agentAPI, agentId, pushRateLimiter, - pushFlushInterval, pushFlushMaxPoints, pushMemoryBufferLimit); - handlerFactory = new ReportableEntityHandlerFactoryImpl(senderTaskFactory, pushBlockedSamples, flushThreads); - - if (pushListenerPorts != null) { - Iterable ports = Splitter.on(",").omitEmptyStrings().trimResults().split(pushListenerPorts); - for (String strPort : ports) { - startGraphiteListener(strPort, handlerFactory, remoteHostAnnotator); - logger.info("listening on port: " + strPort + " for Wavefront metrics"); - } + hostnameResolver = + new CachingHostnameLookupResolver( + proxyConfig.isDisableRdnsLookup(), ExpectedAgentMetric.RDNS_CACHE_SIZE.metricName); + + if (proxyConfig.isSqsQueueBuffer()) { + taskQueueFactory = + new SQSQueueFactoryImpl( + proxyConfig.getSqsQueueNameTemplate(), + proxyConfig.getSqsQueueRegion(), + proxyConfig.getSqsQueueIdentifier(), + proxyConfig.isPurgeBuffer()); + } else { + taskQueueFactory = + new TaskQueueFactoryImpl( + proxyConfig.getBufferFile(), + proxyConfig.isPurgeBuffer(), + proxyConfig.isDisableBufferSharding(), + proxyConfig.getBufferShardSize()); } - { - // Histogram bootstrap. - Iterator histMinPorts = Strings.isNullOrEmpty(histogramMinuteListenerPorts) ? - Collections.emptyIterator() : - Splitter.on(",").omitEmptyStrings().trimResults().split(histogramMinuteListenerPorts).iterator(); - - Iterator histHourPorts = Strings.isNullOrEmpty(histogramHourListenerPorts) ? - Collections.emptyIterator() : - Splitter.on(",").omitEmptyStrings().trimResults().split(histogramHourListenerPorts).iterator(); - - Iterator histDayPorts = Strings.isNullOrEmpty(histogramDayListenerPorts) ? - Collections.emptyIterator() : - Splitter.on(",").omitEmptyStrings().trimResults().split(histogramDayListenerPorts).iterator(); - - Iterator histDistPorts = Strings.isNullOrEmpty(histogramDistListenerPorts) ? - Collections.emptyIterator() : - Splitter.on(",").omitEmptyStrings().trimResults().split(histogramDistListenerPorts).iterator(); - - int activeHistogramAggregationTypes = (histDayPorts.hasNext() ? 1 : 0) + (histHourPorts.hasNext() ? 1 : 0) + - (histMinPorts.hasNext() ? 1 : 0) + (histDistPorts.hasNext() ? 1 : 0); - if (activeHistogramAggregationTypes > 0) { /*Histograms enabled*/ - histogramExecutor = Executors.newScheduledThreadPool(1 + activeHistogramAggregationTypes, - new NamedThreadFactory("histogram-service")); - histogramFlushExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() / 2, - new NamedThreadFactory("histogram-flush")); - histogramScanExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() / 2, - new NamedThreadFactory("histogram-scan")); - managedExecutors.add(histogramExecutor); - managedExecutors.add(histogramFlushExecutor); - managedExecutors.add(histogramScanExecutor); - - File baseDirectory = new File(histogramStateDirectory); - if (persistMessages || persistAccumulator) { - // Check directory - checkArgument(baseDirectory.isDirectory(), baseDirectory.getAbsolutePath() + " must be a directory!"); - checkArgument(baseDirectory.canWrite(), baseDirectory.getAbsolutePath() + " must be write-able!"); - } - - // Central dispatch - PointHandler histogramHandler = new PointHandlerImpl( - "histogram ports", - pushValidationLevel, - pushBlockedSamples, - prefix, - getFlushTasks(Constants.PUSH_FORMAT_HISTOGRAM, "histogram ports")); - - // Input queue factory - TapeDeck> accumulatorDeck = new TapeDeck<>( - persistMessagesCompression - ? TapeStringListConverter.getCompressionEnabledInstance() - : TapeStringListConverter.getDefaultInstance(), - persistMessages); - - Decoder distributionDecoder = new HistogramDecoder("unknown"); - Decoder graphiteDecoder = new GraphiteDecoder("unknown", customSourceTags); - if (histMinPorts.hasNext()) { - startHistogramListeners(histMinPorts, graphiteDecoder, histogramHandler, accumulatorDeck, - Utils.Granularity.MINUTE, histogramMinuteFlushSecs, histogramMinuteAccumulators, - histogramMinuteMemoryCache, baseDirectory, histogramMinuteAccumulatorSize, histogramMinuteAvgKeyBytes, - histogramMinuteAvgDigestBytes, histogramMinuteCompression); - } + remoteHostAnnotator = + new SharedGraphiteHostAnnotator(proxyConfig.getCustomSourceTags(), hostnameResolver); + queueingFactory = + new QueueingFactoryImpl( + apiContainer, agentId, taskQueueFactory, entityPropertiesFactoryMap); + senderTaskFactory = + new SenderTaskFactoryImpl( + apiContainer, agentId, taskQueueFactory, queueingFactory, entityPropertiesFactoryMap); + // MONIT-25479: when multicasting histogram, use the central cluster histogram accuracy + if (proxyConfig.isHistogramPassthroughRecompression()) { + histogramRecompressor = + new HistogramRecompressor( + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .getGlobalProperties() + .getHistogramStorageAccuracy()); + } + handlerFactory = + new ReportableEntityHandlerFactoryImpl( + senderTaskFactory, + proxyConfig.getPushBlockedSamples(), + validationConfiguration, + blockedPointsLogger, + blockedHistogramsLogger, + blockedSpansLogger, + histogramRecompressor, + entityPropertiesFactoryMap, + blockedLogsLogger); + if (proxyConfig.isTrafficShaping()) { + new TrafficShapingRateLimitAdjuster( + entityPropertiesFactoryMap, + proxyConfig.getTrafficShapingWindowSeconds(), + proxyConfig.getTrafficShapingHeadroom()) + .start(); + } + healthCheckManager = new HealthCheckManagerImpl(proxyConfig); + tokenAuthenticator = configureTokenAuthenticator(); - if (histHourPorts.hasNext()) { - startHistogramListeners(histHourPorts, graphiteDecoder, histogramHandler, accumulatorDeck, - Utils.Granularity.HOUR, histogramHourFlushSecs, histogramHourAccumulators, - histogramHourMemoryCache, baseDirectory, histogramHourAccumulatorSize, histogramHourAvgKeyBytes, - histogramHourAvgDigestBytes, histogramHourCompression); - } + shutdownTasks.add(() -> senderTaskFactory.shutdown()); + shutdownTasks.add(() -> senderTaskFactory.drainBuffersToQueue(null)); - if (histDayPorts.hasNext()) { - startHistogramListeners(histDayPorts, graphiteDecoder, histogramHandler, accumulatorDeck, - Utils.Granularity.DAY, histogramDayFlushSecs, histogramDayAccumulators, - histogramDayMemoryCache, baseDirectory, histogramDayAccumulatorSize, histogramDayAvgKeyBytes, - histogramDayAvgDigestBytes, histogramDayCompression); - } + SpanSampler spanSampler = createSpanSampler(); - if (histDistPorts.hasNext()) { - startHistogramListeners(histDistPorts, distributionDecoder, histogramHandler, accumulatorDeck, - null, histogramDistFlushSecs, histogramDistAccumulators, - histogramDistMemoryCache, baseDirectory, histogramDistAccumulatorSize, histogramDistAvgKeyBytes, - histogramDistAvgDigestBytes, histogramDistCompression); - } - } + if (proxyConfig.getAdminApiListenerPort() > 0) { + startAdminListener(proxyConfig.getAdminApiListenerPort()); } - if (StringUtils.isNotBlank(graphitePorts) || StringUtils.isNotBlank(picklePorts)) { + csvToList(proxyConfig.getHttpHealthCheckPorts()) + .forEach(strPort -> startHealthCheckListener(Integer.parseInt(strPort))); + + csvToList(proxyConfig.getPushListenerPorts()) + .forEach( + strPort -> { + startGraphiteListener(strPort, handlerFactory, remoteHostAnnotator, spanSampler); + logger.info("listening on port: " + strPort + " for Wavefront metrics"); + }); + + csvToList(proxyConfig.getDeltaCountersAggregationListenerPorts()) + .forEach( + strPort -> { + startDeltaCounterListener( + strPort, remoteHostAnnotator, senderTaskFactory, spanSampler); + logger.info("listening on port: " + strPort + " for Wavefront delta counter metrics"); + }); + + bootstrapHistograms(spanSampler); + + if (StringUtils.isNotBlank(proxyConfig.getGraphitePorts()) + || StringUtils.isNotBlank(proxyConfig.getPicklePorts())) { if (tokenAuthenticator.authRequired()) { logger.warning("Graphite mode is not compatible with HTTP authentication, ignoring"); } else { - Preconditions.checkNotNull(graphiteFormat, "graphiteFormat must be supplied to enable graphite support"); - Preconditions.checkNotNull(graphiteDelimiters, "graphiteDelimiters must be supplied to enable graphite support"); - GraphiteFormatter graphiteFormatter = new GraphiteFormatter(graphiteFormat, graphiteDelimiters, - graphiteFieldsToRemove); - Iterable ports = Splitter.on(",").omitEmptyStrings().trimResults().split(graphitePorts); - for (String strPort : ports) { - preprocessors.forPort(strPort).forPointLine().addTransformer(0, graphiteFormatter); - startGraphiteListener(strPort, handlerFactory, null); - logger.info("listening on port: " + strPort + " for graphite metrics"); - } - if (picklePorts != null) { - Splitter.on(",").omitEmptyStrings().trimResults().split(picklePorts).forEach( - strPort -> { - PointHandler pointHandler = new PointHandlerImpl(strPort, pushValidationLevel, - pushBlockedSamples, getFlushTasks(strPort)); - startPickleListener(strPort, pointHandler, graphiteFormatter); - } - ); - } + Preconditions.checkNotNull( + proxyConfig.getGraphiteFormat(), + "graphiteFormat must be supplied to enable graphite support"); + Preconditions.checkNotNull( + proxyConfig.getGraphiteDelimiters(), + "graphiteDelimiters must be supplied to enable graphite support"); + GraphiteFormatter graphiteFormatter = + new GraphiteFormatter( + proxyConfig.getGraphiteFormat(), + proxyConfig.getGraphiteDelimiters(), + proxyConfig.getGraphiteFieldsToRemove()); + csvToList(proxyConfig.getGraphitePorts()) + .forEach( + strPort -> { + preprocessors + .getSystemPreprocessor(strPort) + .forPointLine() + .addTransformer(0, graphiteFormatter); + startGraphiteListener(strPort, handlerFactory, null, spanSampler); + logger.info("listening on port: " + strPort + " for graphite metrics"); + }); + csvToList(proxyConfig.getPicklePorts()) + .forEach(strPort -> startPickleListener(strPort, handlerFactory, graphiteFormatter)); } } - if (opentsdbPorts != null) { - Splitter.on(",").omitEmptyStrings().trimResults().split(opentsdbPorts).forEach( - strPort -> startOpenTsdbListener(strPort, handlerFactory) - ); - } - if (dataDogJsonPorts != null) { - HttpClient httpClient = HttpClientBuilder.create(). - useSystemProperties(). - setUserAgent(httpUserAgent). - setConnectionTimeToLive(1, TimeUnit.MINUTES). - setRetryHandler(new DefaultHttpRequestRetryHandler(httpAutoRetries, true)). - setDefaultRequestConfig( - RequestConfig.custom(). - setContentCompressionEnabled(true). - setRedirectsEnabled(true). - setConnectTimeout(httpConnectTimeout). - setConnectionRequestTimeout(httpConnectTimeout). - setSocketTimeout(httpRequestTimeout).build()). - build(); - - Splitter.on(",").omitEmptyStrings().trimResults().split(dataDogJsonPorts).forEach( - strPort -> startDataDogListener(strPort, handlerFactory, httpClient) - ); - } - if (traceListenerPorts != null) { - Splitter.on(",").omitEmptyStrings().trimResults().split(traceListenerPorts).forEach( - strPort -> startTraceListener(strPort, handlerFactory) - ); - } - if (traceJaegerListenerPorts != null) { - Splitter.on(",").omitEmptyStrings().trimResults().split(traceJaegerListenerPorts).forEach( - strPort -> startTraceJaegerListener(strPort, handlerFactory) - ); - } - if (pushRelayListenerPorts != null) { - Splitter.on(",").omitEmptyStrings().trimResults().split(pushRelayListenerPorts).forEach( - strPort -> startRelayListener(strPort, handlerFactory) - ); + + csvToList(proxyConfig.getOpentsdbPorts()) + .forEach(strPort -> startOpenTsdbListener(strPort, handlerFactory)); + + if (proxyConfig.getDataDogJsonPorts() != null) { + HttpClient httpClient = + HttpClientBuilder.create() + .useSystemProperties() + .setUserAgent(proxyConfig.getHttpUserAgent()) + .setConnectionTimeToLive(1, TimeUnit.MINUTES) + .setMaxConnPerRoute(100) + .setMaxConnTotal(100) + .setRetryHandler( + new DefaultHttpRequestRetryHandler(proxyConfig.getHttpAutoRetries(), true)) + .setDefaultRequestConfig( + RequestConfig.custom() + .setContentCompressionEnabled(true) + .setRedirectsEnabled(true) + .setConnectTimeout(proxyConfig.getHttpConnectTimeout()) + .setConnectionRequestTimeout(proxyConfig.getHttpConnectTimeout()) + .setSocketTimeout(proxyConfig.getHttpRequestTimeout()) + .build()) + .build(); + + csvToList(proxyConfig.getDataDogJsonPorts()) + .forEach(strPort -> startDataDogListener(strPort, handlerFactory, httpClient)); } - if (traceZipkinListenerPorts != null) { - Iterable ports = Splitter.on(",").omitEmptyStrings().trimResults().split(traceZipkinListenerPorts); - for (String strPort : ports) { - startTraceZipkinListener(strPort, handlerFactory); - logger.info("listening on port: " + traceZipkinListenerPorts + " for Zipkin trace data."); + + startDistributedTracingListeners(spanSampler); + + startOtlpListeners(spanSampler); + + csvToList(proxyConfig.getPushRelayListenerPorts()) + .forEach(strPort -> startRelayListener(strPort, handlerFactory, remoteHostAnnotator)); + csvToList(proxyConfig.getJsonListenerPorts()) + .forEach(strPort -> startJsonListener(strPort, handlerFactory)); + csvToList(proxyConfig.getWriteHttpJsonListenerPorts()) + .forEach(strPort -> startWriteHttpJsonListener(strPort, handlerFactory)); + + // Logs ingestion. + if (proxyConfig.getFilebeatPort() > 0 || proxyConfig.getRawLogsPort() > 0) { + if (loadLogsIngestionConfig() != null) { + logger.info("Initializing logs ingestion"); + try { + final LogsIngester logsIngester = + new LogsIngester( + handlerFactory, this::loadLogsIngestionConfig, proxyConfig.getPrefix()); + logsIngester.start(); + + if (proxyConfig.getFilebeatPort() > 0) { + startLogsIngestionListener(proxyConfig.getFilebeatPort(), logsIngester); + } + if (proxyConfig.getRawLogsPort() > 0) { + startRawLogsIngestionListener(proxyConfig.getRawLogsPort(), logsIngester); + } + } catch (ConfigurationException e) { + logger.log(Level.SEVERE, "Cannot start logsIngestion", e); + } + } else { + logger.warning("Cannot start logsIngestion: invalid configuration or no config specified"); } } - if (jsonListenerPorts != null) { - Splitter.on(",").omitEmptyStrings().trimResults().split(jsonListenerPorts).forEach(this::startJsonListener); - } - if (writeHttpJsonListenerPorts != null) { - Splitter.on(",").omitEmptyStrings().trimResults().split(writeHttpJsonListenerPorts). - forEach(this::startWriteHttpJsonListener); + setupMemoryGuard(); + } + + private void startDistributedTracingListeners(SpanSampler spanSampler) { + csvToList(proxyConfig.getTraceListenerPorts()) + .forEach(strPort -> startTraceListener(strPort, handlerFactory, spanSampler)); + csvToList(proxyConfig.getCustomTracingListenerPorts()) + .forEach( + strPort -> + startCustomTracingListener( + strPort, + handlerFactory, + new InternalProxyWavefrontClient(handlerFactory, strPort), + spanSampler)); + csvToList(proxyConfig.getTraceJaegerListenerPorts()) + .forEach( + strPort -> { + PreprocessorRuleMetrics ruleMetrics = + new PreprocessorRuleMetrics( + Metrics.newCounter( + new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), + null, + null); + preprocessors + .getSystemPreprocessor(strPort) + .forSpan() + .addTransformer(new SpanSanitizeTransformer(ruleMetrics)); + startTraceJaegerListener( + strPort, + handlerFactory, + new InternalProxyWavefrontClient(handlerFactory, strPort), + spanSampler); + }); + + csvToList(proxyConfig.getTraceJaegerGrpcListenerPorts()) + .forEach( + strPort -> { + PreprocessorRuleMetrics ruleMetrics = + new PreprocessorRuleMetrics( + Metrics.newCounter( + new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), + null, + null); + preprocessors + .getSystemPreprocessor(strPort) + .forSpan() + .addTransformer(new SpanSanitizeTransformer(ruleMetrics)); + startTraceJaegerGrpcListener( + strPort, + handlerFactory, + new InternalProxyWavefrontClient(handlerFactory, strPort), + spanSampler); + }); + csvToList(proxyConfig.getTraceJaegerHttpListenerPorts()) + .forEach( + strPort -> { + PreprocessorRuleMetrics ruleMetrics = + new PreprocessorRuleMetrics( + Metrics.newCounter( + new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), + null, + null); + preprocessors + .getSystemPreprocessor(strPort) + .forSpan() + .addTransformer(new SpanSanitizeTransformer(ruleMetrics)); + startTraceJaegerHttpListener( + strPort, + handlerFactory, + new InternalProxyWavefrontClient(handlerFactory, strPort), + spanSampler); + }); + csvToList(proxyConfig.getTraceZipkinListenerPorts()) + .forEach( + strPort -> { + PreprocessorRuleMetrics ruleMetrics = + new PreprocessorRuleMetrics( + Metrics.newCounter( + new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), + null, + null); + preprocessors + .getSystemPreprocessor(strPort) + .forSpan() + .addTransformer(new SpanSanitizeTransformer(ruleMetrics)); + startTraceZipkinListener( + strPort, + handlerFactory, + new InternalProxyWavefrontClient(handlerFactory, strPort), + spanSampler); + }); + } + + private void startOtlpListeners(SpanSampler spanSampler) { + csvToList(proxyConfig.getOtlpGrpcListenerPorts()) + .forEach( + strPort -> { + PreprocessorRuleMetrics ruleMetrics = + new PreprocessorRuleMetrics( + Metrics.newCounter( + new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), + null, + null); + preprocessors + .getSystemPreprocessor(strPort) + .forSpan() + .addTransformer(new SpanSanitizeTransformer(ruleMetrics)); + startOtlpGrpcListener( + strPort, + handlerFactory, + new InternalProxyWavefrontClient(handlerFactory, strPort), + spanSampler); + }); + + csvToList(proxyConfig.getOtlpHttpListenerPorts()) + .forEach( + strPort -> { + PreprocessorRuleMetrics ruleMetrics = + new PreprocessorRuleMetrics( + Metrics.newCounter( + new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), + null, + null); + preprocessors + .getSystemPreprocessor(strPort) + .forSpan() + .addTransformer(new SpanSanitizeTransformer(ruleMetrics)); + startOtlpHttpListener( + strPort, + handlerFactory, + new InternalProxyWavefrontClient(handlerFactory, strPort), + spanSampler); + }); + } + + private SpanSampler createSpanSampler() { + rateSampler.setSamplingRate( + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .getGlobalProperties() + .getTraceSamplingRate()); + Sampler durationSampler = + SpanSamplerUtils.getDurationSampler(proxyConfig.getTraceSamplingDuration()); + List samplers = SpanSamplerUtils.fromSamplers(rateSampler, durationSampler); + SpanSampler spanSampler = + new SpanSampler( + new CompositeSampler(samplers), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .getGlobalProperties() + .getActiveSpanSamplingPolicies()); + return spanSampler; + } + + private void bootstrapHistograms(SpanSampler spanSampler) throws Exception { + List histMinPorts = csvToList(proxyConfig.getHistogramMinuteListenerPorts()); + List histHourPorts = csvToList(proxyConfig.getHistogramHourListenerPorts()); + List histDayPorts = csvToList(proxyConfig.getHistogramDayListenerPorts()); + List histDistPorts = csvToList(proxyConfig.getHistogramDistListenerPorts()); + + int activeHistogramAggregationTypes = + (histDayPorts.size() > 0 ? 1 : 0) + + (histHourPorts.size() > 0 ? 1 : 0) + + (histMinPorts.size() > 0 ? 1 : 0) + + (histDistPorts.size() > 0 ? 1 : 0); + if (activeHistogramAggregationTypes > 0) { + /*Histograms enabled*/ + histogramExecutor = + Executors.newScheduledThreadPool( + 1 + activeHistogramAggregationTypes, new NamedThreadFactory("histogram-service")); + histogramFlushExecutor = + Executors.newScheduledThreadPool( + Runtime.getRuntime().availableProcessors() / 2, + new NamedThreadFactory("histogram-flush")); + managedExecutors.add(histogramExecutor); + managedExecutors.add(histogramFlushExecutor); + + File baseDirectory = new File(proxyConfig.getHistogramStateDirectory()); + + // Central dispatch + ReportableEntityHandler pointHandler = + handlerFactory.getHandler( + HandlerKey.of(ReportableEntityType.HISTOGRAM, "histogram_ports")); + + startHistogramListeners( + histMinPorts, + pointHandler, + remoteHostAnnotator, + Granularity.MINUTE, + proxyConfig.getHistogramMinuteFlushSecs(), + proxyConfig.isHistogramMinuteMemoryCache(), + baseDirectory, + proxyConfig.getHistogramMinuteAccumulatorSize(), + proxyConfig.getHistogramMinuteAvgKeyBytes(), + proxyConfig.getHistogramMinuteAvgDigestBytes(), + proxyConfig.getHistogramMinuteCompression(), + proxyConfig.isHistogramMinuteAccumulatorPersisted(), + spanSampler); + startHistogramListeners( + histHourPorts, + pointHandler, + remoteHostAnnotator, + Granularity.HOUR, + proxyConfig.getHistogramHourFlushSecs(), + proxyConfig.isHistogramHourMemoryCache(), + baseDirectory, + proxyConfig.getHistogramHourAccumulatorSize(), + proxyConfig.getHistogramHourAvgKeyBytes(), + proxyConfig.getHistogramHourAvgDigestBytes(), + proxyConfig.getHistogramHourCompression(), + proxyConfig.isHistogramHourAccumulatorPersisted(), + spanSampler); + startHistogramListeners( + histDayPorts, + pointHandler, + remoteHostAnnotator, + Granularity.DAY, + proxyConfig.getHistogramDayFlushSecs(), + proxyConfig.isHistogramDayMemoryCache(), + baseDirectory, + proxyConfig.getHistogramDayAccumulatorSize(), + proxyConfig.getHistogramDayAvgKeyBytes(), + proxyConfig.getHistogramDayAvgDigestBytes(), + proxyConfig.getHistogramDayCompression(), + proxyConfig.isHistogramDayAccumulatorPersisted(), + spanSampler); + startHistogramListeners( + histDistPorts, + pointHandler, + remoteHostAnnotator, + null, + proxyConfig.getHistogramDistFlushSecs(), + proxyConfig.isHistogramDistMemoryCache(), + baseDirectory, + proxyConfig.getHistogramDistAccumulatorSize(), + proxyConfig.getHistogramDistAvgKeyBytes(), + proxyConfig.getHistogramDistAvgDigestBytes(), + proxyConfig.getHistogramDistCompression(), + proxyConfig.isHistogramDistAccumulatorPersisted(), + spanSampler); } + } - // Logs ingestion. - if (loadLogsIngestionConfig() != null) { - logger.info("Loading logs ingestion."); - PointHandler pointHandler = new PointHandlerImpl("logs-ingester", pushValidationLevel, pushBlockedSamples, - getFlushTasks("logs-ingester")); - startLogsIngestionListeners(filebeatPort, rawLogsPort, pointHandler); + @Nullable + protected SslContext getSslContext(String port) { + return (secureAllPorts || tlsPorts.contains(port)) ? sslContext : null; + } + + @Nullable + protected CorsConfig getCorsConfig(String port) { + List ports = proxyConfig.getCorsEnabledPorts(); + List corsOrigin = proxyConfig.getCorsOrigin(); + if (ports.equals(ImmutableList.of("*")) || ports.contains(port)) { + CorsConfigBuilder builder; + if (corsOrigin.equals(ImmutableList.of("*"))) { + builder = CorsConfigBuilder.forOrigin(corsOrigin.get(0)); + } else { + builder = CorsConfigBuilder.forOrigins(corsOrigin.toArray(new String[0])); + } + builder.allowedRequestHeaders("Content-Type", "Referer", "User-Agent"); + builder.allowedRequestMethods(HttpMethod.GET, HttpMethod.POST, HttpMethod.PUT); + if (proxyConfig.isCorsAllowNullOrigin()) { + builder.allowNullOrigin(); + } + return builder.build(); } else { - logger.info("Not loading logs ingestion -- no config specified."); + return null; } } - protected void startJsonListener(String strPort) { + protected void startJsonListener(String strPort, ReportableEntityHandlerFactory handlerFactory) { + final int port = Integer.parseInt(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + + ChannelHandler channelHandler = + new JsonMetricsPortUnificationHandler( + strPort, + tokenAuthenticator, + healthCheckManager, + handlerFactory, + proxyConfig.getPrefix(), + proxyConfig.getHostname(), + preprocessors.get(strPort)); + + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getPushListenerMaxReceivedLength(), + proxyConfig.getPushListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-plaintext-json-" + port); + logger.info("listening on port: " + strPort + " for JSON metrics data"); + } + + protected void startWriteHttpJsonListener( + String strPort, ReportableEntityHandlerFactory handlerFactory) { + final int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + + ChannelHandler channelHandler = + new WriteHttpJsonPortUnificationHandler( + strPort, + tokenAuthenticator, + healthCheckManager, + handlerFactory, + proxyConfig.getHostname(), + preprocessors.get(strPort)); + + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getPushListenerMaxReceivedLength(), + proxyConfig.getPushListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-plaintext-writehttpjson-" + port); + logger.info("listening on port: " + strPort + " for write_http data"); + } + + protected void startOpenTsdbListener( + final String strPort, ReportableEntityHandlerFactory handlerFactory) { + int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + + ReportableEntityDecoder openTSDBDecoder = + new ReportPointDecoderWrapper( + new OpenTSDBDecoder("unknown", proxyConfig.getCustomSourceTags())); + + ChannelHandler channelHandler = + new OpenTSDBPortUnificationHandler( + strPort, + tokenAuthenticator, + healthCheckManager, + openTSDBDecoder, + handlerFactory, + preprocessors.get(strPort), + hostnameResolver); + + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getPushListenerMaxReceivedLength(), + proxyConfig.getPushListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-plaintext-opentsdb-" + port); + logger.info("listening on port: " + strPort + " for OpenTSDB metrics"); + } + + protected void startDataDogListener( + final String strPort, ReportableEntityHandlerFactory handlerFactory, HttpClient httpClient) { if (tokenAuthenticator.authRequired()) { - logger.warning("Port " + strPort + " (jsonListener) is not compatible with HTTP authentication, ignoring"); + logger.warning( + "Port: " + strPort + " (DataDog) is not compatible with HTTP authentication, ignoring"); return; } - preprocessors.forPort(strPort).forReportPoint() - .addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours)); - - startAsManagedThread(() -> { - activeListeners.inc(); - try { - org.eclipse.jetty.server.Server server = new org.eclipse.jetty.server.Server(Integer.parseInt(strPort)); - server.setHandler(new JsonMetricsEndpoint(strPort, hostname, prefix, - pushValidationLevel, pushBlockedSamples, getFlushTasks(strPort), preprocessors.forPort(strPort))); - server.start(); - server.join(); - } catch (InterruptedException e) { - logger.warning("Http Json server interrupted."); - } catch (Exception e) { - if (e instanceof BindException) { - bindErrors.inc(); - logger.severe("Unable to start listener - port " + String.valueOf(strPort) + " is already in use!"); - } else { - logger.log(Level.SEVERE, "HttpJson exception", e); - } - } finally { - activeListeners.dec(); - } - }, "listener-plaintext-json-" + strPort); + int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + + ChannelHandler channelHandler = + new DataDogPortUnificationHandler( + strPort, + healthCheckManager, + handlerFactory, + proxyConfig.getDataDogRequestRelayAsyncThreads(), + proxyConfig.isDataDogRequestRelaySyncMode(), + proxyConfig.isDataDogProcessSystemMetrics(), + proxyConfig.isDataDogProcessServiceChecks(), + httpClient, + proxyConfig.getDataDogRequestRelayTarget(), + preprocessors.get(strPort)); + + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getPushListenerMaxReceivedLength(), + proxyConfig.getPushListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-plaintext-datadog-" + port); + logger.info("listening on port: " + strPort + " for DataDog metrics"); } - protected void startWriteHttpJsonListener(String strPort) { + protected void startPickleListener( + String strPort, ReportableEntityHandlerFactory handlerFactory, GraphiteFormatter formatter) { if (tokenAuthenticator.authRequired()) { - logger.warning("Port " + strPort + " (writeHttpJson) is not compatible with HTTP authentication, ignoring"); + logger.warning( + "Port: " + + strPort + + " (pickle format) is not compatible with HTTP authentication, ignoring"); return; } - preprocessors.forPort(strPort).forReportPoint() - .addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours)); - - startAsManagedThread(() -> { - activeListeners.inc(); - try { - org.eclipse.jetty.server.Server server = new org.eclipse.jetty.server.Server(Integer.parseInt(strPort)); - server.setHandler(new WriteHttpJsonMetricsEndpoint(strPort, hostname, prefix, - pushValidationLevel, pushBlockedSamples, getFlushTasks(strPort), preprocessors.forPort(strPort))); - server.start(); - server.join(); - } catch (InterruptedException e) { - logger.warning("WriteHttpJson server interrupted."); - } catch (Exception e) { - if (e instanceof BindException) { - bindErrors.inc(); - logger.severe("Unable to start listener - port " + String.valueOf(strPort) + " is already in use!"); - } else { - logger.log(Level.SEVERE, "WriteHttpJson exception", e); - } - } finally { - activeListeners.dec(); - } - }, "listener-plaintext-writehttpjson-" + strPort); + int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + + // Set up a custom handler + ChannelHandler channelHandler = + new ChannelByteArrayHandler( + new PickleProtocolDecoder( + "unknown", proxyConfig.getCustomSourceTags(), formatter.getMetricMangler(), port), + handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, strPort)), + preprocessors.get(strPort), + blockedPointsLogger); + + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + ImmutableList.of( + () -> + new LengthFieldBasedFrameDecoder( + ByteOrder.BIG_ENDIAN, 1000000, 0, 4, 0, 4, false), + ByteArrayDecoder::new, + () -> channelHandler), + port, + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-binary-pickle-" + strPort); + logger.info("listening on port: " + strPort + " for Graphite/pickle protocol metrics"); + } + + protected void startTraceListener( + final String strPort, ReportableEntityHandlerFactory handlerFactory, SpanSampler sampler) { + final int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + + ChannelHandler channelHandler = + new TracePortUnificationHandler( + strPort, + tokenAuthenticator, + healthCheckManager, + new SpanDecoder("unknown"), + new SpanLogsDecoder(), + preprocessors.get(strPort), + handlerFactory, + sampler, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled()); + + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getTraceListenerMaxReceivedLength(), + proxyConfig.getTraceListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-plaintext-trace-" + port); + logger.info("listening on port: " + strPort + " for trace data"); + } + + @VisibleForTesting + protected void startCustomTracingListener( + final String strPort, + ReportableEntityHandlerFactory handlerFactory, + @Nullable WavefrontSender wfSender, + SpanSampler sampler) { + final int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + WavefrontInternalReporter wfInternalReporter = null; + if (wfSender != null) { + wfInternalReporter = + new WavefrontInternalReporter.Builder() + .prefixedWith("tracing.derived") + .withSource("custom_tracing") + .reportMinuteDistribution() + .build(wfSender); + // Start the reporter + wfInternalReporter.start(1, TimeUnit.MINUTES); + } + + ChannelHandler channelHandler = + new CustomTracingPortUnificationHandler( + strPort, + tokenAuthenticator, + healthCheckManager, + new SpanDecoder("unknown"), + new SpanLogsDecoder(), + preprocessors.get(strPort), + handlerFactory, + sampler, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled(), + wfSender, + wfInternalReporter, + proxyConfig.getTraceDerivedCustomTagKeys(), + proxyConfig.getCustomTracingApplicationName(), + proxyConfig.getCustomTracingServiceName()); + + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getTraceListenerMaxReceivedLength(), + proxyConfig.getTraceListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-custom-trace-" + port); + logger.info("listening on port: " + strPort + " for custom trace data"); } - protected void startLogsIngestionListeners(int portFilebeat, int portRawLogs, PointHandler pointHandler) { + protected void startTraceJaegerListener( + String strPort, + ReportableEntityHandlerFactory handlerFactory, + @Nullable WavefrontSender wfSender, + SpanSampler sampler) { if (tokenAuthenticator.authRequired()) { - logger.warning("Logs ingestion is not compatible with HTTP authentication, ignoring"); + logger.warning("Port: " + strPort + " is not compatible with HTTP authentication, ignoring"); return; } - try { - final LogsIngester logsIngester = new LogsIngester(pointHandler, this::loadLogsIngestionConfig, prefix, - System::currentTimeMillis); - logsIngester.start(); - - if (portFilebeat > 0) { - final Server filebeatServer = new Server(portFilebeat); - filebeatServer.setMessageListener(new FilebeatIngester(logsIngester, System::currentTimeMillis)); - startAsManagedThread(() -> { + startAsManagedThread( + Integer.parseInt(strPort), + () -> { + activeListeners.inc(); try { - activeListeners.inc(); - filebeatServer.listen(); + TChannel server = + new TChannel.Builder("jaeger-collector") + .setServerPort(Integer.parseInt(strPort)) + .build(); + server + .makeSubChannel("jaeger-collector", Connection.Direction.IN) + .register( + "Collector::submitBatches", + new JaegerTChannelCollectorHandler( + strPort, + handlerFactory, + wfSender, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled(), + preprocessors.get(strPort), + sampler, + proxyConfig.getTraceJaegerApplicationName(), + proxyConfig.getTraceDerivedCustomTagKeys())); + server.listen().channel().closeFuture().sync(); + server.shutdown(false); } catch (InterruptedException e) { - logger.log(Level.SEVERE, "Filebeat server interrupted.", e); + logger.info("Listener on port " + strPort + " shut down."); } catch (Exception e) { - // ChannelFuture throws undeclared checked exceptions, so we need to handle it - if (e instanceof BindException) { - bindErrors.inc(); - logger.severe("Unable to start listener - port " + String.valueOf(portRawLogs) + " is already in use!"); - } else { - logger.log(Level.SEVERE, "Filebeat exception", e); - } + logger.log(Level.SEVERE, "Jaeger trace collector exception", e); } finally { activeListeners.dec(); } - }, "listener-logs-filebeat-" + portFilebeat); - } + }, + "listener-jaeger-tchannel-" + strPort); + logger.info("listening on port: " + strPort + " for trace data (Jaeger format over TChannel)"); + } - if (portRawLogs > 0) { - RawLogsIngester rawLogsIngester = new RawLogsIngester(logsIngester, portRawLogs, System::currentTimeMillis). - withChannelIdleTimeout(listenerIdleConnectionTimeout). - withMaxLength(rawLogsMaxReceivedLength); - startAsManagedThread(() -> { + protected void startTraceJaegerHttpListener( + final String strPort, + ReportableEntityHandlerFactory handlerFactory, + @Nullable WavefrontSender wfSender, + SpanSampler sampler) { + final int port = Integer.parseInt(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + + ChannelHandler channelHandler = + new JaegerPortUnificationHandler( + strPort, + tokenAuthenticator, + healthCheckManager, + handlerFactory, + wfSender, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled(), + preprocessors.get(strPort), + sampler, + proxyConfig.getTraceJaegerApplicationName(), + proxyConfig.getTraceDerivedCustomTagKeys()); + + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getTraceListenerMaxReceivedLength(), + proxyConfig.getTraceListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-jaeger-http-" + port); + logger.info("listening on port: " + strPort + " for trace data (Jaeger format over HTTP)"); + } + + protected void startTraceJaegerGrpcListener( + final String strPort, + ReportableEntityHandlerFactory handlerFactory, + @Nullable WavefrontSender wfSender, + SpanSampler sampler) { + if (tokenAuthenticator.authRequired()) { + logger.warning("Port: " + strPort + " is not compatible with HTTP authentication, ignoring"); + return; + } + final int port = Integer.parseInt(strPort); + startAsManagedThread( + port, + () -> { + activeListeners.inc(); try { - activeListeners.inc(); - rawLogsIngester.listen(); - } catch (InterruptedException e) { - logger.log(Level.SEVERE, "Raw logs server interrupted.", e); + io.grpc.Server server = + NettyServerBuilder.forPort(port) + .addService( + new JaegerGrpcCollectorHandler( + strPort, + handlerFactory, + wfSender, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled(), + preprocessors.get(strPort), + sampler, + proxyConfig.getTraceJaegerApplicationName(), + proxyConfig.getTraceDerivedCustomTagKeys())) + .build(); + server.start(); } catch (Exception e) { - // ChannelFuture throws undeclared checked exceptions, so we need to handle it - if (e instanceof BindException) { - bindErrors.inc(); - logger.severe("Unable to start listener - port " + String.valueOf(portRawLogs) + " is already in use!"); - } else { - logger.log(Level.SEVERE, "RawLogs exception", e); - } + logger.log(Level.SEVERE, "Jaeger gRPC trace collector exception", e); } finally { activeListeners.dec(); } - }, "listener-logs-raw-" + portRawLogs); - } - } catch (ConfigurationException e) { - logger.log(Level.SEVERE, "Cannot start logsIngestion", e); - } + }, + "listener-jaeger-grpc-" + strPort); + logger.info( + "listening on port: " + + strPort + + " for trace data " + + "(Jaeger Protobuf format over gRPC)"); } - protected void startOpenTsdbListener(final String strPort, ReportableEntityHandlerFactory handlerFactory) { - if (prefix != null && !prefix.isEmpty()) { - preprocessors.forPort(strPort).forReportPoint().addTransformer(new ReportPointAddPrefixTransformer(prefix)); - } - preprocessors.forPort(strPort).forReportPoint() - .addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours)); + protected void startOtlpGrpcListener( + final String strPort, + ReportableEntityHandlerFactory handlerFactory, + @Nullable WavefrontSender wfSender, + SpanSampler sampler) { final int port = Integer.parseInt(strPort); - ReportableEntityDecoder openTSDBDecoder = new ReportPointDecoderWrapper( - new OpenTSDBDecoder("unknown", customSourceTags)); - - ChannelHandler channelHandler = new OpenTSDBPortUnificationHandler(strPort, tokenAuthenticator, openTSDBDecoder, - handlerFactory, preprocessors.forPort(strPort), remoteHostAnnotator); - - startAsManagedThread(new TcpIngester(createInitializer(channelHandler, strPort), port) - .withChildChannelOptions(childChannelOptions), "listener-plaintext-opentsdb-" + port); - logger.info("listening on port: " + strPort + " for OpenTSDB metrics"); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + startAsManagedThread( + port, + () -> { + activeListeners.inc(); + try { + OtlpGrpcTraceHandler traceHandler = + new OtlpGrpcTraceHandler( + strPort, + handlerFactory, + wfSender, + preprocessors.get(strPort), + sampler, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled(), + proxyConfig.getHostname(), + proxyConfig.getTraceDerivedCustomTagKeys()); + OtlpGrpcMetricsHandler metricsHandler = + new OtlpGrpcMetricsHandler( + strPort, + handlerFactory, + preprocessors.get(strPort), + proxyConfig.getHostname(), + proxyConfig.isOtlpResourceAttrsOnMetricsIncluded(), + proxyConfig.isOtlpAppTagsOnMetricsIncluded()); + io.grpc.Server server = + NettyServerBuilder.forPort(port) + .addService(traceHandler) + .addService(metricsHandler) + .build(); + server.start(); + } catch (Exception e) { + logger.log(Level.SEVERE, "OTLP gRPC collector exception", e); + } finally { + activeListeners.dec(); + } + }, + "listener-otlp-grpc-" + strPort); + logger.info("listening on port: " + strPort + " for OTLP data over gRPC"); } - protected void startDataDogListener(final String strPort, ReportableEntityHandlerFactory handlerFactory, - HttpClient httpClient) { - if (tokenAuthenticator.authRequired()) { - logger.warning("Port: " + strPort + " (DataDog) is not compatible with HTTP authentication, ignoring"); - return; - } - if (prefix != null && !prefix.isEmpty()) { - preprocessors.forPort(strPort).forReportPoint().addTransformer(new ReportPointAddPrefixTransformer(prefix)); - } - preprocessors.forPort(strPort).forReportPoint() - .addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours)); + protected void startOtlpHttpListener( + String strPort, + ReportableEntityHandlerFactory handlerFactory, + @Nullable WavefrontSender wfSender, + SpanSampler sampler) { final int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + + ChannelHandler channelHandler = + new OtlpHttpHandler( + handlerFactory, + tokenAuthenticator, + healthCheckManager, + strPort, + wfSender, + preprocessors.get(strPort), + sampler, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled(), + proxyConfig.getHostname(), + proxyConfig.getTraceDerivedCustomTagKeys(), + proxyConfig.isOtlpResourceAttrsOnMetricsIncluded(), + proxyConfig.isOtlpAppTagsOnMetricsIncluded()); - ChannelHandler channelHandler = new DataDogPortUnificationHandler(strPort, handlerFactory, - dataDogProcessSystemMetrics, dataDogProcessServiceChecks, httpClient, dataDogRequestRelayTarget, - preprocessors.forPort(strPort)); + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getPushListenerMaxReceivedLength(), + proxyConfig.getPushListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-otlp-http-" + port); + logger.info("listening on port: " + strPort + " for OTLP data over HTTP"); + } - startAsManagedThread(new TcpIngester(createInitializer(channelHandler, strPort), port) - .withChildChannelOptions(childChannelOptions), "listener-plaintext-datadog-" + port); - logger.info("listening on port: " + strPort + " for DataDog metrics"); + protected void startTraceZipkinListener( + String strPort, + ReportableEntityHandlerFactory handlerFactory, + @Nullable WavefrontSender wfSender, + SpanSampler sampler) { + final int port = Integer.parseInt(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + ChannelHandler channelHandler = + new ZipkinPortUnificationHandler( + strPort, + healthCheckManager, + handlerFactory, + wfSender, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled(), + preprocessors.get(strPort), + sampler, + proxyConfig.getTraceZipkinApplicationName(), + proxyConfig.getTraceDerivedCustomTagKeys()); + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getTraceListenerMaxReceivedLength(), + proxyConfig.getTraceListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-zipkin-trace-" + port); + logger.info("listening on port: " + strPort + " for trace data (Zipkin format)"); } - protected void startPickleListener(String strPort, PointHandler pointHandler, GraphiteFormatter formatter) { - if (tokenAuthenticator.authRequired()) { - logger.warning("Port: " + strPort + " (pickle format) is not compatible with HTTP authentication, ignoring"); - return; - } - if (prefix != null && !prefix.isEmpty()) { - preprocessors.forPort(strPort).forReportPoint().addTransformer(new ReportPointAddPrefixTransformer(prefix)); - } - preprocessors.forPort(strPort).forReportPoint() - .addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours)); - int port = Integer.parseInt(strPort); - // Set up a custom handler - ChannelHandler channelHandler = new ChannelByteArrayHandler( - new PickleProtocolDecoder("unknown", customSourceTags, formatter.getMetricMangler(), port), - pointHandler, preprocessors.forPort(strPort)); - - // create a class to use for StreamIngester to get a new FrameDecoder - // for each request (not shareable since it's storing how many bytes - // read, etc) - // the pickle listener for carbon-relay streams data in its own format: - // [Length of pickled data to follow in a 4 byte unsigned int] - // [pickled data of the given length] - // - // the LengthFieldBasedFrameDecoder() parses out the length and grabs - // bytes from the stream and passes that chunk as a byte array - // to the decoder. - class FrameDecoderFactoryImpl implements StreamIngester.FrameDecoderFactory { - @Override - public ChannelInboundHandler getDecoder() { - return new LengthFieldBasedFrameDecoder(ByteOrder.BIG_ENDIAN, 1000000, 0, 4, 0, 4, false); - } - } + @VisibleForTesting + protected void startGraphiteListener( + String strPort, + ReportableEntityHandlerFactory handlerFactory, + SharedGraphiteHostAnnotator hostAnnotator, + SpanSampler sampler) { + final int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + + WavefrontPortUnificationHandler wavefrontPortUnificationHandler = + new WavefrontPortUnificationHandler( + strPort, + tokenAuthenticator, + healthCheckManager, + decoderSupplier.get(), + handlerFactory, + hostAnnotator, + preprocessors.get(strPort), + // histogram/trace/span log feature flags consult to the central cluster + // configuration + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.HISTOGRAM) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled(), + sampler, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.LOGS) + .isFeatureDisabled(), + proxyConfig.receivedLogServerDetails(), + proxyConfig.enableHyperlogsConvergedCsp()); - startAsManagedThread(new StreamIngester(new FrameDecoderFactoryImpl(), channelHandler, port) - .withChildChannelOptions(childChannelOptions), "listener-binary-pickle-" + port); - logger.info("listening on port: " + strPort + " for pickle protocol metrics"); + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + wavefrontPortUnificationHandler, + port, + proxyConfig.getPushListenerMaxReceivedLength(), + proxyConfig.getPushListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-graphite-" + port); } - protected void startTraceListener(final String strPort, ReportableEntityHandlerFactory handlerFactory) { - if (prefix != null && !prefix.isEmpty()) { - preprocessors.forPort(strPort).forReportPoint().addTransformer(new ReportPointAddPrefixTransformer(prefix)); - } - preprocessors.forPort(strPort).forReportPoint() - .addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours)); + @VisibleForTesting + protected void startDeltaCounterListener( + String strPort, + SharedGraphiteHostAnnotator hostAnnotator, + SenderTaskFactory senderTaskFactory, + SpanSampler sampler) { final int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + + if (this.deltaCounterHandlerFactory == null) { + this.deltaCounterHandlerFactory = + new ReportableEntityHandlerFactory() { + private final Map> handlers = + new ConcurrentHashMap<>(); + + @Override + public ReportableEntityHandler getHandler(HandlerKey handlerKey) { + //noinspection unchecked + return (ReportableEntityHandler) + handlers.computeIfAbsent( + handlerKey.getHandle(), + k -> + new DeltaCounterAccumulationHandlerImpl( + handlerKey, + proxyConfig.getPushBlockedSamples(), + senderTaskFactory.createSenderTasks(handlerKey), + validationConfiguration, + proxyConfig.getDeltaCountersAggregationIntervalSeconds(), + (tenantName, rate) -> + entityPropertiesFactoryMap + .get(tenantName) + .get(ReportableEntityType.POINT) + .reportReceivedRate(handlerKey.getHandle(), rate), + blockedPointsLogger, + VALID_POINTS_LOGGER)); + } - Sampler rateSampler = SpanSamplerUtils.getRateSampler(traceSamplingRate); - Sampler durationSampler = SpanSamplerUtils.getDurationSampler(traceSamplingDuration); - List samplers = SpanSamplerUtils.fromSamplers(rateSampler, durationSampler); - Sampler compositeSampler = new CompositeSampler(samplers); + @Override + public void shutdown(@Nonnull String handle) { + if (handlers.containsKey(handle)) { + handlers.values().forEach(ReportableEntityHandler::shutdown); + } + } + }; + } + shutdownTasks.add(() -> deltaCounterHandlerFactory.shutdown(strPort)); + + WavefrontPortUnificationHandler wavefrontPortUnificationHandler = + new WavefrontPortUnificationHandler( + strPort, + tokenAuthenticator, + healthCheckManager, + decoderSupplier.get(), + deltaCounterHandlerFactory, + hostAnnotator, + preprocessors.get(strPort), + () -> false, + () -> false, + () -> false, + sampler, + () -> false, + proxyConfig.receivedLogServerDetails(), + proxyConfig.enableHyperlogsConvergedCsp()); - ChannelHandler channelHandler = new TracePortUnificationHandler(strPort, tokenAuthenticator, - new SpanDecoder("unknown"), preprocessors.forPort(strPort), handlerFactory, compositeSampler); + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + wavefrontPortUnificationHandler, + port, + proxyConfig.getPushListenerMaxReceivedLength(), + proxyConfig.getPushListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-deltaCounter-" + port); + } - startAsManagedThread(new TcpIngester(createInitializer(channelHandler, strPort), port) - .withChildChannelOptions(childChannelOptions), "listener-plaintext-trace-" + port); - logger.info("listening on port: " + strPort + " for trace data"); + @VisibleForTesting + protected void startRelayListener( + String strPort, + ReportableEntityHandlerFactory handlerFactory, + SharedGraphiteHostAnnotator hostAnnotator) { + final int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + + ReportableEntityHandlerFactory handlerFactoryDelegate = + proxyConfig.isPushRelayHistogramAggregator() + ? new DelegatingReportableEntityHandlerFactoryImpl(handlerFactory) { + @Override + public ReportableEntityHandler getHandler(HandlerKey handlerKey) { + if (handlerKey.getEntityType() == ReportableEntityType.HISTOGRAM) { + ChronicleMap accumulator = + ChronicleMap.of(HistogramKey.class, AgentDigest.class) + .keyMarshaller(HistogramKeyMarshaller.get()) + .valueMarshaller(AgentDigestMarshaller.get()) + .entries(proxyConfig.getPushRelayHistogramAggregatorAccumulatorSize()) + .averageKeySize(proxyConfig.getHistogramDistAvgKeyBytes()) + .averageValueSize(proxyConfig.getHistogramDistAvgDigestBytes()) + .maxBloatFactor(1000) + .create(); + AgentDigestFactory agentDigestFactory = + new AgentDigestFactory( + () -> + (short) + Math.min( + proxyConfig.getPushRelayHistogramAggregatorCompression(), + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .getGlobalProperties() + .getHistogramStorageAccuracy()), + TimeUnit.SECONDS.toMillis( + proxyConfig.getPushRelayHistogramAggregatorFlushSecs()), + proxyConfig.getTimeProvider()); + AccumulationCache cachedAccumulator = + new AccumulationCache( + accumulator, + agentDigestFactory, + 0, + "histogram.accumulator.distributionRelay", + null); + //noinspection unchecked + return (ReportableEntityHandler) + new HistogramAccumulationHandlerImpl( + handlerKey, + cachedAccumulator, + proxyConfig.getPushBlockedSamples(), + null, + validationConfiguration, + true, + (tenantName, rate) -> + entityPropertiesFactoryMap + .get(tenantName) + .get(ReportableEntityType.HISTOGRAM) + .reportReceivedRate(handlerKey.getHandle(), rate), + blockedHistogramsLogger, + VALID_HISTOGRAMS_LOGGER); + } + return delegate.getHandler(handlerKey); + } + } + : handlerFactory; + + Map> filteredDecoders = + decoderSupplier.get().entrySet().stream() + .filter(x -> !x.getKey().equals(ReportableEntityType.SOURCE_TAG)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + ChannelHandler channelHandler = + new RelayPortUnificationHandler( + strPort, + tokenAuthenticator, + healthCheckManager, + filteredDecoders, + handlerFactoryDelegate, + preprocessors.get(strPort), + hostAnnotator, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.HISTOGRAM) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.LOGS) + .isFeatureDisabled(), + apiContainer, + proxyConfig); + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getPushListenerMaxReceivedLength(), + proxyConfig.getPushListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-relay-" + port); } - protected void startTraceJaegerListener(String strPort, ReportableEntityHandlerFactory handlerFactory) { + protected void startLogsIngestionListener(int port, LogsIngester logsIngester) { if (tokenAuthenticator.authRequired()) { - logger.warning("Port: " + strPort + " is not compatible with HTTP authentication, ignoring"); + logger.warning("Filebeat log ingestion is not compatible with HTTP authentication, ignoring"); return; } - startAsManagedThread(() -> { - activeListeners.inc(); - try { - TChannel server = new TChannel.Builder("jaeger-collector"). - setServerPort(Integer.valueOf(strPort)). - build(); - server. - makeSubChannel("jaeger-collector", Connection.Direction.IN). - register("Collector::submitBatches", new JaegerThriftCollectorHandler(strPort, handlerFactory, - traceDisabled)); - server.listen().channel().closeFuture().sync(); - server.shutdown(false); - } catch (InterruptedException e) { - logger.info("Listener on port " + strPort + " shut down."); - } catch (Exception e) { - logger.log(Level.SEVERE, "Jaeger trace collector exception", e); - } finally { - activeListeners.dec(); - } - }, "listener-jaeger-thrift-" + strPort); - logger.info("listening on port: " + strPort + " for trace data (Jaeger format)"); + final Server filebeatServer = + new Server( + "0.0.0.0", + port, + proxyConfig.getListenerIdleConnectionTimeout(), + Runtime.getRuntime().availableProcessors()); + filebeatServer.setMessageListener( + new FilebeatIngester(logsIngester, System::currentTimeMillis)); + startAsManagedThread( + port, + () -> { + try { + activeListeners.inc(); + filebeatServer.listen(); + } catch (InterruptedException e) { + logger.info("Filebeat server on port " + port + " shut down"); + } catch (Exception e) { + // ChannelFuture throws undeclared checked exceptions, so we need to handle + // it + //noinspection ConstantConditions + if (e instanceof BindException) { + bindErrors.inc(); + logger.severe("Unable to start listener - port " + port + " is already in use!"); + } else { + logger.log(Level.SEVERE, "Filebeat exception", e); + } + } finally { + activeListeners.dec(); + } + }, + "listener-logs-filebeat-" + port); + logger.info("listening on port: " + port + " for Filebeat logs"); } - protected void startTraceZipkinListener(String strPort, ReportableEntityHandlerFactory handlerFactory) { - final int port = Integer.parseInt(strPort); - ChannelHandler channelHandler = new ZipkinPortUnificationHandler(strPort, handlerFactory, traceDisabled); + @VisibleForTesting + protected void startRawLogsIngestionListener(int port, LogsIngester logsIngester) { + String strPort = String.valueOf(port); + if (proxyConfig.isHttpHealthCheckAllPorts()) healthCheckManager.enableHealthcheck(port); + ChannelHandler channelHandler = + new RawLogsIngesterPortUnificationHandler( + strPort, + logsIngester, + hostnameResolver, + tokenAuthenticator, + healthCheckManager, + preprocessors.get(strPort)); - startAsManagedThread(new TcpIngester(createInitializer(channelHandler, strPort), port). - withChildChannelOptions(childChannelOptions), "listener-zipkin-trace-" + port); + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getRawLogsMaxReceivedLength(), + proxyConfig.getRawLogsHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-logs-raw-" + port); + logger.info("listening on port: " + strPort + " for raw logs"); } @VisibleForTesting - protected void startGraphiteListener(String strPort, ReportableEntityHandlerFactory handlerFactory, - CachingGraphiteHostAnnotator hostAnnotator) { - final int port = Integer.parseInt(strPort); + protected void startAdminListener(int port) { + String strPort = String.valueOf(port); + ChannelHandler channelHandler = + new AdminPortUnificationHandler( + tokenAuthenticator, + healthCheckManager, + String.valueOf(port), + proxyConfig.getAdminApiRemoteIpAllowRegex()); - if (prefix != null && !prefix.isEmpty()) { - preprocessors.forPort(strPort).forReportPoint().addTransformer(new ReportPointAddPrefixTransformer(prefix)); - } - preprocessors.forPort(strPort).forReportPoint() - .addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours)); - - Map decoders = ImmutableMap.of( - ReportableEntityType.POINT, getDecoderInstance(), - ReportableEntityType.SOURCE_TAG, new ReportSourceTagDecoder(), - ReportableEntityType.HISTOGRAM, new ReportPointDecoderWrapper(new HistogramDecoder("unknown"))); - ChannelHandler channelHandler = new WavefrontPortUnificationHandler(strPort, tokenAuthenticator, decoders, - handlerFactory, hostAnnotator, preprocessors.forPort(strPort)); startAsManagedThread( - new TcpIngester(createInitializer(channelHandler, strPort), port). - withChildChannelOptions(childChannelOptions), "listener-graphite-" + port); + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getPushListenerMaxReceivedLength(), + proxyConfig.getPushListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-http-admin-" + port); + logger.info("Admin port: " + port); } @VisibleForTesting - protected void startRelayListener(String strPort, ReportableEntityHandlerFactory handlerFactory) { - final int port = Integer.parseInt(strPort); + protected void startHealthCheckListener(int port) { + String strPort = String.valueOf(port); + healthCheckManager.enableHealthcheck(port); + ChannelHandler channelHandler = new HttpHealthCheckEndpointHandler(healthCheckManager, port); - if (prefix != null && !prefix.isEmpty()) { - preprocessors.forPort(strPort).forReportPoint().addTransformer(new ReportPointAddPrefixTransformer(prefix)); - } - preprocessors.forPort(strPort).forReportPoint() - .addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours)); - - Map decoders = ImmutableMap.of( - ReportableEntityType.POINT, getDecoderInstance(), - ReportableEntityType.HISTOGRAM, new ReportPointDecoderWrapper(new HistogramDecoder("unknown"))); - ChannelHandler channelHandler = new RelayPortUnificationHandler(strPort, tokenAuthenticator, decoders, - handlerFactory, preprocessors.forPort(strPort)); startAsManagedThread( - new TcpIngester(createInitializer(channelHandler, strPort), port). - withChildChannelOptions(childChannelOptions), "listener-relay-" + port); + port, + new TcpIngester( + createInitializer( + channelHandler, + port, + proxyConfig.getPushListenerMaxReceivedLength(), + proxyConfig.getPushListenerHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-http-healthcheck-" + port); + logger.info("Health check port enabled: " + port); } - protected void startHistogramListeners(Iterator ports, Decoder decoder, PointHandler pointHandler, - TapeDeck> receiveDeck, @Nullable Utils.Granularity granularity, - int flushSecs, int fanout, boolean memoryCacheEnabled, File baseDirectory, - Long accumulatorSize, int avgKeyBytes, int avgDigestBytes, short compression) { - if (tokenAuthenticator.authRequired()) { - logger.warning("Histograms are not compatible with HTTP authentication, ignoring"); - return; - } - String listenerBinType = Utils.Granularity.granularityToString(granularity); + protected void startHistogramListeners( + List ports, + ReportableEntityHandler pointHandler, + SharedGraphiteHostAnnotator hostAnnotator, + @Nullable Granularity granularity, + int flushSecs, + boolean memoryCacheEnabled, + File baseDirectory, + Long accumulatorSize, + int avgKeyBytes, + int avgDigestBytes, + short compression, + boolean persist, + SpanSampler sampler) + throws Exception { + if (ports.size() == 0) return; + String listenerBinType = HistogramUtils.granularityToString(granularity); // Accumulator - MapLoader mapLoader = new MapLoader<>( - HistogramKey.class, - AgentDigest.class, - accumulatorSize, - avgKeyBytes, - avgDigestBytes, - HistogramKeyMarshaller.get(), - AgentDigestMarshaller.get(), - persistAccumulator); + if (persist) { + // Check directory + checkArgument( + baseDirectory.isDirectory(), baseDirectory.getAbsolutePath() + " must be a directory!"); + checkArgument( + baseDirectory.canWrite(), baseDirectory.getAbsolutePath() + " must be write-able!"); + } + MapLoader mapLoader = + new MapLoader<>( + HistogramKey.class, + AgentDigest.class, + accumulatorSize, + avgKeyBytes, + avgDigestBytes, + HistogramKeyMarshaller.get(), + AgentDigestMarshaller.get(), + persist); File accumulationFile = new File(baseDirectory, "accumulator." + listenerBinType); ChronicleMap accumulator = mapLoader.get(accumulationFile); histogramExecutor.scheduleWithFixedDelay( () -> { - // warn if accumulator is more than 1.5x the original size, as ChronicleMap starts losing efficiency + // warn if accumulator is more than 1.5x the original size, + // as ChronicleMap starts losing efficiency if (accumulator.size() > accumulatorSize * 5) { - logger.severe("Histogram " + listenerBinType + " accumulator size (" + accumulator.size() + - ") is more than 5x higher than currently configured size (" + accumulatorSize + - "), which may cause severe performance degradation issues or data loss! " + - "If the data volume is expected to stay at this level, we strongly recommend increasing the value " + - "for accumulator size in wavefront.conf and restarting the proxy."); + logger.severe( + "Histogram " + + listenerBinType + + " accumulator size (" + + accumulator.size() + + ") is more than 5x higher than currently configured size (" + + accumulatorSize + + "), which may cause severe performance degradation issues " + + "or data loss! If the data volume is expected to stay at this level, we strongly " + + "recommend increasing the value for accumulator size in wavefront.conf and " + + "restarting the proxy."); } else if (accumulator.size() > accumulatorSize * 2) { - logger.warning("Histogram " + listenerBinType + " accumulator size (" + accumulator.size() + - ") is more than 2x higher than currently configured size (" + accumulatorSize + - "), which may cause performance issues. " + - "If the data volume is expected to stay at this level, we strongly recommend increasing the value " + - "for accumulator size in wavefront.conf and restarting the proxy."); + logger.warning( + "Histogram " + + listenerBinType + + " accumulator size (" + + accumulator.size() + + ") is more than 2x higher than currently configured size (" + + accumulatorSize + + "), which may cause performance issues. If the data volume is " + + "expected to stay at this level, we strongly recommend increasing the value " + + "for accumulator size in wavefront.conf and restarting the proxy."); } }, 10, 10, TimeUnit.SECONDS); - AccumulationCache cachedAccumulator = new AccumulationCache(accumulator, - (memoryCacheEnabled ? accumulatorSize : 0), null); + AgentDigestFactory agentDigestFactory = + new AgentDigestFactory( + () -> + (short) + Math.min( + compression, + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .getGlobalProperties() + .getHistogramStorageAccuracy()), + TimeUnit.SECONDS.toMillis(flushSecs), + proxyConfig.getTimeProvider()); + Accumulator cachedAccumulator = + new AccumulationCache( + accumulator, + agentDigestFactory, + (memoryCacheEnabled ? accumulatorSize : 0), + "histogram.accumulator." + HistogramUtils.granularityToString(granularity), + null); // Schedule write-backs histogramExecutor.scheduleWithFixedDelay( - cachedAccumulator.getResolveTask(), - histogramAccumulatorResolveInterval, - histogramAccumulatorResolveInterval, + cachedAccumulator::flush, + proxyConfig.getHistogramAccumulatorResolveInterval(), + proxyConfig.getHistogramAccumulatorResolveInterval(), TimeUnit.MILLISECONDS); + histogramFlushRunnables.add(cachedAccumulator::flush); + + PointHandlerDispatcher dispatcher = + new PointHandlerDispatcher( + cachedAccumulator, + pointHandler, + proxyConfig.getTimeProvider(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.HISTOGRAM) + .isFeatureDisabled(), + proxyConfig.getHistogramAccumulatorFlushMaxBatchSize() < 0 + ? null + : proxyConfig.getHistogramAccumulatorFlushMaxBatchSize(), + granularity); - PointHandlerDispatcher dispatcher = new PointHandlerDispatcher(cachedAccumulator, pointHandler, - histogramAccumulatorFlushMaxBatchSize < 0 ? null : histogramAccumulatorFlushMaxBatchSize, granularity); - - histogramExecutor.scheduleWithFixedDelay(dispatcher, histogramAccumulatorFlushInterval, - histogramAccumulatorFlushInterval, TimeUnit.MILLISECONDS); + histogramExecutor.scheduleWithFixedDelay( + dispatcher, + proxyConfig.getHistogramAccumulatorFlushInterval(), + proxyConfig.getHistogramAccumulatorFlushInterval(), + TimeUnit.MILLISECONDS); + histogramFlushRunnables.add(dispatcher); // gracefully shutdown persisted accumulator (ChronicleMap) on proxy exit - shutdownTasks.add(() -> { - try { - logger.fine("Flushing in-flight histogram accumulator digests: " + listenerBinType); - cachedAccumulator.getResolveTask().run(); - logger.fine("Shutting down histogram accumulator cache: " + listenerBinType); - accumulator.close(); - } catch (Throwable t) { - logger.log(Level.SEVERE, "Error flushing " + listenerBinType + " accumulator, possibly unclean shutdown: ", t); - } - }); + shutdownTasks.add( + () -> { + try { + logger.fine("Flushing in-flight histogram accumulator digests: " + listenerBinType); + cachedAccumulator.flush(); + logger.fine("Shutting down histogram accumulator cache: " + listenerBinType); + accumulator.close(); + } catch (Throwable t) { + logger.log( + Level.SEVERE, + "Error flushing " + listenerBinType + " accumulator, possibly unclean shutdown: ", + t); + } + }); + + ReportableEntityHandlerFactory histogramHandlerFactory = + new ReportableEntityHandlerFactory() { + private final Map> handlers = + new ConcurrentHashMap<>(); + + @SuppressWarnings("unchecked") + @Override + public ReportableEntityHandler getHandler(HandlerKey handlerKey) { + return (ReportableEntityHandler) + handlers.computeIfAbsent( + handlerKey, + k -> + new HistogramAccumulationHandlerImpl( + handlerKey, + cachedAccumulator, + proxyConfig.getPushBlockedSamples(), + granularity, + validationConfiguration, + granularity == null, + null, + blockedHistogramsLogger, + VALID_HISTOGRAMS_LOGGER)); + } - ports.forEachRemaining(port -> { - startHistogramListener( - port, - decoder, - pointHandler, - cachedAccumulator, - baseDirectory, - granularity, - receiveDeck, - TimeUnit.SECONDS.toMillis(flushSecs), - fanout, - compression - ); - logger.info("listening on port: " + port + " for histogram samples, accumulating to the " + - listenerBinType); - }); + @Override + public void shutdown(@Nonnull String handle) { + handlers.values().forEach(ReportableEntityHandler::shutdown); + } + }; + + ports.forEach( + strPort -> { + int port = Integer.parseInt(strPort); + registerPrefixFilter(strPort); + registerTimestampFilter(strPort); + if (proxyConfig.isHttpHealthCheckAllPorts()) { + healthCheckManager.enableHealthcheck(port); + } + WavefrontPortUnificationHandler wavefrontPortUnificationHandler = + new WavefrontPortUnificationHandler( + strPort, + tokenAuthenticator, + healthCheckManager, + decoderSupplier.get(), + histogramHandlerFactory, + hostAnnotator, + preprocessors.get(strPort), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.HISTOGRAM) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE) + .isFeatureDisabled(), + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .isFeatureDisabled(), + sampler, + () -> + entityPropertiesFactoryMap + .get(CENTRAL_TENANT_NAME) + .get(ReportableEntityType.LOGS) + .isFeatureDisabled(), + proxyConfig.receivedLogServerDetails(), + proxyConfig.enableHyperlogsConvergedCsp()); + + startAsManagedThread( + port, + new TcpIngester( + createInitializer( + wavefrontPortUnificationHandler, + port, + proxyConfig.getHistogramMaxReceivedLength(), + proxyConfig.getHistogramHttpBufferSize(), + proxyConfig.getListenerIdleConnectionTimeout(), + getSslContext(strPort), + getCorsConfig(strPort)), + port) + .withChildChannelOptions(childChannelOptions), + "listener-histogram-" + port); + logger.info( + "listening on port: " + + port + + " for histogram samples, accumulating to the " + + listenerBinType); + }); + } + private void registerTimestampFilter(String strPort) { + preprocessors + .getSystemPreprocessor(strPort) + .forReportPoint() + .addFilter( + 0, + new ReportPointTimestampInRangeFilter( + proxyConfig.getDataBackfillCutoffHours(), proxyConfig.getDataPrefillCutoffHours())); } - /** - * Needs to set up a queueing handler and a consumer/lexer for the queue - */ - private void startHistogramListener( - String portAsString, - Decoder decoder, - PointHandler handler, - AccumulationCache accumulationCache, - File directory, - @Nullable Utils.Granularity granularity, - TapeDeck> receiveDeck, - long timeToLiveMillis, - int fanout, - short compression) { - - int port = Integer.parseInt(portAsString); - List handlers = new ArrayList<>(); - - for (int i = 0; i < fanout; ++i) { - File tapeFile = new File(directory, "Port_" + portAsString + "_" + i); - ObjectQueue> receiveTape = receiveDeck.getTape(tapeFile); - - // Set-up scanner - AccumulationTask scanTask = new AccumulationTask( - receiveTape, - accumulationCache, - decoder, - handler, - Validation.Level.valueOf(pushValidationLevel), - timeToLiveMillis, - granularity, - compression); - - histogramScanExecutor.scheduleWithFixedDelay(scanTask, - histogramProcessingQueueScanInterval, histogramProcessingQueueScanInterval, TimeUnit.MILLISECONDS); - - QueuingChannelHandler inputHandler = new QueuingChannelHandler<>(receiveTape, - pushFlushMaxPoints.get(), histogramDisabled); - handlers.add(inputHandler); - histogramFlushExecutor.scheduleWithFixedDelay(inputHandler.getBufferFlushTask(), - histogramReceiveBufferFlushInterval, histogramReceiveBufferFlushInterval, TimeUnit.MILLISECONDS); + private void registerPrefixFilter(String strPort) { + if (proxyConfig.getPrefix() != null && !proxyConfig.getPrefix().isEmpty()) { + preprocessors + .getSystemPreprocessor(strPort) + .forReportPoint() + .addTransformer(new ReportPointAddPrefixTransformer(proxyConfig.getPrefix())); } - - // Set-up producer - startAsManagedThread(new HistogramLineIngester(handlers, port). - withChannelIdleTimeout(listenerIdleConnectionTimeout). - withMaxLength(histogramMaxReceivedLength), - "listener-plaintext-histogram-" + port); - } - - private ChannelInitializer createInitializer(ChannelHandler channelHandler, String strPort) { - ChannelHandler idleStateEventHandler = new IdleStateEventHandler( - Metrics.newCounter(new TaggedMetricName("listeners", "connections.idle.closed", "port", strPort))); - ChannelHandler connectionTracker = new ConnectionTrackingHandler( - Metrics.newCounter(new TaggedMetricName("listeners", "connections.accepted", "port", strPort)), - Metrics.newCounter(new TaggedMetricName("listeners", "connections.active", "port", strPort))); - return new ChannelInitializer() { - @Override - public void initChannel(SocketChannel ch) { - ChannelPipeline pipeline = ch.pipeline(); - - pipeline.addFirst("idlehandler", new IdleStateHandler(listenerIdleConnectionTimeout, 0, 0)); - pipeline.addLast("idlestateeventhandler", idleStateEventHandler); - pipeline.addLast("connectiontracker", connectionTracker); - pipeline.addLast(new PlainTextOrHttpFrameDecoder(channelHandler, pushListenerMaxReceivedLength, - pushListenerHttpBufferSize)); - } - }; } /** * Push agent configuration during check-in by the collector. * + * @param tenantName The tenant name to which config corresponding * @param config The configuration to process. */ @Override - protected void processConfiguration(AgentConfiguration config) { + protected void processConfiguration(String tenantName, AgentConfiguration config) { try { - agentAPI.agentConfigProcessed(agentId); + boolean configHasPreprocessorRules = config.getPreprocessorRules() != null && !config.getPreprocessorRules().isEmpty(); +// apply new preprocessor rules after checkin + if (ProxyCheckInScheduler.isRulesSetInFE.get() && configHasPreprocessorRules) { + loadPreprocessors(config.getPreprocessorRules(), true); + } + // check if we want to read local file + else if (!ProxyCheckInScheduler.isRulesSetInFE.get() && + proxyConfig.getPreprocessorConfigFile() != null && + !usingLocalFileRules.get()) { // are we already reading local file + loadPreprocessors(proxyConfig.getPreprocessorConfigFile(), false); + } + Long pointsPerBatch = config.getPointsPerBatch(); + EntityPropertiesFactory tenantSpecificEntityProps = + entityPropertiesFactoryMap.get(tenantName); if (BooleanUtils.isTrue(config.getCollectorSetsPointsPerBatch())) { if (pointsPerBatch != null) { // if the collector is in charge and it provided a setting, use it - pushFlushMaxPoints.set(pointsPerBatch.intValue()); + tenantSpecificEntityProps + .get(ReportableEntityType.POINT) + .setDataPerBatch(pointsPerBatch.intValue()); logger.fine("Proxy push batch set to (remotely) " + pointsPerBatch); } // otherwise don't change the setting } else { - // restores the agent setting - pushFlushMaxPoints.set(pushFlushMaxPointsInitialValue); - logger.fine("Proxy push batch set to (locally) " + pushFlushMaxPoints.get()); + // restore the original setting + tenantSpecificEntityProps.get(ReportableEntityType.POINT).setDataPerBatch(null); + logger.fine( + "Proxy push batch set to (locally) " + + tenantSpecificEntityProps.get(ReportableEntityType.POINT).getDataPerBatch()); } - - if (BooleanUtils.isTrue(config.getCollectorSetsRateLimit())) { - Long collectorRateLimit = config.getCollectorRateLimit(); - if (pushRateLimiter != null && collectorRateLimit != null && pushRateLimiter.getRate() != collectorRateLimit) { - pushRateLimiter.setRate(collectorRateLimit); - logger.warning("Proxy rate limit set to " + collectorRateLimit + " remotely"); - } - } else { - if (pushRateLimiter != null && pushRateLimiter.getRate() != pushRateLimit) { - pushRateLimiter.setRate(pushRateLimit); - if (pushRateLimit >= 10_000_000) { - logger.warning("Proxy rate limit no longer enforced by remote"); - } else { - logger.warning("Proxy rate limit restored to " + pushRateLimit); - } + if (config.getHistogramStorageAccuracy() != null) { + tenantSpecificEntityProps + .getGlobalProperties() + .setHistogramStorageAccuracy(config.getHistogramStorageAccuracy().shortValue()); + } + if (!proxyConfig.isBackendSpanHeadSamplingPercentIgnored()) { + double previousSamplingRate = + tenantSpecificEntityProps.getGlobalProperties().getTraceSamplingRate(); + tenantSpecificEntityProps + .getGlobalProperties() + .setTraceSamplingRate(config.getSpanSamplingRate()); + rateSampler.setSamplingRate( + tenantSpecificEntityProps.getGlobalProperties().getTraceSamplingRate()); + if (previousSamplingRate + != tenantSpecificEntityProps.getGlobalProperties().getTraceSamplingRate()) { + logger.info( + "Proxy trace span sampling rate set to " + + tenantSpecificEntityProps.getGlobalProperties().getTraceSamplingRate()); } } + tenantSpecificEntityProps + .getGlobalProperties() + .setDropSpansDelayedMinutes(config.getDropSpansDelayedMinutes()); + tenantSpecificEntityProps + .getGlobalProperties() + .setActiveSpanSamplingPolicies(config.getActiveSpanSamplingPolicies()); + + updateRateLimiter( + tenantName, + ReportableEntityType.POINT, + config.getCollectorSetsRateLimit(), + config.getCollectorRateLimit(), + config.getGlobalCollectorRateLimit()); + updateRateLimiter( + tenantName, + ReportableEntityType.HISTOGRAM, + config.getCollectorSetsRateLimit(), + config.getHistogramRateLimit(), + config.getGlobalHistogramRateLimit()); + updateRateLimiter( + tenantName, + ReportableEntityType.SOURCE_TAG, + config.getCollectorSetsRateLimit(), + config.getSourceTagsRateLimit(), + config.getGlobalSourceTagRateLimit()); + updateRateLimiter( + tenantName, + ReportableEntityType.TRACE, + config.getCollectorSetsRateLimit(), + config.getSpanRateLimit(), + config.getGlobalSpanRateLimit()); + updateRateLimiter( + tenantName, + ReportableEntityType.TRACE_SPAN_LOGS, + config.getCollectorSetsRateLimit(), + config.getSpanLogsRateLimit(), + config.getGlobalSpanLogsRateLimit()); + updateRateLimiter( + tenantName, + ReportableEntityType.EVENT, + config.getCollectorSetsRateLimit(), + config.getEventsRateLimit(), + config.getGlobalEventRateLimit()); + updateRateLimiter( + tenantName, + ReportableEntityType.LOGS, + config.getCollectorSetsRateLimit(), + config.getLogsRateLimit(), + config.getGlobalLogsRateLimit()); if (BooleanUtils.isTrue(config.getCollectorSetsRetryBackoff())) { if (config.getRetryBackoffBaseSeconds() != null) { // if the collector is in charge and it provided a setting, use it - retryBackoffBaseSeconds.set(config.getRetryBackoffBaseSeconds()); - logger.fine("Proxy backoff base set to (remotely) " + - config.getRetryBackoffBaseSeconds()); + tenantSpecificEntityProps + .getGlobalProperties() + .setRetryBackoffBaseSeconds(config.getRetryBackoffBaseSeconds()); + logger.fine( + "Proxy backoff base set to (remotely) " + config.getRetryBackoffBaseSeconds()); } // otherwise don't change the setting } else { // restores the agent setting - retryBackoffBaseSeconds.set(retryBackoffBaseSecondsInitialValue); - logger.fine("Proxy backoff base set to (locally) " + retryBackoffBaseSeconds.get()); + tenantSpecificEntityProps.getGlobalProperties().setRetryBackoffBaseSeconds(null); + logger.fine( + "Proxy backoff base set to (locally) " + + tenantSpecificEntityProps.getGlobalProperties().getRetryBackoffBaseSeconds()); } - - histogramDisabled.set(BooleanUtils.toBoolean(config.getHistogramDisabled())); - traceDisabled.set(BooleanUtils.toBoolean(config.getTraceDisabled())); + tenantSpecificEntityProps + .get(ReportableEntityType.HISTOGRAM) + .setFeatureDisabled(BooleanUtils.isTrue(config.getHistogramDisabled())); + tenantSpecificEntityProps + .get(ReportableEntityType.TRACE) + .setFeatureDisabled(BooleanUtils.isTrue(config.getTraceDisabled())); + tenantSpecificEntityProps + .get(ReportableEntityType.TRACE_SPAN_LOGS) + .setFeatureDisabled(BooleanUtils.isTrue(config.getSpanLogsDisabled())); + tenantSpecificEntityProps + .get(ReportableEntityType.LOGS) + .setFeatureDisabled(BooleanUtils.isTrue(config.getLogsDisabled())); + validationConfiguration.updateFrom(config.getValidationConfiguration()); } catch (RuntimeException e) { - // cannot throw or else configuration update thread would die. + // cannot throw or else configuration update thread would die, so just log it. + logger.log(Level.WARNING, "Error during configuration update", e); + } + try { + super.processConfiguration(tenantName, config); + } catch (RuntimeException e) { + // cannot throw or else configuration update thread would die. it's ok to ignore these. + } + } + + private void updateRateLimiter( + String tenantName, + ReportableEntityType entityType, + @Nullable Boolean collectorSetsRateLimit, + @Nullable Number collectorRateLimit, + @Nullable Number globalRateLimit) { + EntityProperties entityProperties = entityPropertiesFactoryMap.get(tenantName).get(entityType); + RecyclableRateLimiter rateLimiter = entityProperties.getRateLimiter(); + if (rateLimiter != null) { + if (BooleanUtils.isTrue(collectorSetsRateLimit)) { + if (collectorRateLimit != null + && rateLimiter.getRate() != collectorRateLimit.doubleValue()) { + rateLimiter.setRate(collectorRateLimit.doubleValue()); + entityProperties.setDataPerBatch( + Math.min(collectorRateLimit.intValue(), entityProperties.getDataPerBatch())); + logger.warning( + "[" + + tenantName + + "]: " + + entityType.toCapitalizedString() + + " rate limit set to " + + collectorRateLimit + + entityType.getRateUnit() + + " remotely"); + } + } else { + double rateLimit = + Math.min( + entityProperties.getRateLimit(), + ObjectUtils.firstNonNull(globalRateLimit, NO_RATE_LIMIT).intValue()); + if (rateLimiter.getRate() != rateLimit) { + rateLimiter.setRate(rateLimit); + if (entityProperties.getDataPerBatchOriginal() > rateLimit) { + entityProperties.setDataPerBatch((int) rateLimit); + } else { + entityProperties.setDataPerBatch(null); + } + if (rateLimit >= NO_RATE_LIMIT) { + logger.warning( + entityType.toCapitalizedString() + + " rate limit is no longer " + + "enforced by remote"); + } else { + if (proxyCheckinScheduler != null + && proxyCheckinScheduler.getSuccessfulCheckinCount() > 1) { + // this will skip printing this message upon init + logger.warning( + entityType.toCapitalizedString() + + " rate limit restored to " + + rateLimit + + entityType.getRateUnit()); + } + } + } + } } } - protected void startAsManagedThread(Runnable target, @Nullable String threadName) { + protected TokenAuthenticator configureTokenAuthenticator() { + HttpClient httpClient = + HttpClientBuilder.create() + .useSystemProperties() + .setUserAgent(proxyConfig.getHttpUserAgent()) + .setMaxConnPerRoute(10) + .setMaxConnTotal(10) + .setConnectionTimeToLive(1, TimeUnit.MINUTES) + .setRetryHandler( + new DefaultHttpRequestRetryHandler(proxyConfig.getHttpAutoRetries(), true)) + .setDefaultRequestConfig( + RequestConfig.custom() + .setContentCompressionEnabled(true) + .setRedirectsEnabled(true) + .setConnectTimeout(proxyConfig.getHttpConnectTimeout()) + .setConnectionRequestTimeout(proxyConfig.getHttpConnectTimeout()) + .setSocketTimeout(proxyConfig.getHttpRequestTimeout()) + .build()) + .build(); + return TokenAuthenticatorBuilder.create() + .setTokenValidationMethod(proxyConfig.getAuthMethod()) + .setHttpClient(httpClient) + .setTokenIntrospectionServiceUrl(proxyConfig.getAuthTokenIntrospectionServiceUrl()) + .setTokenIntrospectionAuthorizationHeader( + proxyConfig.getAuthTokenIntrospectionAuthorizationHeader()) + .setAuthResponseRefreshInterval(proxyConfig.getAuthResponseRefreshInterval()) + .setAuthResponseMaxTtl(proxyConfig.getAuthResponseMaxTtl()) + .setStaticToken(proxyConfig.getAuthStaticToken()) + .build(); + } + + protected void startAsManagedThread(int port, Runnable target, @Nullable String threadName) { Thread thread = new Thread(target); if (threadName != null) { thread.setName(threadName); } - managedThreads.add(thread); + listeners.put(port, thread); thread.start(); } @Override public void stopListeners() { - for (Thread thread : managedThreads) { - thread.interrupt(); - try { - thread.join(TimeUnit.SECONDS.toMillis(10)); - } catch (InterruptedException e) { - // ignore + listeners.values().forEach(Thread::interrupt); + listeners + .values() + .forEach( + thread -> { + try { + thread.join(TimeUnit.SECONDS.toMillis(10)); + } catch (InterruptedException e) { + // ignore + } + }); + } + + @Override + protected void stopListener(int port) { + Thread listener = listeners.remove(port); + if (listener == null) return; + listener.interrupt(); + try { + listener.join(TimeUnit.SECONDS.toMillis(10)); + } catch (InterruptedException e) { + // ignore + } + handlerFactory.shutdown(String.valueOf(port)); + senderTaskFactory.shutdown(String.valueOf(port)); + } + + @Override + protected void truncateBacklog() { + senderTaskFactory.truncateBuffers(); + } + + private void loadPreprocessors(String preprocessorStr, boolean readfromFE) { + try { + if (readfromFE && preprocessorStr != null) { + // preprocessorStr is str of rules itself + preprocessors.loadFERules(preprocessorStr); + usingLocalFileRules.set(false); + } else { + // preprocessorStr is file path + preprocessors.loadFile(preprocessorStr); + usingLocalFileRules.set(true); } + } catch (FileNotFoundException e) { + throw new RuntimeException("Unable to load preprocessor rules - file does not exist: " + preprocessorStr); } } } diff --git a/proxy/src/main/java/com/wavefront/agent/PushAgentDaemon.java b/proxy/src/main/java/com/wavefront/agent/PushAgentDaemon.java deleted file mode 100644 index f544e7b76..000000000 --- a/proxy/src/main/java/com/wavefront/agent/PushAgentDaemon.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.wavefront.agent; - -import org.apache.commons.daemon.Daemon; -import org.apache.commons.daemon.DaemonContext; -import org.apache.commons.daemon.DaemonInitException; - -/** - * @author Mori Bellamy (mori@wavefront.com) - */ -public class PushAgentDaemon implements Daemon { - - private PushAgent agent; - private DaemonContext daemonContext; - - @Override - public void init(DaemonContext daemonContext) throws DaemonInitException, Exception { - this.daemonContext = daemonContext; - agent = new PushAgent(); - } - - @Override - public void start() throws Exception { - agent.start(daemonContext.getArguments()); - } - - @Override - public void stop() throws Exception { - agent.shutdown(); - } - - @Override - public void destroy() { - - } -} diff --git a/proxy/src/main/java/com/wavefront/agent/QueuedAgentService.java b/proxy/src/main/java/com/wavefront/agent/QueuedAgentService.java deleted file mode 100644 index 0d6b7db93..000000000 --- a/proxy/src/main/java/com/wavefront/agent/QueuedAgentService.java +++ /dev/null @@ -1,1013 +0,0 @@ -package com.wavefront.agent; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Function; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.base.Throwables; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.AtomicDouble; -import com.google.common.util.concurrent.RateLimiter; -import com.google.common.util.concurrent.RecyclableRateLimiter; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -import com.fasterxml.jackson.databind.JsonNode; -import com.github.benmanes.caffeine.cache.CacheLoader; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.LoadingCache; -import com.squareup.tape.FileException; -import com.squareup.tape.FileObjectQueue; -import com.squareup.tape.ObjectQueue; -import com.wavefront.agent.api.ForceQueueEnabledAgentAPI; -import com.wavefront.api.WavefrontAPI; -import com.wavefront.api.agent.AgentConfiguration; -import com.wavefront.api.agent.ShellOutputDTO; -import com.wavefront.common.NamedThreadFactory; -import com.wavefront.ingester.StringLineIngester; -import com.wavefront.metrics.ExpectedAgentMetric; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Counter; -import com.yammer.metrics.core.Gauge; -import com.yammer.metrics.core.Histogram; -import com.yammer.metrics.core.Meter; -import com.yammer.metrics.core.MetricName; -import com.yammer.metrics.core.MetricsRegistry; - -import net.jpountz.lz4.LZ4BlockInputStream; -import net.jpountz.lz4.LZ4BlockOutputStream; - -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.OutputStream; -import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import javax.ws.rs.core.Response; - -import static com.google.common.collect.ImmutableList.of; - -/** - * A wrapper for any WavefrontAPI that queues up any result posting if the backend is not available. - * Current data will always be submitted right away (thus prioritizing live data) while background - * threads will submit backlogged data. - * - * @author Clement Pang (clement@wavefront.com) - */ -public class QueuedAgentService implements ForceQueueEnabledAgentAPI { - - private static final Logger logger = Logger.getLogger(QueuedAgentService.class.getCanonicalName()); - private static final String SERVER_ERROR = "Server error"; - - private final Gson resubmissionTaskMarshaller; - private final WavefrontAPI wrapped; - private final List taskQueues; - private static AtomicInteger splitBatchSize = new AtomicInteger(50000); - private static AtomicDouble retryBackoffBaseSeconds = new AtomicDouble(2.0); - private boolean lastKnownQueueSizeIsPositive = true; - private boolean lastKnownSourceTagQueueSizeIsPositive = true; - private final ExecutorService executorService; - private final String token; - - /** - * A loading cache for tracking queue sizes (refreshed once a minute). Calculating the number of objects across - * all queues can be a non-trivial operation, hence the once-a-minute refresh. - */ - private final LoadingCache queueSizes; - private final List sourceTagTaskQueues; - - private MetricsRegistry metricsRegistry = new MetricsRegistry(); - private Meter resultPostingMeter = metricsRegistry.newMeter(QueuedAgentService.class, "post-result", "results", - TimeUnit.MINUTES); - private Counter permitsGranted = Metrics.newCounter(new MetricName("limiter", "", "permits-granted")); - private Counter permitsDenied = Metrics.newCounter(new MetricName("limiter", "", "permits-denied")); - private Counter permitsRetried = Metrics.newCounter(new MetricName("limiter", "", "permits-retried")); - private final AtomicLong queuePointsCount = new AtomicLong(); - private Gauge queuedPointsCountGauge = null; - /** - * Biases result sizes to the last 5 minutes heavily. This histogram does not see all result - * sizes. The executor only ever processes one posting at any given time and drops the rest. - * {@link #resultPostingMeter} records the actual rate (i.e. sees all posting calls). - */ - private Histogram resultPostingSizes = metricsRegistry.newHistogram(QueuedAgentService.class, "result-size", true); - /** - * A single threaded bounded work queue to update result posting sizes. - */ - private ExecutorService resultPostingSizerExecutorService = new ThreadPoolExecutor(1, 1, 60L, TimeUnit.SECONDS, - new ArrayBlockingQueue(1), new NamedThreadFactory("result-posting-sizer")); - - /** - * Only size postings once every 5 seconds. - */ - private final RateLimiter resultSizingRateLimier = RateLimiter.create(0.2); - /** - * @return bytes per minute for requests submissions. Null if no data is available yet. - */ - @Nullable - public Long getBytesPerMinute() { - if (resultPostingMeter.fifteenMinuteRate() == 0 || resultPostingSizes.mean() == 0 || resultPostingSizes.count() < - 50) { - return null; - } - return (long) (resultPostingSizes.mean() * resultPostingMeter.fifteenMinuteRate()); - } - - public QueuedAgentService(WavefrontAPI service, String bufferFile, final int retryThreads, - final ScheduledExecutorService executorService, boolean purge, - final UUID agentId, final boolean splitPushWhenRateLimited, - @Nullable final RecyclableRateLimiter pushRateLimiter, - @Nullable final String token) throws IOException { - if (retryThreads <= 0) { - logger.severe("You have no retry threads set up. Any points that get rejected will be lost.\n Change this by " + - "setting retryThreads to a value > 0"); - } - if (pushRateLimiter != null && pushRateLimiter.getRate() < 10_000_000) { - logger.info("Point rate limited at the proxy at : " + String.valueOf(pushRateLimiter.getRate())); - } else { - logger.info("No rate limit configured."); - } - resubmissionTaskMarshaller = new GsonBuilder(). - registerTypeHierarchyAdapter(ResubmissionTask.class, new ResubmissionTaskDeserializer()).create(); - this.wrapped = service; - this.taskQueues = Lists.newArrayListWithExpectedSize(retryThreads); - this.sourceTagTaskQueues = Lists.newArrayListWithExpectedSize(retryThreads); - String bufferFileSourceTag = bufferFile + "SourceTag"; - this.executorService = executorService; - this.token = token; - - queueSizes = Caffeine.newBuilder() - .refreshAfterWrite(15, TimeUnit.SECONDS) - .build(new CacheLoader() { - @Override - public AtomicInteger load(@Nonnull ResubmissionTaskQueue key) throws Exception { - return new AtomicInteger(key.size()); - } - - // reuse old object if possible - @Override - public AtomicInteger reload(@Nonnull ResubmissionTaskQueue key, - @Nonnull AtomicInteger oldValue) { - oldValue.set(key.size()); - return oldValue; - } - }); - - for (int i = 0; i < retryThreads; i++) { - final int threadId = i; - File buffer = new File(bufferFile + "." + i); - File bufferSourceTag = new File(bufferFileSourceTag + "." + i); - if (purge) { - if (buffer.delete()) { - logger.warning("Retry buffer has been purged: " + buffer.getAbsolutePath()); - } - if (bufferSourceTag.delete()) { - logger.warning("SourceTag retry buffer has been purged: " + bufferSourceTag - .getAbsolutePath()); - } - } - - ObjectQueue queue = createTaskQueue(agentId, buffer); - - // Having two proxy processes write to the same buffer file simultaneously causes buffer file corruption. - // To prevent concurrent access from another process, we try to obtain exclusive access to each buffer file - // trylock() is platform-specific so there is no iron-clad guarantee, but it works well in most cases - try { - FileChannel channel = new RandomAccessFile(buffer, "rw").getChannel(); - Preconditions.checkNotNull(channel.tryLock()); // fail if tryLock() returns null (lock couldn't be acquired) - } catch (Exception e) { - logger.severe("WF-005: Error requesting exclusive access to the buffer file " + bufferFile + "." + i + - " - please make sure that no other processes access this file and restart the proxy"); - System.exit(-1); - } - - final ResubmissionTaskQueue taskQueue = new ResubmissionTaskQueue(queue, - task -> { - task.service = wrapped; - task.currentAgentId = agentId; - task.token = token; - } - ); - - Runnable taskRunnable = createRunnable(executorService, splitPushWhenRateLimited, threadId, taskQueue, pushRateLimiter); - - executorService.schedule(taskRunnable, (long) (Math.random() * retryThreads), TimeUnit.SECONDS); - taskQueues.add(taskQueue); - - ObjectQueue sourceTagTaskQueue = createTaskQueue(agentId, - bufferSourceTag); - - // Having two proxy processes write to the same buffer file simultaneously causes buffer - // file corruption. To prevent concurrent access from another process, we try to obtain - // exclusive access to each buffer file trylock() is platform-specific so there is no - // iron-clad guarantee, but it works well in most cases - try { - FileChannel channel = new RandomAccessFile(bufferSourceTag, "rw").getChannel(); - Preconditions.checkNotNull(channel.tryLock()); // fail if tryLock() returns null (lock - // couldn't be acquired) - } catch (Exception e) { - logger.severe("WF-005: Error requesting exclusive access to the buffer file " + - bufferFileSourceTag + "." + i + - " - please make sure that no other processes access this file and restart the proxy"); - System.exit(-1); - } - final ResubmissionTaskQueue sourceTagQueue = new ResubmissionTaskQueue(sourceTagTaskQueue, - task -> { - task.service = wrapped; - task.currentAgentId = agentId; - task.token = token; - } - ); - // create a new rate-limiter for the source tag retry queue, because the API calls are - // rate-limited on the server-side as well. We don't want the retry logic to keep hitting - // that rate-limit. - Runnable sourceTagTaskRunnable = createRunnable(executorService, splitPushWhenRateLimited, - threadId, sourceTagQueue, RecyclableRateLimiter.create(1, 1)); - executorService.schedule(sourceTagTaskRunnable, (long) (Math.random() * retryThreads), - TimeUnit.SECONDS); - sourceTagTaskQueues.add(sourceTagQueue); - } - - if (retryThreads > 0) { - executorService.scheduleAtFixedRate(() -> { - try { - Supplier> sizes = () -> taskQueues.stream() - .map(k -> Math.max(0, queueSizes.get(k).intValue())); - if (sizes.get().anyMatch(i -> i > 0)) { - lastKnownQueueSizeIsPositive = true; - logger.info("current retry queue sizes: [" + - sizes.get().map(Object::toString).collect(Collectors.joining("/")) + "]"); - } else if (lastKnownQueueSizeIsPositive) { - lastKnownQueueSizeIsPositive = false; - queuePointsCount.set(0); - if (queuedPointsCountGauge == null) { - // since we don't persist the number of points in the queue between proxy restarts yet, and Tape library - // only lets us know the number of tasks in the queue, start reporting ~agent.buffer.points-count - // metric only after it's confirmed that the retry queue is empty, as going through the entire queue - // to calculate the number of points can be a very costly operation. - queuedPointsCountGauge = Metrics.newGauge(new MetricName("buffer", "", "points-count"), - new Gauge() { - @Override - public Long value() { - return queuePointsCount.get(); - } - } - ); - } - logger.info("retry queue has been cleared"); - } - - // do the same thing for sourceTagQueues - List sourceTagQueueSizes = Lists.newArrayList(Lists.transform - (sourceTagTaskQueues, new Function, Integer>() { - @Override - public Integer apply(ObjectQueue input) { - return input.size(); - } - })); - - if (Iterables.tryFind(sourceTagQueueSizes, new Predicate() { - @Override - public boolean apply(Integer input) { - return input > 0; - } - }).isPresent()) { - lastKnownSourceTagQueueSizeIsPositive = true; - logger.warning("current source tag retry queue sizes: [" + Joiner.on("/").join - (sourceTagQueueSizes) + "]"); - } else if (lastKnownSourceTagQueueSizeIsPositive) { - lastKnownSourceTagQueueSizeIsPositive = false; - logger.warning("source tag retry queue has been cleared"); - } - } catch (Exception ex) { - logger.warning("Exception " + ex); - } - }, 0, 5, TimeUnit.SECONDS); - } - - Metrics.newGauge(ExpectedAgentMetric.BUFFER_BYTES_PER_MINUTE.metricName, new Gauge() { - @Override - public Long value() { - return getBytesPerMinute(); - } - }); - - Metrics.newGauge(ExpectedAgentMetric.CURRENT_QUEUE_SIZE.metricName, new Gauge() { - @Override - public Long value() { - return getQueuedTasksCount(); - } - }); - } - - private Runnable createRunnable(final ScheduledExecutorService executorService, - final boolean splitPushWhenRateLimited, - final int threadId, - final ResubmissionTaskQueue taskQueue, - final RecyclableRateLimiter pushRateLimiter) { - return new Runnable() { - private int backoffExponent = 1; - - @Override - public void run() { - int successes = 0; - int failures = 0; - boolean rateLimiting = false; - try { - logger.fine("[RETRY THREAD " + threadId + "] TASK STARTING"); - while (taskQueue.size() > 0 && taskQueue.size() > failures) { - if (Thread.currentThread().isInterrupted()) return; - ResubmissionTask task = taskQueue.peek(); - int taskSize = task == null ? 0 : task.size(); - if (pushRateLimiter != null && pushRateLimiter.getAvailablePermits() < pushRateLimiter.getRate()) { - // if there's less than 1 second worth of accumulated credits, don't process the backlog queue - rateLimiting = true; - permitsDenied.inc(taskSize); - break; - } - - if (pushRateLimiter != null && taskSize > 0) { - pushRateLimiter.acquire(taskSize); - permitsGranted.inc(taskSize); - } - - boolean removeTask = true; - try { - if (task != null) { - task.execute(null); - successes++; - } - } catch (Exception ex) { - if (pushRateLimiter != null) { - pushRateLimiter.recyclePermits(taskSize); - permitsRetried.inc(taskSize); - } - failures++; - //noinspection ThrowableResultOfMethodCallIgnored - if (Throwables.getRootCause(ex) instanceof QueuedPushTooLargeException) { - // this should split this task, remove it from the queue, and not try more tasks - logger.warning("[RETRY THREAD " + threadId + "] Wavefront server rejected push with " + - "HTTP 413: request too large - splitting data into smaller chunks to retry. "); - List splitTasks = task.splitTask(); - for (ResubmissionTask smallerTask : splitTasks) { - taskQueue.add(smallerTask); - queueSizes.get(taskQueue).incrementAndGet(); - queuePointsCount.addAndGet(smallerTask.size()); - } - break; - } else //noinspection ThrowableResultOfMethodCallIgnored - if (Throwables.getRootCause(ex) instanceof RejectedExecutionException) { - // this should either split and remove the original task or keep it at front - // it also should not try any more tasks - logger.warning("[RETRY THREAD " + threadId + "] Wavefront server rejected the submission " + - "(global rate limit exceeded) - will attempt later."); - if (splitPushWhenRateLimited) { - List splitTasks = task.splitTask(); - for (ResubmissionTask smallerTask : splitTasks) { - taskQueue.add(smallerTask); - queueSizes.get(taskQueue).incrementAndGet(); - queuePointsCount.addAndGet(smallerTask.size()); - } - } else { - removeTask = false; - } - break; - } else { - logger.log(Level.WARNING, "[RETRY THREAD " + threadId + "] cannot submit data to Wavefront servers. Will " + - "re-attempt later", Throwables.getRootCause(ex)); - } - // this can potentially cause a duplicate task to be injected (but since submission is mostly - // idempotent it's not really a big deal) - task.service = null; - task.currentAgentId = null; - taskQueue.add(task); - queueSizes.get(taskQueue).incrementAndGet(); - queuePointsCount.addAndGet(taskSize); - if (failures > 10) { - logger.warning("[RETRY THREAD " + threadId + "] saw too many submission errors. Will " + - "re-attempt later"); - break; - } - } finally { - if (removeTask) { - taskQueue.remove(); - queueSizes.get(taskQueue).decrementAndGet(); - queuePointsCount.addAndGet(-taskSize); - } - } - } - } catch (Throwable ex) { - logger.log(Level.WARNING, "[RETRY THREAD " + threadId + "] unexpected exception", ex); - } finally { - logger.fine("[RETRY THREAD " + threadId + "] Successful Batches: " + successes + - ", Failed Batches: " + failures); - if (rateLimiting) { - logger.fine("[RETRY THREAD " + threadId + "] Rate limit reached, will re-attempt later"); - // if proxy rate limit exceeded, try again in 250..500ms (to introduce some degree of fairness) - executorService.schedule(this, 250 + (int) (Math.random() * 250), TimeUnit.MILLISECONDS); - } else { - if (successes == 0 && failures != 0) { - backoffExponent = Math.min(4, backoffExponent + 1); // caps at 2*base^4 - } else { - backoffExponent = 1; - } - long next = (long) ((Math.random() + 1.0) * - Math.pow(retryBackoffBaseSeconds.get(), backoffExponent)); - logger.fine("[RETRY THREAD " + threadId + "] RESCHEDULING in " + next); - executorService.schedule(this, next, TimeUnit.SECONDS); - } - } - } - }; - } - - private ObjectQueue createTaskQueue(final UUID agentId, File buffer) throws - IOException { - return new FileObjectQueue<>(buffer, - new FileObjectQueue.Converter() { - @Override - public ResubmissionTask from(byte[] bytes) throws IOException { - try { - ObjectInputStream ois = new ObjectInputStream(new LZ4BlockInputStream(new ByteArrayInputStream(bytes))); - return (ResubmissionTask) ois.readObject(); - } catch (Throwable t) { - logger.warning("Failed to read a single retry submission from buffer, ignoring: " + t); - return null; - } - } - - @Override - public void toStream(ResubmissionTask o, OutputStream bytes) throws IOException { - LZ4BlockOutputStream lz4BlockOutputStream = new LZ4BlockOutputStream(bytes); - ObjectOutputStream oos = new ObjectOutputStream(lz4BlockOutputStream); - oos.writeObject(o); - oos.close(); - lz4BlockOutputStream.close(); - } - }); - } - - public void shutdown() { - executorService.shutdown(); - } - - public static void setRetryBackoffBaseSeconds(AtomicDouble newSecs) { - retryBackoffBaseSeconds = newSecs; - } - - public static void setSplitBatchSize(AtomicInteger newSize) { - splitBatchSize = newSize; - } - - public long getQueuedTasksCount() { - long toReturn = 0; - for (ResubmissionTaskQueue taskQueue : taskQueues) { - toReturn += taskQueue.size(); - } - return toReturn; - } - - public long getQueuedSourceTagTasksCount() { - long toReturn = 0; - for (ObjectQueue taskQueue : sourceTagTaskQueues) { - toReturn += taskQueue.size(); - } - return toReturn; - } - - private ResubmissionTaskQueue getSmallestQueue() { - Optional smallestQueue = taskQueues.stream() - .min(Comparator.comparingInt(q -> queueSizes.get(q).intValue())); - return smallestQueue.orElse(null); - } - - private Runnable getPostingSizerTask(final ResubmissionTask task) { - return () -> { - try { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - LZ4BlockOutputStream lz4OutputStream = new LZ4BlockOutputStream(outputStream); - ObjectOutputStream oos = new ObjectOutputStream(lz4OutputStream); - oos.writeObject(task); - oos.close(); - lz4OutputStream.close(); - resultPostingSizes.update(outputStream.size()); - } catch (Throwable t) { - // ignored. this is a stats task. - } - }; - } - - private void scheduleTaskForSizing(ResubmissionTask task) { - try { - if (resultSizingRateLimier.tryAcquire()) { - resultPostingSizerExecutorService.submit(getPostingSizerTask(task)); - } - } catch (RejectedExecutionException ex) { - // ignored. - } catch (RuntimeException ex) { - logger.warning("cannot size a submission task for stats tracking: " + ex); - } - } - - @Override - public AgentConfiguration getConfig(UUID agentId, String hostname, Long currentMillis, - Long bytesLeftForbuffer, Long bytesPerMinuteForBuffer, Long currentQueueSize, - String token, String version) { - return wrapped.getConfig(agentId, hostname, currentMillis, bytesLeftForbuffer, bytesPerMinuteForBuffer, - currentQueueSize, token, version); - } - - @Override - public AgentConfiguration checkin(UUID agentId, String hostname, String token, String version, Long currentMillis, - Boolean localAgent, JsonNode agentMetrics, Boolean pushAgent, Boolean ephemeral) { - return wrapped.checkin(agentId, hostname, token, version, currentMillis, localAgent, agentMetrics, pushAgent, ephemeral); - } - - @Override - public Response postWorkUnitResult(final UUID agentId, final UUID workUnitId, final UUID targetId, - final ShellOutputDTO shellOutputDTO) { - throw new UnsupportedOperationException("postWorkUnitResult is deprecated"); - } - - @Override - public Response postWorkUnitResult(UUID agentId, UUID workUnitId, UUID targetId, ShellOutputDTO shellOutputDTO, - boolean forceToQueue) { - throw new UnsupportedOperationException("postWorkUnitResult is deprecated"); - } - - @Override - public Response postPushData(final UUID agentId, final UUID workUnitId, final Long currentMillis, - final String format, final String pushData) { - return this.postPushData(agentId, workUnitId, currentMillis, format, pushData, false); - } - - @Override - public Response postPushData(UUID agentId, UUID workUnitId, Long currentMillis, String format, String pushData, - boolean forceToQueue) { - PostPushDataResultTask task = new PostPushDataResultTask(agentId, workUnitId, currentMillis, format, pushData); - - if (forceToQueue) { - // bypass the charade of posting to the wrapped agentAPI. Just go straight to the retry queue - addTaskToSmallestQueue(task); - return Response.status(Response.Status.NOT_ACCEPTABLE).build(); - } else { - try { - resultPostingMeter.mark(); - parsePostingResponse(wrapped.postPushData(agentId, workUnitId, currentMillis, format, pushData)); - - scheduleTaskForSizing(task); - } catch (RuntimeException ex) { - List splitTasks = handleTaskRetry(ex, task); - for (PostPushDataResultTask splitTask : splitTasks) { - // we need to ensure that we use the latest agent id. - postPushData(agentId, splitTask.getWorkUnitId(), splitTask.getCurrentMillis(), - splitTask.getFormat(), splitTask.getPushData()); - } - return Response.status(Response.Status.NOT_ACCEPTABLE).build(); - } - return Response.ok().build(); - } - } - - /** - * @return list of tasks to immediately retry - */ - private > List handleTaskRetry(RuntimeException failureException, T taskToRetry) { - if (failureException instanceof QueuedPushTooLargeException) { - List resubmissionTasks = taskToRetry.splitTask(); - // there are split tasks, so go ahead and return them - // otherwise, nothing got split, so this should just get queued up - if (resubmissionTasks.size() > 1) { - return resubmissionTasks; - } - } - logger.warning("Cannot post push data result to Wavefront servers. " + - "Will enqueue and retry later: " + Throwables.getRootCause(failureException)); - addTaskToSmallestQueue(taskToRetry); - return Collections.emptyList(); - } - - private void handleSourceTagTaskRetry(RuntimeException failureException, - PostSourceTagResultTask taskToRetry) { - logger.warning("Cannot post push data result to Wavefront servers. Will enqueue and retry " + - "later: " + failureException); - addSourceTagTaskToSmallestQueue(taskToRetry); - } - - private void addSourceTagTaskToSmallestQueue(PostSourceTagResultTask taskToRetry) { - // we need to make sure the we preserve the order of operations for each source - ResubmissionTaskQueue queue = sourceTagTaskQueues.get(Math.abs(taskToRetry.id.hashCode()) % sourceTagTaskQueues.size()); - if (queue != null) { - try { - queue.add(taskToRetry); - } catch (FileException ex) { - logger.log(Level.SEVERE, "CRITICAL (Losing sourceTags!): WF-1: Submission queue is " + - "full.", ex); - } - } else { - logger.severe("CRITICAL (Losing sourceTags!): WF-2: No retry queues found."); - } - } - - private void addTaskToSmallestQueue(ResubmissionTask taskToRetry) { - ResubmissionTaskQueue queue = getSmallestQueue(); - if (queue != null) { - try { - queue.add(taskToRetry); - queueSizes.get(queue).incrementAndGet(); - queuePointsCount.addAndGet(taskToRetry.size()); - } catch (FileException e) { - logger.log(Level.SEVERE, "CRITICAL (Losing points!): WF-1: Submission queue is full.", e); - } - } else { - logger.severe("CRITICAL (Losing points!): WF-2: No retry queues found."); - } - } - - private static void parsePostingResponse(Response response) { - if (response == null) throw new RuntimeException("No response from server"); - try { - if (response.getStatus() < 200 || response.getStatus() >= 300) { - if (response.getStatus() == Response.Status.NOT_ACCEPTABLE.getStatusCode()) { - throw new RejectedExecutionException("Response not accepted by server: " + response.getStatus()); - } else if (response.getStatus() == Response.Status.REQUEST_ENTITY_TOO_LARGE.getStatusCode()) { - throw new QueuedPushTooLargeException("Request too large: " + response.getStatus()); - } else if (response.getStatus() == 407 || response.getStatus() == 408) { - boolean isWavefrontResponse = false; - // check if the HTTP 407/408 response was actually received from Wavefront - if it's a JSON object - // containing "code" key, with value equal to the HTTP response code, it's most likely from us. - try { - Map resp = new HashMap<>(); - resp = (Map) new Gson().fromJson(response.readEntity(String.class), resp.getClass()); - if (resp.containsKey("code") && resp.get("code") instanceof Number && - ((Number) resp.get("code")).intValue() == response.getStatus()) { - isWavefrontResponse = true; - } - } catch (Exception ex) { - // ignore - } - if (isWavefrontResponse) { - throw new RuntimeException("Response not accepted by server: " + response.getStatus() + - " unclaimed proxy - please verify that your token is valid and has Agent Management permission!"); - } else { - throw new RuntimeException("HTTP " + response.getStatus() + ": Please verify your " + - "network/HTTP proxy settings!"); - } - } else { - throw new RuntimeException(SERVER_ERROR + ": " + response.getStatus()); - } - } - } finally { - response.close(); - } - } - - @Override - public void agentError(UUID agentId, String details) { - wrapped.agentError(agentId, details); - } - - @Override - public void agentConfigProcessed(UUID agentId) { - wrapped.agentConfigProcessed(agentId); - } - - @Override - public void hostConnectionFailed(UUID agentId, UUID hostId, String details) { - throw new UnsupportedOperationException("Invalid operation"); - } - - @Override - public void hostConnectionEstablished(UUID agentId, UUID hostId) { - throw new UnsupportedOperationException("Invalid operation"); - } - - @Override - public void hostAuthenticated(UUID agentId, UUID hostId) { - throw new UnsupportedOperationException("Invalid operation"); - } - - @Override - public Response removeTag(String id, String token, String tagValue) { - return removeTag(id, tagValue, false); - } - - @Override - public Response removeDescription(String id, String token) { - return removeDescription(id, false); - } - - @Override - public Response setTags(String id, String token, List tagValuesToSet) { - return setTags(id, tagValuesToSet, false); - } - - @Override - public Response setDescription(String id, String token, String description) { - return setDescription(id, description, false); - } - - @Override - public Response setTags(String id, List tagValuesToSet, boolean forceToQueue) { - PostSourceTagResultTask task = new PostSourceTagResultTask(id, tagValuesToSet, - PostSourceTagResultTask.ActionType.save, PostSourceTagResultTask.MessageType.tag, token); - - if (forceToQueue) { - // bypass the charade of posting to the wrapped agentAPI. Just go straight to the retry queue - addSourceTagTaskToSmallestQueue(task); - return Response.status(Response.Status.NOT_ACCEPTABLE).build(); - } else { - // invoke server side API - try { - Response response = wrapped.setTags(id, token, tagValuesToSet); - logger.info("Received response status = " + response.getStatus()); - parsePostingResponse(response); - } catch (RuntimeException ex) { - // If it is a server error then no need of retrying - if (!ex.getMessage().startsWith(SERVER_ERROR)) - handleSourceTagTaskRetry(ex, task); - logger.warning("Unable to process the source tag request" + ExceptionUtils - .getFullStackTrace(ex)); - return Response.status(Response.Status.NOT_ACCEPTABLE).build(); - } - return Response.ok().build(); - } - } - - @Override - public Response removeDescription(String id, boolean forceToQueue) { - PostSourceTagResultTask task = new PostSourceTagResultTask(id, StringUtils.EMPTY, - PostSourceTagResultTask.ActionType.delete, PostSourceTagResultTask.MessageType.desc, token); - - if (forceToQueue) { - // bypass the charade of posting to the wrapped agentAPI. Just go straight to the retry queue - addSourceTagTaskToSmallestQueue(task); - return Response.status(Response.Status.NOT_ACCEPTABLE).build(); - } else { - // invoke server side API - try { - parsePostingResponse(wrapped.removeDescription(id, token)); - } catch (RuntimeException ex) { - // If it is a server error then no need of retrying - if (!ex.getMessage().startsWith(SERVER_ERROR)) - handleSourceTagTaskRetry(ex, task); - logger.warning("Unable to process the source tag request" + ExceptionUtils - .getFullStackTrace(ex)); - return Response.status(Response.Status.NOT_ACCEPTABLE).build(); - } - return Response.ok().build(); - } - } - - @Override - public Response setDescription(String id, String desc, boolean forceToQueue) { - PostSourceTagResultTask task = new PostSourceTagResultTask(id, desc, - PostSourceTagResultTask.ActionType.save, PostSourceTagResultTask.MessageType.desc, token); - - if (forceToQueue) { - // bypass the charade of posting to the wrapped agentAPI. Just go straight to the retry queue - addSourceTagTaskToSmallestQueue(task); - return Response.status(Response.Status.NOT_ACCEPTABLE).build(); - } else { - // invoke server side API - try { - parsePostingResponse(wrapped.setDescription(id, token, desc)); - } catch (RuntimeException ex) { - // If it is a server error then no need of retrying - if (!ex.getMessage().startsWith(SERVER_ERROR)) - handleSourceTagTaskRetry(ex, task); - logger.warning("Unable to process the source tag request" + ExceptionUtils - .getFullStackTrace(ex)); - return Response.status(Response.Status.NOT_ACCEPTABLE).build(); - } - return Response.ok().build(); - } - } - - @Override - public Response removeTag(String id, String tagValue, boolean forceToQueue) { - - PostSourceTagResultTask task = new PostSourceTagResultTask(id, tagValue, - PostSourceTagResultTask.ActionType.delete, PostSourceTagResultTask.MessageType.tag, token); - - if (forceToQueue) { - // bypass the charade of posting to the wrapped agentAPI. Just go straight to the retry queue - addSourceTagTaskToSmallestQueue(task); - return Response.status(Response.Status.NOT_ACCEPTABLE).build(); - } else { - // invoke server side API - try { - parsePostingResponse(wrapped.removeTag(id, token, tagValue)); - } catch (RuntimeException ex) { - // If it is a server error then no need of retrying - if (!ex.getMessage().startsWith(SERVER_ERROR)) - handleSourceTagTaskRetry(ex, task); - logger.warning("Unable to process the source tag request" + ExceptionUtils - .getFullStackTrace(ex)); - return Response.status(Response.Status.NOT_ACCEPTABLE).build(); - } - return Response.ok().build(); - } - } - - public static class PostSourceTagResultTask extends ResubmissionTask { - private final String id; - private final String[] tagValues; - private final String description; - private final int taskSize; - - public enum ActionType {save, delete} - public enum MessageType {tag, desc} - private final ActionType actionType; - private final MessageType messageType; - - public PostSourceTagResultTask(String id, String tagValue, ActionType actionType, MessageType msgType, - String token) { - this.id = id; - if (msgType == MessageType.desc) { - description = tagValue; - tagValues = ArrayUtils.EMPTY_STRING_ARRAY; - } - else { - tagValues = new String[]{tagValue}; - description = StringUtils.EMPTY; - } - this.actionType = actionType; - this.messageType = msgType; - this.taskSize = 1; - this.token = token; - } - - public PostSourceTagResultTask(String id, List tagValuesToSet, ActionType actionType, MessageType msgType, - String token) { - this.id = id; - this.tagValues = tagValuesToSet.toArray(new String[tagValuesToSet.size()]); - description = StringUtils.EMPTY; - this.actionType = actionType; - this.messageType = msgType; - this.taskSize = 1; - this.token = token; - } - - @Override - public List splitTask() { - // currently this is a no-op - List splitTasks = Lists.newArrayList(); - splitTasks.add(new PostSourceTagResultTask(id, tagValues[0], this.actionType, - this.messageType, this.token)); - return splitTasks; - } - - @Override - public int size() { - return taskSize; - } - - @Override - public void execute(Object callback) { - Response response; - try { - switch (messageType) { - case tag: - if (actionType == ActionType.delete) - response = service.removeTag(id, token, tagValues[0]); - else - response = service.setTags(id, token, Arrays.asList(tagValues)); - break; - case desc: - if (actionType == ActionType.delete) - response = service.removeDescription(id, token); - else - response = service.setDescription(id, token, description); - break; - default: - logger.warning("Invalid message type."); - response = Response.serverError().build(); - } - } catch (Exception ex) { - throw new RuntimeException(SERVER_ERROR + ": " + Throwables.getRootCause(ex)); - } - parsePostingResponse(response); - } - } - - public static class PostPushDataResultTask extends ResubmissionTask { - private static final long serialVersionUID = 1973695079812309903L; // to ensure backwards compatibility - - private final UUID agentId; - private final UUID workUnitId; - private final Long currentMillis; - private final String format; - private final String pushData; - private final int taskSize; - - private transient Histogram timeSpentInQueue; - - public PostPushDataResultTask(UUID agentId, UUID workUnitId, Long currentMillis, String format, String pushData) { - this.agentId = agentId; - this.workUnitId = workUnitId; - this.currentMillis = currentMillis; - this.format = format; - this.pushData = pushData; - this.taskSize = StringLineIngester.pushDataSize(pushData); - } - - @Override - public void execute(Object callback) { - parsePostingResponse(service.postPushData(currentAgentId, workUnitId, currentMillis, format, pushData)); - if (timeSpentInQueue == null) { - timeSpentInQueue = Metrics.newHistogram(new MetricName("buffer", "", "queue-time")); - } - // timestamps on PostPushDataResultTask are local system clock, not drift-corrected clock - timeSpentInQueue.update(System.currentTimeMillis() - currentMillis); - } - - @Override - public List splitTask() { - // pull the pushdata back apart to split and put back together - List splitTasks = Lists.newArrayList(); - - List dataIndex = StringLineIngester.indexPushData(pushData); - - int numDatum = dataIndex.size() / 2; - if (numDatum > 1) { - // in this case, at least split the strings in 2 batches. batch size must be less - // than splitBatchSize - int stride = Math.min(splitBatchSize.get(), (int) Math.ceil((float) numDatum / 2.0)); - int endingIndex = 0; - for (int startingIndex = 0; endingIndex < numDatum - 1; startingIndex += stride) { - endingIndex = Math.min(numDatum, startingIndex + stride) - 1; - splitTasks.add(new PostPushDataResultTask(agentId, workUnitId, currentMillis, format, - pushData.substring(dataIndex.get(startingIndex * 2), dataIndex.get(endingIndex * 2 + 1)))); - } - } else { - // 1 or 0 - splitTasks.add(new PostPushDataResultTask(agentId, workUnitId, currentMillis, format, pushData)); - } - - return splitTasks; - } - - @Override - public int size() { - return taskSize; - } - - @VisibleForTesting - public UUID getAgentId() { - return agentId; - } - - @VisibleForTesting - public UUID getWorkUnitId() { - return workUnitId; - } - - @VisibleForTesting - public Long getCurrentMillis() { - return currentMillis; - } - - @VisibleForTesting - public String getFormat() { - return format; - } - - @VisibleForTesting - public String getPushData() { - return pushData; - } - } -} diff --git a/proxy/src/main/java/com/wavefront/agent/QueuedPushTooLargeException.java b/proxy/src/main/java/com/wavefront/agent/QueuedPushTooLargeException.java deleted file mode 100644 index 8365035b0..000000000 --- a/proxy/src/main/java/com/wavefront/agent/QueuedPushTooLargeException.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.wavefront.agent; - -import java.util.concurrent.RejectedExecutionException; - -/** - * @author Andrew Kao (andrew@wavefront.com) - */ -public class QueuedPushTooLargeException extends RejectedExecutionException { - public QueuedPushTooLargeException(String message) { - super(message); - } -} diff --git a/proxy/src/main/java/com/wavefront/agent/ResubmissionTask.java b/proxy/src/main/java/com/wavefront/agent/ResubmissionTask.java deleted file mode 100644 index d26139625..000000000 --- a/proxy/src/main/java/com/wavefront/agent/ResubmissionTask.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.wavefront.agent; - -import com.squareup.tape.Task; -import com.wavefront.api.WavefrontAPI; - -import java.io.Serializable; -import java.util.List; -import java.util.UUID; - -/** - * A task for resubmission. - * - * @author Clement Pang (clement@wavefront.com). - */ -public abstract class ResubmissionTask> implements Task, Serializable { - - /** - * To be injected. Should be null when serialized. - */ - protected transient WavefrontAPI service = null; - - /** - * To be injected. Should be null when serialized. - */ - protected transient UUID currentAgentId = null; - - /** - * To be injected. Should be null when serialized. - */ - protected transient String token = null; - - /** - * @return The relative size of the task - */ - public abstract int size(); - - public abstract List splitTask(); -} diff --git a/proxy/src/main/java/com/wavefront/agent/ResubmissionTaskDeserializer.java b/proxy/src/main/java/com/wavefront/agent/ResubmissionTaskDeserializer.java deleted file mode 100644 index bdb8bfe76..000000000 --- a/proxy/src/main/java/com/wavefront/agent/ResubmissionTaskDeserializer.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.wavefront.agent; - -import com.google.gson.Gson; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; - -import java.lang.reflect.Type; - -/** - * Deserializer of ResubmissionTasks from JSON. - * - * @author Clement Pang (clement@wavefront.com) - */ -public class ResubmissionTaskDeserializer implements - JsonSerializer, JsonDeserializer { - - private static final String CLASS_META_KEY = "CLASS_META_KEY"; - private static final Gson gson = new Gson(); - - @Override - public Object deserialize(JsonElement jsonElement, Type type, - JsonDeserializationContext jsonDeserializationContext) - throws JsonParseException { - JsonObject jsonObj = jsonElement.getAsJsonObject(); - if (!jsonObj.has(CLASS_META_KEY)) { - // cannot deserialize. - return null; - } - String className = jsonObj.get(CLASS_META_KEY).getAsString(); - try { - Class clz = Class.forName(className); - return gson.fromJson(jsonElement, clz); - } catch (ClassNotFoundException e) { - // can no longer parse results. - return null; - } - } - - @Override - public JsonElement serialize(Object object, Type type, - JsonSerializationContext jsonSerializationContext) { - JsonElement jsonEle = gson.toJsonTree(object); - jsonEle.getAsJsonObject().addProperty(CLASS_META_KEY, - object.getClass().getName()); - return jsonEle; - } - -} diff --git a/proxy/src/main/java/com/wavefront/agent/ResubmissionTaskQueue.java b/proxy/src/main/java/com/wavefront/agent/ResubmissionTaskQueue.java deleted file mode 100644 index f987fce13..000000000 --- a/proxy/src/main/java/com/wavefront/agent/ResubmissionTaskQueue.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.wavefront.agent; - -import com.squareup.tape.ObjectQueue; -import com.squareup.tape.TaskInjector; -import com.squareup.tape.TaskQueue; - -import java.util.concurrent.locks.ReentrantLock; - -/** - * Thread-safe TaskQueue for holding ResubmissionTask objects - * - * @author vasily@wavefront.com - */ -public class ResubmissionTaskQueue extends TaskQueue { - - // maintain a fair lock on the queue - private ReentrantLock queueLock = new ReentrantLock(true); - - public ResubmissionTaskQueue(ObjectQueue objectQueue, TaskInjector taskInjector) { - super(objectQueue, taskInjector); - } - - @Override - public void add(ResubmissionTask task) { - queueLock.lock(); - try { - super.add(task); - } finally { - queueLock.unlock(); - } - } - - @Override - public ResubmissionTask peek() { - ResubmissionTask task; - queueLock.lock(); - try { - task = super.peek(); - } finally { - queueLock.unlock(); - } - return task; - } - - @Override - public void remove() { - queueLock.lock(); - try { - super.remove(); - } finally { - queueLock.unlock(); - } - } - - -} diff --git a/proxy/src/main/java/com/wavefront/agent/SSLConnectionSocketFactoryImpl.java b/proxy/src/main/java/com/wavefront/agent/SSLConnectionSocketFactoryImpl.java index 91696cd27..9cdffe379 100644 --- a/proxy/src/main/java/com/wavefront/agent/SSLConnectionSocketFactoryImpl.java +++ b/proxy/src/main/java/com/wavefront/agent/SSLConnectionSocketFactoryImpl.java @@ -1,14 +1,13 @@ package com.wavefront.agent; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; import org.apache.http.HttpHost; import org.apache.http.conn.socket.LayeredConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.protocol.HttpContext; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; - /** * Delegated SSLConnectionSocketFactory that sets SoTimeout explicitly (for Apache HttpClient). * @@ -31,15 +30,23 @@ public Socket createSocket(HttpContext context) throws IOException { } @Override - public Socket connectSocket(int connectTimeout, Socket sock, HttpHost host, InetSocketAddress remoteAddress, - InetSocketAddress localAddress, HttpContext context) throws IOException { - Socket socket1 = delegate.connectSocket(soTimeout, sock, host, remoteAddress, localAddress, context); + public Socket connectSocket( + int connectTimeout, + Socket sock, + HttpHost host, + InetSocketAddress remoteAddress, + InetSocketAddress localAddress, + HttpContext context) + throws IOException { + Socket socket1 = + delegate.connectSocket(soTimeout, sock, host, remoteAddress, localAddress, context); socket1.setSoTimeout(soTimeout); return socket1; } @Override - public Socket createLayeredSocket(Socket socket, String target, int port, HttpContext context) throws IOException { + public Socket createLayeredSocket(Socket socket, String target, int port, HttpContext context) + throws IOException { Socket socket1 = delegate.createLayeredSocket(socket, target, port, context); socket1.setSoTimeout(soTimeout); return socket1; diff --git a/proxy/src/main/java/com/wavefront/agent/SSLSocketFactoryImpl.java b/proxy/src/main/java/com/wavefront/agent/SSLSocketFactoryImpl.java deleted file mode 100644 index 9ebcf28d9..000000000 --- a/proxy/src/main/java/com/wavefront/agent/SSLSocketFactoryImpl.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.wavefront.agent; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.Socket; - -import javax.net.ssl.SSLSocketFactory; - -/** - * Delegated SSLSocketFactory that sets SoTimeout explicitly. - * - * @author Clement Pang (clement@wavefront.com). - */ -public class SSLSocketFactoryImpl extends SSLSocketFactory { - private final SSLSocketFactory delegate; - private final int soTimeout; - - public SSLSocketFactoryImpl(SSLSocketFactory delegate, int soTimeoutMs) { - this.delegate = delegate; - this.soTimeout = soTimeoutMs; - } - - @Override - public Socket createSocket(Socket socket, String s, int i, boolean b) throws IOException { - Socket socket1 = delegate.createSocket(socket, s, i, b); - socket1.setSoTimeout(soTimeout); - return socket1; - } - - @Override - public String[] getDefaultCipherSuites() { - return delegate.getDefaultCipherSuites(); - } - - @Override - public String[] getSupportedCipherSuites() { - return delegate.getSupportedCipherSuites(); - } - - @Override - public Socket createSocket() throws IOException { - Socket socket = delegate.createSocket(); - socket.setSoTimeout(soTimeout); - return socket; - } - - @Override - public Socket createSocket(InetAddress inetAddress, int i) throws IOException { - Socket socket = delegate.createSocket(inetAddress, i); - socket.setSoTimeout(soTimeout); - return socket; - } - - @Override - public Socket createSocket(InetAddress inetAddress, int i, InetAddress inetAddress1, int i1) throws IOException { - Socket socket = delegate.createSocket(inetAddress, i, inetAddress1, i1); - socket.setSoTimeout(soTimeout); - return socket; - } - - @Override - public Socket createSocket(String s, int i) throws IOException { - Socket socket = delegate.createSocket(s, i); - socket.setSoTimeout(soTimeout); - return socket; - } - - @Override - public Socket createSocket(String s, int i, InetAddress inetAddress, int i1) throws IOException { - Socket socket = delegate.createSocket(s, i, inetAddress, i1); - socket.setSoTimeout(soTimeout); - return socket; - } -} diff --git a/proxy/src/main/java/com/wavefront/agent/SharedMetricsRegistry.java b/proxy/src/main/java/com/wavefront/agent/SharedMetricsRegistry.java index 35f850ad0..014bc19e1 100644 --- a/proxy/src/main/java/com/wavefront/agent/SharedMetricsRegistry.java +++ b/proxy/src/main/java/com/wavefront/agent/SharedMetricsRegistry.java @@ -4,7 +4,9 @@ public class SharedMetricsRegistry extends MetricsRegistry { - private static SharedMetricsRegistry INSTANCE = new SharedMetricsRegistry(); + private static final SharedMetricsRegistry INSTANCE = new SharedMetricsRegistry(); + + private SharedMetricsRegistry() {} public static SharedMetricsRegistry getInstance() { return INSTANCE; diff --git a/proxy/src/main/java/com/wavefront/agent/TenantInfo.java b/proxy/src/main/java/com/wavefront/agent/TenantInfo.java new file mode 100644 index 000000000..44a074f59 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/TenantInfo.java @@ -0,0 +1,7 @@ +package com.wavefront.agent; + +public interface TenantInfo { + String getWFServer(); + + String getBearerToken(); +} diff --git a/proxy/src/main/java/com/wavefront/agent/TokenExchangeResponseDTO.java b/proxy/src/main/java/com/wavefront/agent/TokenExchangeResponseDTO.java new file mode 100644 index 000000000..04c7a206d --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/TokenExchangeResponseDTO.java @@ -0,0 +1,81 @@ +package com.wavefront.agent; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Objects; +import javax.annotation.Nonnull; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class TokenExchangeResponseDTO { + + @JsonProperty("id_token") + private String idToken; + + @JsonProperty("token_type") + private String tokenType; + + @JsonProperty("expires_in") + private int expiresIn; + + private String scope; + + @JsonProperty("access_token") + private String accessToken; + + @JsonProperty("refresh_token") + private String refreshToken; + + public int getExpiresIn() { + return expiresIn; + } + + @Nonnull + public String getAccessToken() { + return accessToken; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TokenExchangeResponseDTO)) { + return false; + } + TokenExchangeResponseDTO that = (TokenExchangeResponseDTO) o; + return expiresIn == that.expiresIn + && Objects.equals(accessToken, that.accessToken) + && Objects.equals(refreshToken, that.refreshToken) + && Objects.equals(idToken, that.idToken) + && Objects.equals(tokenType, that.tokenType) + && Objects.equals(scope, that.scope); + } + + @Override + public int hashCode() { + return Objects.hash(accessToken, refreshToken, idToken, tokenType, expiresIn, scope); + } + + @Override + public String toString() { + return "Token{" + + "accessToken='" + + accessToken + + '\'' + + ", refresh_token='" + + refreshToken + + '\'' + + ", idToken='" + + idToken + + '\'' + + ", tokenType='" + + tokenType + + '\'' + + ", expiresIn=" + + expiresIn + + ", scope='" + + scope + + '\'' + + '}'; + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/TokenManager.java b/proxy/src/main/java/com/wavefront/agent/TokenManager.java new file mode 100644 index 000000000..5e864ef81 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/TokenManager.java @@ -0,0 +1,42 @@ +package com.wavefront.agent; + +import com.google.common.collect.Maps; +import com.wavefront.agent.api.APIContainer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.jetbrains.annotations.TestOnly; + +public class TokenManager { + private static final Map multicastingTenantList = Maps.newHashMap(); + private static List scheduledWorkers = new ArrayList(); + private static List externalWorkers = new ArrayList(); + + public static void addTenant(String tenantName, TenantInfo tokenWorker) { + multicastingTenantList.put(tenantName, tokenWorker); + + if (tokenWorker instanceof TokenWorker.Scheduled) { + scheduledWorkers.add((TokenWorker.Scheduled) tokenWorker); + } + + if (tokenWorker instanceof TokenWorker.External) { + externalWorkers.add((TokenWorker.External) tokenWorker); + } + } + + public static void start(APIContainer apiContainer) { + externalWorkers.forEach(external -> external.setAPIContainer(apiContainer)); + scheduledWorkers.forEach(tenantInfo -> tenantInfo.run()); + } + + public static Map getMulticastingTenantList() { + return multicastingTenantList; + } + + @TestOnly + public static void reset() { + externalWorkers.clear(); + scheduledWorkers.clear(); + multicastingTenantList.clear(); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/TokenWorker.java b/proxy/src/main/java/com/wavefront/agent/TokenWorker.java new file mode 100644 index 000000000..84dfe6a24 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/TokenWorker.java @@ -0,0 +1,14 @@ +package com.wavefront.agent; + +import com.wavefront.agent.api.APIContainer; + +public interface TokenWorker { + + interface Scheduled { + void run(); + } + + interface External { + void setAPIContainer(APIContainer apiContainer); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/TokenWorkerCSP.java b/proxy/src/main/java/com/wavefront/agent/TokenWorkerCSP.java new file mode 100644 index 000000000..a6937926d --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/TokenWorkerCSP.java @@ -0,0 +1,206 @@ +package com.wavefront.agent; + +import static com.google.common.base.Preconditions.checkNotNull; +import static javax.ws.rs.core.Response.Status.Family.SUCCESSFUL; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import com.wavefront.agent.api.APIContainer; +import com.wavefront.agent.api.CSPAPI; +import com.wavefront.common.LazySupplier; +import com.wavefront.common.NamedThreadFactory; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Counter; +import com.yammer.metrics.core.MetricName; +import com.yammer.metrics.core.Timer; +import com.yammer.metrics.core.TimerContext; +import java.util.Base64; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nonnull; +import javax.ws.rs.core.Response; + +/** + * If the tenant came from CSP, the tenant's token will be frequently updated by the class, which + * stores tenant-related data. + * + * @author Norayr Chaparyan(nchaparyan@vmware.com). + */ +public class TokenWorkerCSP + implements TokenWorker, TokenWorker.External, TokenWorker.Scheduled, TenantInfo { + private static final String GET_CSP_ACCESS_TOKEN_ERROR_MESSAGE = + "Failed to get access token from CSP (%s). %s"; + private static final ScheduledExecutorService executor = + Executors.newScheduledThreadPool(1, new NamedThreadFactory("csp-token-updater")); + private static final Logger log = Logger.getLogger(TokenWorkerCSP.class.getCanonicalName()); + private static final Supplier errors = + LazySupplier.of( + () -> Metrics.newCounter(new MetricName("csp.token.update", "", "exceptions"))); + private static final Supplier executions = + LazySupplier.of( + () -> Metrics.newCounter(new MetricName("csp.token.update", "", "executions"))); + private static final Supplier successfully = + LazySupplier.of( + () -> Metrics.newCounter(new MetricName("csp.token.update", "", "successfully"))); + private static final Supplier duration = + LazySupplier.of( + () -> + Metrics.newTimer( + new MetricName("csp.token.update", "", "duration"), + TimeUnit.MILLISECONDS, + TimeUnit.MINUTES)); + + private enum ProxyAuthMethod { + API_TOKEN, + CLIENT_CREDENTIALS + } + + private final String wfServer; + private final ProxyAuthMethod proxyAuthMethod; + private String appId; + private String appSecret; + private String orgId; + private String token; + private String bearerToken; + private CSPAPI api; + + public TokenWorkerCSP( + final String appId, final String appSecret, final String orgId, final String wfServer) { + this.appId = checkNotNull(appId); + this.appSecret = checkNotNull(appSecret); + this.orgId = orgId; + this.wfServer = checkNotNull(wfServer); + proxyAuthMethod = ProxyAuthMethod.CLIENT_CREDENTIALS; + } + + public TokenWorkerCSP(final String token, final String wfServer) { + this.token = checkNotNull(token); + this.wfServer = checkNotNull(wfServer); + proxyAuthMethod = ProxyAuthMethod.API_TOKEN; + } + + @Override + public void setAPIContainer(APIContainer apiContainer) { + api = apiContainer.getCSPApi(); + } + + @Nonnull + public String getWFServer() { + return wfServer; + } + + @Override + public String getBearerToken() { + return bearerToken; + } + + @Override + public void run() { + executions.get().inc(); + TimerContext timer = duration.get().time(); + log.info("Updating CSP access token " + "(" + this.proxyAuthMethod + ")."); + long next = 10; // in case of unexpected exception + try { + switch (proxyAuthMethod) { + case CLIENT_CREDENTIALS: + next = processResponse(loadAccessTokenByClientCredentials()); + break; + case API_TOKEN: + next = processResponse(loadAccessTokenByAPIToken()); + break; + } + } catch (Throwable e) { + errors.get().inc(); + log.log( + Level.SEVERE, GET_CSP_ACCESS_TOKEN_ERROR_MESSAGE + "(" + this.proxyAuthMethod + ")", e); + } finally { + timer.stop(); + executor.schedule(this::run, next, TimeUnit.SECONDS); + } + } + + public Response loadAccessTokenByAPIToken() { + return api.getTokenByAPIToken("api_token", token); + } + + /** + * When the CSP access token has expired, obtain it using server to server OAuth app's client id + * and client secret. + */ + public Response loadAccessTokenByClientCredentials() { + String auth = + String.format( + "Basic %s", + Base64.getEncoder() + .encodeToString(String.format("%s:%s", appId, appSecret).getBytes())); + + if (Strings.isNullOrEmpty(orgId)) { + return api.getTokenByClientCredentials(auth, "client_credentials"); + } + + return api.getTokenByClientCredentialsWithOrgId(auth, "client_credentials", orgId); + } + + private long processResponse(final Response response) { + Metrics.newCounter( + new MetricName( + "csp.token.update.response", "", "" + response.getStatusInfo().getStatusCode())) + .inc(); + long nextIn = 10; + if (response.getStatusInfo().getFamily() != SUCCESSFUL) { + errors.get().inc(); + String jsonString = response.readEntity(String.class); + // Parse the JSON response + ObjectMapper objectMapper = new ObjectMapper(); + try { + JsonNode jsonNode = objectMapper.readTree(jsonString); + String message = jsonNode.get("message").asText(); + + log.severe( + String.format(GET_CSP_ACCESS_TOKEN_ERROR_MESSAGE, this.proxyAuthMethod, message) + + ". Status: " + + response.getStatusInfo().getStatusCode()); + + } catch (Exception e) { + log.severe( + String.format(GET_CSP_ACCESS_TOKEN_ERROR_MESSAGE, this.proxyAuthMethod, jsonString) + + ". Status: " + + response.getStatusInfo().getStatusCode()); + } + } else { + try { + final TokenExchangeResponseDTO tokenExchangeResponseDTO = + response.readEntity(TokenExchangeResponseDTO.class); + this.bearerToken = tokenExchangeResponseDTO.getAccessToken(); + nextIn = getTimeOffset(tokenExchangeResponseDTO.getExpiresIn()); + successfully.get().inc(); + } catch (Throwable e) { + errors.get().inc(); + log.log( + Level.SEVERE, GET_CSP_ACCESS_TOKEN_ERROR_MESSAGE + "(" + this.proxyAuthMethod + ")", e); + } + } + return nextIn; + } + + /** + * Calculates the time offset for scheduling regular requests to a CSP based on the expiration + * time of a CSP access token. If the access token expiration time is less than 10 minutes, + * schedule requests 30 seconds before it expires. If the access token expiration time is 10 + * minutes or more, schedule requests 3 minutes before it expires. + * + * @param expiresIn the expiration time of the CSP access token in seconds. + * @return the calculated time offset. + */ + private int getTimeOffset(int expiresIn) { + if (expiresIn < 600) { + return expiresIn - 30; + } + return expiresIn - 180; + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/TokenWorkerWF.java b/proxy/src/main/java/com/wavefront/agent/TokenWorkerWF.java new file mode 100644 index 000000000..a65cce9df --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/TokenWorkerWF.java @@ -0,0 +1,23 @@ +package com.wavefront.agent; + +import javax.annotation.Nonnull; + +public class TokenWorkerWF implements TokenWorker, TenantInfo { + private String token; + private String server; + + public TokenWorkerWF(@Nonnull final String token, @Nonnull final String server) { + this.token = token; + this.server = server; + } + + @Override + public String getWFServer() { + return server; + } + + @Override + public String getBearerToken() { + return token; + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/Utils.java b/proxy/src/main/java/com/wavefront/agent/Utils.java deleted file mode 100644 index cc257ff36..000000000 --- a/proxy/src/main/java/com/wavefront/agent/Utils.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.wavefront.agent; - -import org.apache.commons.lang.StringUtils; - -import java.util.function.Supplier; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.annotation.Nullable; - -/** - * A placeholder class for miscellaneous utility methods. - * - * @author vasily@wavefront.com - */ -public abstract class Utils { - - private static final Pattern patternUuid = Pattern.compile( - "(\\w{8})(\\w{4})(\\w{4})(\\w{4})(\\w{12})"); - /** - * A lazy initialization wrapper for {@code Supplier} - * - * @param supplier {@code Supplier} to lazy-initialize - * @return lazy wrapped supplier - */ - public static Supplier lazySupplier(Supplier supplier) { - return new Supplier() { - private volatile T value = null; - @Override - public T get() { - if (value == null) { - synchronized (this) { - if (value == null) { - value = supplier.get(); - } - } - } - return value; - } - }; - } - - /** - * Requires an input uuid string Encoded as 32 hex characters. - * For example {@code cced093a76eea418ffdc9bb9a6453df3} - * - * @param uuid string encoded as 32 hex characters. - * @return uuid string encoded in 8-4-4-4-12 (rfc4122) format. - */ - public static String addHyphensToUuid(String uuid) { - Matcher matcherUuid = patternUuid.matcher(uuid); - return matcherUuid.replaceAll("$1-$2-$3-$4-$5"); - } - - /** - * Method converts a string Id to {@code UUID}. - * This Method specifically converts id's with less than 32 digit hex characters into UUID - * format (See RFC 4122: A - * Universally Unique IDentifier (UUID) URN Namespace) by left padding id with Zeroes - * and adding hyphens. It assumes that if the input id contains hyphens it is already an UUID. - * Please don't use this method to validate/guarantee your id as an UUID. - * - * @param id a string encoded in hex characters. - * @return a UUID string. - */ - @Nullable - public static String convertToUuidString(@Nullable String id) { - if (id == null || id.contains("-")) { - return id; - } - return addHyphensToUuid(StringUtils.leftPad(id, 32, '0')); - } -} diff --git a/proxy/src/main/java/com/wavefront/agent/WavefrontProxyService.java b/proxy/src/main/java/com/wavefront/agent/WavefrontProxyService.java new file mode 100644 index 000000000..497fa4286 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/WavefrontProxyService.java @@ -0,0 +1,30 @@ +package com.wavefront.agent; + +import org.apache.commons.daemon.Daemon; +import org.apache.commons.daemon.DaemonContext; + +/** @author Mori Bellamy (mori@wavefront.com) */ +public class WavefrontProxyService implements Daemon { + + private PushAgent agent; + private DaemonContext daemonContext; + + @Override + public void init(DaemonContext daemonContext) { + this.daemonContext = daemonContext; + agent = new PushAgent(); + } + + @Override + public void start() { + agent.start(daemonContext.getArguments()); + } + + @Override + public void stop() { + agent.shutdown(); + } + + @Override + public void destroy() {} +} diff --git a/proxy/src/main/java/com/wavefront/agent/api/APIContainer.java b/proxy/src/main/java/com/wavefront/agent/api/APIContainer.java new file mode 100644 index 000000000..bf7fc4e07 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/api/APIContainer.java @@ -0,0 +1,393 @@ +package com.wavefront.agent.api; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import com.wavefront.agent.*; +import com.wavefront.agent.channel.DisableGZIPEncodingInterceptor; +import com.wavefront.agent.channel.GZIPEncodingInterceptorWithVariableCompression; +import com.wavefront.api.EventAPI; +import com.wavefront.api.LogAPI; +import com.wavefront.api.ProxyV2API; +import com.wavefront.api.SourceTagAPI; +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.ws.rs.client.ClientRequestFilter; +import javax.ws.rs.ext.WriterInterceptor; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpRequest; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; +import org.apache.http.impl.client.HttpClientBuilder; +import org.jboss.resteasy.client.jaxrs.ClientHttpEngine; +import org.jboss.resteasy.client.jaxrs.ResteasyClient; +import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; +import org.jboss.resteasy.client.jaxrs.engines.ApacheHttpClient43Engine; +import org.jboss.resteasy.client.jaxrs.internal.LocalResteasyProviderFactory; +import org.jboss.resteasy.client.jaxrs.internal.ResteasyClientBuilderImpl; +import org.jboss.resteasy.plugins.interceptors.AcceptEncodingGZIPFilter; +import org.jboss.resteasy.plugins.interceptors.GZIPDecodingInterceptor; +import org.jboss.resteasy.plugins.providers.jackson.ResteasyJackson2Provider; +import org.jboss.resteasy.spi.ResteasyProviderFactory; + +/** + * Container for all Wavefront back-end API objects (proxy, source tag, event) + * + * @author vasily@wavefront.com + */ +public class APIContainer { + public static final String CENTRAL_TENANT_NAME = "central"; + public static final String LE_MANS_INGESTION_PATH = + "le-mans/v1/streams/ingestion-pipeline-stream"; + + private final ProxyConfig proxyConfig; + private final ResteasyProviderFactory resteasyProviderFactory; + private final ClientHttpEngine clientHttpEngine; + private final boolean discardData; + private final CSPAPI cspAPI; + + private Map proxyV2APIsForMulticasting; + private Map sourceTagAPIsForMulticasting; + private Map eventAPIsForMulticasting; + + private LogAPI logAPI; + + private String logServerToken; + private String logServerEndpointUrl; + + private static final Logger logger = Logger.getLogger(APIContainer.class.getCanonicalName()); + + /** + * @param proxyConfig proxy configuration settings + * @param discardData run proxy in test mode (don't actually send the data) + */ + public APIContainer(ProxyConfig proxyConfig, boolean discardData) { + this.proxyConfig = proxyConfig; + this.logServerToken = "NOT_SET"; + this.logServerEndpointUrl = "NOT_SET"; + this.resteasyProviderFactory = createProviderFactory(); + this.clientHttpEngine = createHttpEngine(); + this.discardData = discardData; + this.logAPI = createService(logServerEndpointUrl, LogAPI.class); + this.cspAPI = createService(proxyConfig.getCSPBaseUrl(), CSPAPI.class); + + // config the multicasting tenants / clusters + proxyV2APIsForMulticasting = Maps.newHashMap(); + sourceTagAPIsForMulticasting = Maps.newHashMap(); + eventAPIsForMulticasting = Maps.newHashMap(); + // tenantInfo: { : {"token": , "server": }} + String tenantName; + String tenantServer; + for (Map.Entry tenantInfoEntry : + TokenManager.getMulticastingTenantList().entrySet()) { + tenantName = tenantInfoEntry.getKey(); + tenantServer = tenantInfoEntry.getValue().getWFServer(); + proxyV2APIsForMulticasting.put(tenantName, createService(tenantServer, ProxyV2API.class)); + sourceTagAPIsForMulticasting.put(tenantName, createService(tenantServer, SourceTagAPI.class)); + eventAPIsForMulticasting.put(tenantName, createService(tenantServer, EventAPI.class)); + } + + if (discardData) { + ProxyV2API proxyV2API = this.proxyV2APIsForMulticasting.get(CENTRAL_TENANT_NAME); + this.proxyV2APIsForMulticasting = Maps.newHashMap(); + this.proxyV2APIsForMulticasting.put(CENTRAL_TENANT_NAME, new NoopProxyV2API(proxyV2API)); + this.sourceTagAPIsForMulticasting = Maps.newHashMap(); + this.sourceTagAPIsForMulticasting.put(CENTRAL_TENANT_NAME, new NoopSourceTagAPI()); + this.eventAPIsForMulticasting = Maps.newHashMap(); + this.eventAPIsForMulticasting.put(CENTRAL_TENANT_NAME, new NoopEventAPI()); + this.logAPI = new NoopLogAPI(); + } + configureHttpProxy(); + } + + /** + * This is for testing only, as it loses ability to invoke updateServerEndpointURL(). + * + * @param proxyV2API RESTeasy proxy for ProxyV2API + * @param sourceTagAPI RESTeasy proxy for SourceTagAPI + * @param eventAPI RESTeasy proxy for EventAPI + * @param logAPI RESTeasy proxy for LogAPI + */ + @VisibleForTesting + public APIContainer( + ProxyV2API proxyV2API, + SourceTagAPI sourceTagAPI, + EventAPI eventAPI, + LogAPI logAPI, + CSPAPI cspAPI) { + this.proxyConfig = null; + this.resteasyProviderFactory = null; + this.clientHttpEngine = null; + this.discardData = false; + this.logAPI = logAPI; + this.cspAPI = cspAPI; + proxyV2APIsForMulticasting = Maps.newHashMap(); + proxyV2APIsForMulticasting.put(CENTRAL_TENANT_NAME, proxyV2API); + sourceTagAPIsForMulticasting = Maps.newHashMap(); + sourceTagAPIsForMulticasting.put(CENTRAL_TENANT_NAME, sourceTagAPI); + eventAPIsForMulticasting = Maps.newHashMap(); + eventAPIsForMulticasting.put(CENTRAL_TENANT_NAME, eventAPI); + } + + /** + * Provide the collection of loaded tenant name list + * + * @return tenant name collection + */ + public Collection getTenantNameList() { + return proxyV2APIsForMulticasting.keySet(); + } + + /** + * Get RESTeasy proxy for {@link ProxyV2API} with given tenant name. + * + * @param tenantName tenant name + * @return proxy object corresponding to tenant + */ + public ProxyV2API getProxyV2APIForTenant(String tenantName) { + return proxyV2APIsForMulticasting.get(tenantName); + } + + /** + * Get RESTeasy proxy for {@link SourceTagAPI} with given tenant name. + * + * @param tenantName tenant name + * @return proxy object corresponding to the tenant name + */ + public SourceTagAPI getSourceTagAPIForTenant(String tenantName) { + return sourceTagAPIsForMulticasting.get(tenantName); + } + + /** + * Get RESTeasy proxy for {@link EventAPI} with given tenant name. + * + * @param tenantName tenant name + * @return proxy object corresponding to the tenant name + */ + public EventAPI getEventAPIForTenant(String tenantName) { + return eventAPIsForMulticasting.get(tenantName); + } + + /** + * Get RESTeasy proxy for {@link LogAPI}. + * + * @return proxy object + */ + public LogAPI getLogAPI() { + return logAPI; + } + + public String getLogServerToken() { + return logServerToken; + } + + public String getLogServerEndpointUrl() { + return logServerEndpointUrl; + } + + /** + * Re-create RESTeasy proxies with new server endpoint URL (allows changing URL at runtime). + * + * @param logServerEndpointUrl new log server endpoint URL. + * @param logServerToken new server token. + */ + public void updateLogServerEndpointURLandToken( + String logServerEndpointUrl, String logServerToken) { + // if either are null or empty, just return + if (StringUtils.isBlank(logServerEndpointUrl) || StringUtils.isBlank(logServerToken)) { + return; + } + // Only recreate if either the url or token have changed + if (!StringUtils.equals(logServerEndpointUrl, this.logServerEndpointUrl) + || !StringUtils.equals(logServerToken, this.logServerToken)) { + this.logServerEndpointUrl = removePathFromURL(logServerEndpointUrl); + this.logServerToken = logServerToken; + this.logAPI = createService(this.logServerEndpointUrl, LogAPI.class, createProviderFactory()); + if (discardData) { + this.logAPI = new NoopLogAPI(); + } + } + } + + private String removePathFromURL(String completeURL) { + if (completeURL != null && completeURL.contains(LE_MANS_INGESTION_PATH)) { + return completeURL.replace(LE_MANS_INGESTION_PATH, ""); + } + return completeURL; + } + + /** + * Re-create RESTeasy proxies with new server endpoint URL (allows changing URL at runtime). + * + * @param tenantName the tenant to be updated + * @param serverEndpointUrl new server endpoint URL. + */ + public void updateServerEndpointURL(String tenantName, String serverEndpointUrl) { + if (proxyConfig == null) { + throw new IllegalStateException("Can't invoke updateServerEndpointURL with this constructor"); + } + proxyV2APIsForMulticasting.put(tenantName, createService(serverEndpointUrl, ProxyV2API.class)); + sourceTagAPIsForMulticasting.put( + tenantName, createService(serverEndpointUrl, SourceTagAPI.class)); + eventAPIsForMulticasting.put(tenantName, createService(serverEndpointUrl, EventAPI.class)); + + if (discardData) { + ProxyV2API proxyV2API = this.proxyV2APIsForMulticasting.get(CENTRAL_TENANT_NAME); + this.proxyV2APIsForMulticasting = Maps.newHashMap(); + this.proxyV2APIsForMulticasting.put(CENTRAL_TENANT_NAME, new NoopProxyV2API(proxyV2API)); + this.sourceTagAPIsForMulticasting = Maps.newHashMap(); + this.sourceTagAPIsForMulticasting.put(CENTRAL_TENANT_NAME, new NoopSourceTagAPI()); + this.eventAPIsForMulticasting = Maps.newHashMap(); + this.eventAPIsForMulticasting.put(CENTRAL_TENANT_NAME, new NoopEventAPI()); + } + } + + private void configureHttpProxy() { + if (proxyConfig.getProxyHost() != null) { + System.setProperty("http.proxyHost", proxyConfig.getProxyHost()); + System.setProperty("https.proxyHost", proxyConfig.getProxyHost()); + System.setProperty("http.proxyPort", String.valueOf(proxyConfig.getProxyPort())); + System.setProperty("https.proxyPort", String.valueOf(proxyConfig.getProxyPort())); + } + if (proxyConfig.getProxyUser() != null && proxyConfig.getProxyPassword() != null) { + Authenticator.setDefault( + new Authenticator() { + @Override + public PasswordAuthentication getPasswordAuthentication() { + if (getRequestorType() == RequestorType.PROXY) { + return new PasswordAuthentication( + proxyConfig.getProxyUser(), proxyConfig.getProxyPassword().toCharArray()); + } else { + return null; + } + } + }); + } + } + + private ResteasyProviderFactory createProviderFactory() { + ResteasyProviderFactory factory = + new LocalResteasyProviderFactory(ResteasyProviderFactory.getInstance()); + factory.registerProvider(JsonNodeWriter.class); + if (!factory.getClasses().contains(ResteasyJackson2Provider.class)) { + factory.registerProvider(ResteasyJackson2Provider.class); + } + factory.register(GZIPDecodingInterceptor.class); + if (proxyConfig.isGzipCompression()) { + WriterInterceptor interceptor = + new GZIPEncodingInterceptorWithVariableCompression(proxyConfig.getGzipCompressionLevel()); + factory.register(interceptor); + } else { + factory.register(DisableGZIPEncodingInterceptor.class); + } + factory.register(AcceptEncodingGZIPFilter.class); + // add authorization header for all proxy endpoints, except for /checkin - since it's also + // passed as a parameter, it's creating duplicate headers that cause the entire request to + // be + // rejected by nginx. unfortunately, RESTeasy is not smart enough to handle that + // automatically. + factory.register( + (ClientRequestFilter) + context -> { + if (context.getUri().getPath().startsWith("/csp")) { + return; + } + if (proxyConfig.isGzipCompression()) { + context.getHeaders().add("Content-Encoding", "gzip"); + } + if ((context.getUri().getPath().contains("/v2/wfproxy") + || context.getUri().getPath().contains("/v2/source") + || context.getUri().getPath().contains("/event")) + && !context.getUri().getPath().endsWith("checkin")) { + context + .getHeaders() + .add( + "Authorization", + "Bearer " + + TokenManager.getMulticastingTenantList() + .get(APIContainer.CENTRAL_TENANT_NAME) + .getBearerToken()); + } else if (context.getUri().getPath().contains("/le-mans")) { + context.getHeaders().add("Authorization", "Bearer " + logServerToken); + } + }); + return factory; + } + + private ClientHttpEngine createHttpEngine() { + HttpClient httpClient = + HttpClientBuilder.create() + .useSystemProperties() + .setUserAgent(proxyConfig.getHttpUserAgent()) + .setMaxConnTotal(proxyConfig.getHttpMaxConnTotal()) + .setMaxConnPerRoute(proxyConfig.getHttpMaxConnPerRoute()) + .setConnectionTimeToLive(1, TimeUnit.MINUTES) + .setDefaultSocketConfig( + SocketConfig.custom().setSoTimeout(proxyConfig.getHttpRequestTimeout()).build()) + .setSSLSocketFactory( + new SSLConnectionSocketFactoryImpl( + SSLConnectionSocketFactory.getSystemSocketFactory(), + proxyConfig.getHttpRequestTimeout())) + .setRetryHandler( + new DefaultHttpRequestRetryHandler(proxyConfig.getHttpAutoRetries(), true) { + @Override + protected boolean handleAsIdempotent(HttpRequest request) { + // by default, retry all http calls (submissions are + // idempotent). + return true; + } + }) + .setDefaultRequestConfig( + RequestConfig.custom() + .setContentCompressionEnabled(true) + .setRedirectsEnabled(true) + .setConnectTimeout(proxyConfig.getHttpConnectTimeout()) + .setConnectionRequestTimeout(proxyConfig.getHttpConnectTimeout()) + .setSocketTimeout(proxyConfig.getHttpRequestTimeout()) + .build()) + .build(); + final ApacheHttpClient43Engine httpEngine = new ApacheHttpClient43Engine(httpClient, true); + // avoid using disk at all + httpEngine.setFileUploadInMemoryThresholdLimit(100); + httpEngine.setFileUploadMemoryUnit(ApacheHttpClient43Engine.MemoryUnit.MB); + return httpEngine; + } + + /** Create RESTeasy proxies for remote calls via HTTP. */ + private T createService(String serverEndpointUrl, Class apiClass) { + return createServiceInternal(serverEndpointUrl, apiClass, resteasyProviderFactory); + } + + /** Create RESTeasy proxies for remote calls via HTTP. */ + private T createService( + String serverEndpointUrl, + Class apiClass, + ResteasyProviderFactory resteasyProviderFactory) { + return createServiceInternal(serverEndpointUrl, apiClass, resteasyProviderFactory); + } + + /** Create RESTeasy proxies for remote calls via HTTP. */ + private T createServiceInternal( + String serverEndpointUrl, + Class apiClass, + ResteasyProviderFactory resteasyProviderFactory) { + ResteasyClient client = + new ResteasyClientBuilderImpl() + .httpEngine(clientHttpEngine) + .providerFactory(resteasyProviderFactory) + .build(); + ResteasyWebTarget target = client.target(serverEndpointUrl); + return target.proxy(apiClass); + } + + public CSPAPI getCSPApi() { + return cspAPI; + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/api/CSPAPI.java b/proxy/src/main/java/com/wavefront/agent/api/CSPAPI.java new file mode 100644 index 000000000..6e944f34b --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/api/CSPAPI.java @@ -0,0 +1,30 @@ +package com.wavefront.agent.api; + +import com.google.common.net.HttpHeaders; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +public interface CSPAPI { + @POST + @Consumes(MediaType.APPLICATION_FORM_URLENCODED) + @Path("/csp/gateway/am/api/auth/api-tokens/authorize") + Response getTokenByAPIToken( + @FormParam("grant_type") final String grantType, + @FormParam("api_token") final String apiToken); + + @POST + @Consumes(MediaType.APPLICATION_FORM_URLENCODED) + @Path("csp/gateway/am/api/auth/authorize") + Response getTokenByClientCredentialsWithOrgId( + @HeaderParam(HttpHeaders.AUTHORIZATION) final String auth, + @FormParam("grant_type") final String grantType, + @FormParam("orgId") final String orgId); + + @POST + @Consumes(MediaType.APPLICATION_FORM_URLENCODED) + @Path("csp/gateway/am/api/auth/authorize") + Response getTokenByClientCredentials( + @HeaderParam(HttpHeaders.AUTHORIZATION) final String auth, + @FormParam("grant_type") final String grantType); +} diff --git a/proxy/src/main/java/com/wavefront/agent/api/ForceQueueEnabledAgentAPI.java b/proxy/src/main/java/com/wavefront/agent/api/ForceQueueEnabledAgentAPI.java deleted file mode 100644 index ce6dcdd70..000000000 --- a/proxy/src/main/java/com/wavefront/agent/api/ForceQueueEnabledAgentAPI.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.wavefront.agent.api; - -import com.wavefront.api.WavefrontAPI; -import com.wavefront.api.agent.ShellOutputDTO; - -import java.util.List; -import java.util.UUID; - -import javax.ws.rs.core.Response; - -/** - * @author Andrew Kao (andrew@wavefront.com) - */ -public interface ForceQueueEnabledAgentAPI extends WavefrontAPI { - - Response postWorkUnitResult(UUID agentId, - UUID workUnitId, - UUID targetId, - ShellOutputDTO shellOutputDTO, - boolean forceToQueue); - - Response postPushData(UUID agentId, - UUID workUnitId, - Long currentMillis, - String format, - String pushData, - boolean forceToQueue); - - Response removeTag(String id, String tagValue, boolean forceToQueue); - - Response setTags(String id, List tagsValuesToSet, boolean forceToQueue); - - Response removeDescription(String id, boolean forceToQueue); - - Response setDescription(String id, String desc, boolean forceToQueue); -} diff --git a/proxy/src/main/java/com/wavefront/agent/api/NoopEventAPI.java b/proxy/src/main/java/com/wavefront/agent/api/NoopEventAPI.java new file mode 100644 index 000000000..cd1a6497f --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/api/NoopEventAPI.java @@ -0,0 +1,19 @@ +package com.wavefront.agent.api; + +import com.wavefront.api.EventAPI; +import com.wavefront.dto.Event; +import java.util.List; +import java.util.UUID; +import javax.ws.rs.core.Response; + +/** + * A no-op SourceTagAPI stub. + * + * @author vasily@wavefront.com + */ +public class NoopEventAPI implements EventAPI { + @Override + public Response proxyEvents(UUID uuid, List list) { + return Response.ok().build(); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/api/NoopLogAPI.java b/proxy/src/main/java/com/wavefront/agent/api/NoopLogAPI.java new file mode 100644 index 000000000..8f4a3fa6b --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/api/NoopLogAPI.java @@ -0,0 +1,18 @@ +package com.wavefront.agent.api; + +import com.wavefront.api.LogAPI; +import com.wavefront.dto.Log; +import java.util.List; +import javax.ws.rs.core.Response; + +/** + * A no-op LogAPI stub. + * + * @author amitw@vmware.com + */ +public class NoopLogAPI implements LogAPI { + @Override + public Response proxyLogs(String agentProxyId, List logs) { + return Response.ok().build(); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/api/NoopProxyV2API.java b/proxy/src/main/java/com/wavefront/agent/api/NoopProxyV2API.java new file mode 100644 index 000000000..946f1f9dc --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/api/NoopProxyV2API.java @@ -0,0 +1,61 @@ +package com.wavefront.agent.api; + +import com.fasterxml.jackson.databind.JsonNode; +import com.wavefront.api.ProxyV2API; +import com.wavefront.api.agent.AgentConfiguration; +import java.util.UUID; +import javax.ws.rs.core.Response; + +/** + * Partial ProxyV2API wrapper stub that passed proxyCheckin/proxyConfigProcessed calls to the + * delegate and replaces proxyReport/proxyError with a no-op. + * + * @author vasily@wavefront.com + */ +public class NoopProxyV2API implements ProxyV2API { + private final ProxyV2API wrapped; + + public NoopProxyV2API(ProxyV2API wrapped) { + this.wrapped = wrapped; + } + + @Override + public AgentConfiguration proxyCheckin( + UUID proxyId, + String authorization, + String hostname, + String proxyname, + String version, + Long currentMillis, + JsonNode agentMetrics, + Boolean ephemeral) { + return wrapped.proxyCheckin( + proxyId, + authorization, + hostname, + proxyname, + version, + currentMillis, + agentMetrics, + ephemeral); + } + + @Override + public void proxySaveConfig(UUID uuid, JsonNode jsonNode) {} + + @Override + public void proxySavePreprocessorRules(UUID uuid, JsonNode jsonNode) {} + + @Override + public Response proxyReport(UUID uuid, String s, String s1) { + return Response.ok().build(); + } + + @Override + public void proxyConfigProcessed(UUID uuid) { + wrapped.proxyConfigProcessed(uuid); + } + + @Override + public void proxyError(UUID uuid, String s) {} +} diff --git a/proxy/src/main/java/com/wavefront/agent/api/NoopSourceTagAPI.java b/proxy/src/main/java/com/wavefront/agent/api/NoopSourceTagAPI.java new file mode 100644 index 000000000..0785d337e --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/api/NoopSourceTagAPI.java @@ -0,0 +1,38 @@ +package com.wavefront.agent.api; + +import com.wavefront.api.SourceTagAPI; +import java.util.List; +import javax.ws.rs.core.Response; + +/** + * A no-op SourceTagAPI stub. + * + * @author vasily@wavefront.com + */ +public class NoopSourceTagAPI implements SourceTagAPI { + + @Override + public Response appendTag(String id, String tagValue) { + return Response.ok().build(); + } + + @Override + public Response removeTag(String id, String tagValue) { + return Response.ok().build(); + } + + @Override + public Response setTags(String id, List tagValuesToSet) { + return Response.ok().build(); + } + + @Override + public Response setDescription(String id, String description) { + return Response.ok().build(); + } + + @Override + public Response removeDescription(String id) { + return Response.ok().build(); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/auth/DummyAuthenticator.java b/proxy/src/main/java/com/wavefront/agent/auth/DummyAuthenticator.java index e1d32d756..4b6e347b7 100644 --- a/proxy/src/main/java/com/wavefront/agent/auth/DummyAuthenticator.java +++ b/proxy/src/main/java/com/wavefront/agent/auth/DummyAuthenticator.java @@ -7,8 +7,7 @@ */ class DummyAuthenticator implements TokenAuthenticator { - DummyAuthenticator() { - } + DummyAuthenticator() {} @Override public boolean authorize(String token) { diff --git a/proxy/src/main/java/com/wavefront/agent/auth/HttpGetTokenIntrospectionAuthenticator.java b/proxy/src/main/java/com/wavefront/agent/auth/HttpGetTokenIntrospectionAuthenticator.java index ba77cc8c9..379b28431 100644 --- a/proxy/src/main/java/com/wavefront/agent/auth/HttpGetTokenIntrospectionAuthenticator.java +++ b/proxy/src/main/java/com/wavefront/agent/auth/HttpGetTokenIntrospectionAuthenticator.java @@ -2,57 +2,64 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; - import com.yammer.metrics.Metrics; import com.yammer.metrics.core.Counter; import com.yammer.metrics.core.MetricName; - +import java.util.function.Supplier; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.util.EntityUtils; -import java.util.function.Supplier; -import java.util.logging.Logger; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - /** - * {@link TokenIntrospectionAuthenticator} that considers any 2xx response to be valid. Token to validate is passed in - * the url, {{token}} placeholder is substituted before the call. + * {@link TokenIntrospectionAuthenticator} that considers any 2xx response to be valid. Token to + * validate is passed in the url, {{token}} placeholder is substituted before the call. * * @author vasily@wavefront.com */ class HttpGetTokenIntrospectionAuthenticator extends TokenIntrospectionAuthenticator { - private static final Logger logger = Logger.getLogger(HttpGetTokenIntrospectionAuthenticator.class.getCanonicalName()); - private final HttpClient httpClient; private final String tokenIntrospectionServiceUrl; private final String tokenIntrospectionServiceAuthorizationHeader; - private final Counter accessGrantedResponses = Metrics.newCounter(new MetricName("auth", "", "access-granted")); - private final Counter accessDeniedResponses = Metrics.newCounter(new MetricName("auth", "", "access-denied")); - - HttpGetTokenIntrospectionAuthenticator(@Nonnull HttpClient httpClient, @Nonnull String tokenIntrospectionServiceUrl, - @Nullable String tokenIntrospectionServiceAuthorizationHeader, - int authResponseRefreshInterval, int authResponseMaxTtl) { - this(httpClient, tokenIntrospectionServiceUrl, tokenIntrospectionServiceAuthorizationHeader, - authResponseRefreshInterval, authResponseMaxTtl, System::currentTimeMillis); + private final Counter accessGrantedResponses = + Metrics.newCounter(new MetricName("auth", "", "access-granted")); + private final Counter accessDeniedResponses = + Metrics.newCounter(new MetricName("auth", "", "access-denied")); + HttpGetTokenIntrospectionAuthenticator( + @Nonnull HttpClient httpClient, + @Nonnull String tokenIntrospectionServiceUrl, + @Nullable String tokenIntrospectionServiceAuthorizationHeader, + int authResponseRefreshInterval, + int authResponseMaxTtl) { + this( + httpClient, + tokenIntrospectionServiceUrl, + tokenIntrospectionServiceAuthorizationHeader, + authResponseRefreshInterval, + authResponseMaxTtl, + System::currentTimeMillis); } @VisibleForTesting - HttpGetTokenIntrospectionAuthenticator(@Nonnull HttpClient httpClient, @Nonnull String tokenIntrospectionServiceUrl, - @Nullable String tokenIntrospectionServiceAuthorizationHeader, - int authResponseRefreshInterval, int authResponseMaxTtl, - @Nonnull Supplier timeSupplier) { + HttpGetTokenIntrospectionAuthenticator( + @Nonnull HttpClient httpClient, + @Nonnull String tokenIntrospectionServiceUrl, + @Nullable String tokenIntrospectionServiceAuthorizationHeader, + int authResponseRefreshInterval, + int authResponseMaxTtl, + @Nonnull Supplier timeSupplier) { super(authResponseRefreshInterval, authResponseMaxTtl, timeSupplier); Preconditions.checkNotNull(httpClient, "httpClient must be set"); - Preconditions.checkNotNull(tokenIntrospectionServiceUrl, "tokenIntrospectionServiceUrl parameter must be set"); + Preconditions.checkNotNull( + tokenIntrospectionServiceUrl, "tokenIntrospectionServiceUrl parameter must be set"); this.httpClient = httpClient; this.tokenIntrospectionServiceUrl = tokenIntrospectionServiceUrl; - this.tokenIntrospectionServiceAuthorizationHeader = tokenIntrospectionServiceAuthorizationHeader; + this.tokenIntrospectionServiceAuthorizationHeader = + tokenIntrospectionServiceAuthorizationHeader; } @Override diff --git a/proxy/src/main/java/com/wavefront/agent/auth/Oauth2TokenIntrospectionAuthenticator.java b/proxy/src/main/java/com/wavefront/agent/auth/Oauth2TokenIntrospectionAuthenticator.java index 212b04107..eb28c0ca7 100644 --- a/proxy/src/main/java/com/wavefront/agent/auth/Oauth2TokenIntrospectionAuthenticator.java +++ b/proxy/src/main/java/com/wavefront/agent/auth/Oauth2TokenIntrospectionAuthenticator.java @@ -1,69 +1,76 @@ package com.wavefront.agent.auth; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yammer.metrics.Metrics; import com.yammer.metrics.core.Counter; import com.yammer.metrics.core.MetricName; - +import java.util.function.Supplier; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpPost; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; -import org.jetbrains.annotations.NotNull; - -import java.util.function.Supplier; -import java.util.logging.Logger; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; /** * {@link TokenIntrospectionAuthenticator} that validates tokens against an OAuth 2.0-compliant - * Token Introspection endpoint, as described in RFC 7662. + * Token Introspection endpoint, as described in RFC + * 7662. * * @author vasily@wavefront.com */ class Oauth2TokenIntrospectionAuthenticator extends TokenIntrospectionAuthenticator { - private static final Logger logger = Logger.getLogger(Oauth2TokenIntrospectionAuthenticator.class.getCanonicalName()); - private final HttpClient httpClient; private final String tokenIntrospectionServiceUrl; private final String tokenIntrospectionAuthorizationHeader; - private final Counter accessGrantedResponses = Metrics.newCounter(new MetricName("auth", "", "access-granted")); - private final Counter accessDeniedResponses = Metrics.newCounter(new MetricName("auth", "", "access-denied")); + private final Counter accessGrantedResponses = + Metrics.newCounter(new MetricName("auth", "", "access-granted")); + private final Counter accessDeniedResponses = + Metrics.newCounter(new MetricName("auth", "", "access-denied")); private static final ObjectMapper JSON_PARSER = new ObjectMapper(); - Oauth2TokenIntrospectionAuthenticator(@Nonnull HttpClient httpClient, @Nonnull String tokenIntrospectionServiceUrl, - @Nullable String tokenIntrospectionAuthorizationHeader, - int authResponseRefreshInterval, int authResponseMaxTtl) { - this(httpClient, tokenIntrospectionServiceUrl, tokenIntrospectionAuthorizationHeader, authResponseRefreshInterval, - authResponseMaxTtl, System::currentTimeMillis); + Oauth2TokenIntrospectionAuthenticator( + @Nonnull HttpClient httpClient, + @Nonnull String tokenIntrospectionServiceUrl, + @Nullable String tokenIntrospectionAuthorizationHeader, + int authResponseRefreshInterval, + int authResponseMaxTtl) { + this( + httpClient, + tokenIntrospectionServiceUrl, + tokenIntrospectionAuthorizationHeader, + authResponseRefreshInterval, + authResponseMaxTtl, + System::currentTimeMillis); } @VisibleForTesting - Oauth2TokenIntrospectionAuthenticator(@Nonnull HttpClient httpClient, @Nonnull String tokenIntrospectionServiceUrl, - @Nullable String tokenIntrospectionAuthorizationHeader, - int authResponseRefreshInterval, int authResponseMaxTtl, - @Nonnull Supplier timeSupplier) { + Oauth2TokenIntrospectionAuthenticator( + @Nonnull HttpClient httpClient, + @Nonnull String tokenIntrospectionServiceUrl, + @Nullable String tokenIntrospectionAuthorizationHeader, + int authResponseRefreshInterval, + int authResponseMaxTtl, + @Nonnull Supplier timeSupplier) { super(authResponseRefreshInterval, authResponseMaxTtl, timeSupplier); Preconditions.checkNotNull(httpClient, "httpClient must be set"); - Preconditions.checkNotNull(tokenIntrospectionServiceUrl, "tokenIntrospectionServiceUrl parameter must be set"); + Preconditions.checkNotNull( + tokenIntrospectionServiceUrl, "tokenIntrospectionServiceUrl parameter must be set"); this.httpClient = httpClient; this.tokenIntrospectionServiceUrl = tokenIntrospectionServiceUrl; this.tokenIntrospectionAuthorizationHeader = tokenIntrospectionAuthorizationHeader; } @Override - boolean callAuthService(@NotNull String token) throws Exception { + boolean callAuthService(@Nonnull String token) throws Exception { boolean result; HttpPost request = new HttpPost(tokenIntrospectionServiceUrl.replace("{{token}}", token)); request.setHeader("Content-Type", "application/x-www-form-urlencoded"); @@ -71,7 +78,8 @@ boolean callAuthService(@NotNull String token) throws Exception { if (tokenIntrospectionAuthorizationHeader != null) { request.setHeader("Authorization", tokenIntrospectionAuthorizationHeader); } - request.setEntity(new UrlEncodedFormEntity(ImmutableList.of(new BasicNameValuePair("token", token)))); + request.setEntity( + new UrlEncodedFormEntity(ImmutableList.of(new BasicNameValuePair("token", token)))); HttpResponse response = httpClient.execute(request); JsonNode node = JSON_PARSER.readTree(EntityUtils.toString(response.getEntity())); if (node.hasNonNull("active") && node.get("active").isBoolean()) { diff --git a/proxy/src/main/java/com/wavefront/agent/auth/StaticTokenAuthenticator.java b/proxy/src/main/java/com/wavefront/agent/auth/StaticTokenAuthenticator.java index d1fc0babf..dbb00256b 100644 --- a/proxy/src/main/java/com/wavefront/agent/auth/StaticTokenAuthenticator.java +++ b/proxy/src/main/java/com/wavefront/agent/auth/StaticTokenAuthenticator.java @@ -1,7 +1,6 @@ package com.wavefront.agent.auth; import com.google.common.base.Preconditions; - import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/proxy/src/main/java/com/wavefront/agent/auth/TokenAuthenticator.java b/proxy/src/main/java/com/wavefront/agent/auth/TokenAuthenticator.java index 53ca75985..450ad8211 100644 --- a/proxy/src/main/java/com/wavefront/agent/auth/TokenAuthenticator.java +++ b/proxy/src/main/java/com/wavefront/agent/auth/TokenAuthenticator.java @@ -8,6 +8,8 @@ * @author vasily@wavefront.com */ public interface TokenAuthenticator { + /** Shared dummy authenticator. */ + TokenAuthenticator DUMMY_AUTHENTICATOR = new DummyAuthenticator(); /** * Validate a token. diff --git a/proxy/src/main/java/com/wavefront/agent/auth/TokenAuthenticatorBuilder.java b/proxy/src/main/java/com/wavefront/agent/auth/TokenAuthenticatorBuilder.java index 51024ff77..3a2f78b64 100644 --- a/proxy/src/main/java/com/wavefront/agent/auth/TokenAuthenticatorBuilder.java +++ b/proxy/src/main/java/com/wavefront/agent/auth/TokenAuthenticatorBuilder.java @@ -30,7 +30,8 @@ private TokenAuthenticatorBuilder() { this.staticToken = null; } - public TokenAuthenticatorBuilder setTokenValidationMethod(TokenValidationMethod tokenValidationMethod) { + public TokenAuthenticatorBuilder setTokenValidationMethod( + TokenValidationMethod tokenValidationMethod) { this.tokenValidationMethod = tokenValidationMethod; return this; } @@ -40,7 +41,8 @@ public TokenAuthenticatorBuilder setHttpClient(HttpClient httpClient) { return this; } - public TokenAuthenticatorBuilder setTokenIntrospectionServiceUrl(String tokenIntrospectionServiceUrl) { + public TokenAuthenticatorBuilder setTokenIntrospectionServiceUrl( + String tokenIntrospectionServiceUrl) { this.tokenIntrospectionServiceUrl = tokenIntrospectionServiceUrl; return this; } @@ -66,9 +68,7 @@ public TokenAuthenticatorBuilder setStaticToken(String staticToken) { return this; } - /** - * @return {@link TokenAuthenticator} instance. - */ + /** @return {@link TokenAuthenticator} instance. */ public TokenAuthenticator build() { switch (tokenValidationMethod) { case NONE: @@ -76,11 +76,19 @@ public TokenAuthenticator build() { case STATIC_TOKEN: return new StaticTokenAuthenticator(staticToken); case HTTP_GET: - return new HttpGetTokenIntrospectionAuthenticator(httpClient, tokenIntrospectionServiceUrl, - tokenIntrospectionAuthorizationHeader, authResponseRefreshInterval, authResponseMaxTtl); + return new HttpGetTokenIntrospectionAuthenticator( + httpClient, + tokenIntrospectionServiceUrl, + tokenIntrospectionAuthorizationHeader, + authResponseRefreshInterval, + authResponseMaxTtl); case OAUTH2: - return new Oauth2TokenIntrospectionAuthenticator(httpClient, tokenIntrospectionServiceUrl, - tokenIntrospectionAuthorizationHeader, authResponseRefreshInterval, authResponseMaxTtl); + return new Oauth2TokenIntrospectionAuthenticator( + httpClient, + tokenIntrospectionServiceUrl, + tokenIntrospectionAuthorizationHeader, + authResponseRefreshInterval, + authResponseMaxTtl); default: throw new IllegalStateException("Unknown token validation method!"); } diff --git a/proxy/src/main/java/com/wavefront/agent/auth/TokenIntrospectionAuthenticator.java b/proxy/src/main/java/com/wavefront/agent/auth/TokenIntrospectionAuthenticator.java index 082b63d07..ee1b36318 100644 --- a/proxy/src/main/java/com/wavefront/agent/auth/TokenIntrospectionAuthenticator.java +++ b/proxy/src/main/java/com/wavefront/agent/auth/TokenIntrospectionAuthenticator.java @@ -6,24 +6,23 @@ import com.yammer.metrics.Metrics; import com.yammer.metrics.core.Counter; import com.yammer.metrics.core.MetricName; - import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; - import javax.annotation.Nonnull; import javax.annotation.Nullable; /** - * {@link TokenAuthenticator} that uses an external webservice for validating tokens. - * Responses are cached and re-validated every {@code authResponseRefreshInterval} seconds; if the service is not + * {@link TokenAuthenticator} that uses an external webservice for validating tokens. Responses are + * cached and re-validated every {@code authResponseRefreshInterval} seconds; if the service is not * available, a cached last valid response may be used until {@code authResponseMaxTtl} expires. * * @author vasily@wavefront.com */ abstract class TokenIntrospectionAuthenticator implements TokenAuthenticator { - private static final Logger logger = Logger.getLogger(TokenIntrospectionAuthenticator.class.getCanonicalName()); + private static final Logger logger = + Logger.getLogger(TokenIntrospectionAuthenticator.class.getCanonicalName()); private final long authResponseMaxTtlMillis; @@ -34,50 +33,55 @@ abstract class TokenIntrospectionAuthenticator implements TokenAuthenticator { private final LoadingCache tokenValidityCache; - TokenIntrospectionAuthenticator(int authResponseRefreshInterval, int authResponseMaxTtl, - @Nonnull Supplier timeSupplier) { - this.authResponseMaxTtlMillis = TimeUnit.MILLISECONDS.convert(authResponseMaxTtl, TimeUnit.SECONDS); + TokenIntrospectionAuthenticator( + int authResponseRefreshInterval, + int authResponseMaxTtl, + @Nonnull Supplier timeSupplier) { + this.authResponseMaxTtlMillis = + TimeUnit.MILLISECONDS.convert(authResponseMaxTtl, TimeUnit.SECONDS); - this.tokenValidityCache = Caffeine.newBuilder() - .maximumSize(50_000) - .refreshAfterWrite(Math.min(authResponseRefreshInterval, authResponseMaxTtl), TimeUnit.SECONDS) - .ticker(() -> timeSupplier.get() * 1_000_000) // millisecond precision is fine - .build(new CacheLoader() { - @Override - public Boolean load(@Nonnull String key) { - serviceCalls.inc(); - boolean result; - try { - result = callAuthService(key); - lastSuccessfulCallTs = timeSupplier.get(); - } catch (Exception e) { - errorCount.inc(); - logger.log(Level.WARNING, "Error during Token Introspection Service call", e); - return null; - } - return result; - } + this.tokenValidityCache = + Caffeine.newBuilder() + .maximumSize(50_000) + .refreshAfterWrite( + Math.min(authResponseRefreshInterval, authResponseMaxTtl), TimeUnit.SECONDS) + .ticker(() -> timeSupplier.get() * 1_000_000) // millisecond precision is fine + .build( + new CacheLoader() { + @Override + public Boolean load(@Nonnull String key) { + serviceCalls.inc(); + boolean result; + try { + result = callAuthService(key); + lastSuccessfulCallTs = timeSupplier.get(); + } catch (Exception e) { + errorCount.inc(); + logger.log(Level.WARNING, "Error during Token Introspection Service call", e); + return null; + } + return result; + } - @Override - public Boolean reload(@Nonnull String key, - @Nonnull Boolean oldValue) { - serviceCalls.inc(); - boolean result; - try { - result = callAuthService(key); - lastSuccessfulCallTs = timeSupplier.get(); - } catch (Exception e) { - errorCount.inc(); - logger.log(Level.WARNING, "Error during Token Introspection Service call", e); - if (lastSuccessfulCallTs != null && - timeSupplier.get() - lastSuccessfulCallTs > authResponseMaxTtlMillis) { - return null; - } - return oldValue; - } - return result; - } - }); + @Override + public Boolean reload(@Nonnull String key, @Nonnull Boolean oldValue) { + serviceCalls.inc(); + boolean result; + try { + result = callAuthService(key); + lastSuccessfulCallTs = timeSupplier.get(); + } catch (Exception e) { + errorCount.inc(); + logger.log(Level.WARNING, "Error during Token Introspection Service call", e); + if (lastSuccessfulCallTs != null + && timeSupplier.get() - lastSuccessfulCallTs > authResponseMaxTtlMillis) { + return null; + } + return oldValue; + } + return result; + } + }); } abstract boolean callAuthService(@Nonnull String token) throws Exception; diff --git a/proxy/src/main/java/com/wavefront/agent/auth/TokenValidationMethod.java b/proxy/src/main/java/com/wavefront/agent/auth/TokenValidationMethod.java index 86001b1bb..f259d132a 100644 --- a/proxy/src/main/java/com/wavefront/agent/auth/TokenValidationMethod.java +++ b/proxy/src/main/java/com/wavefront/agent/auth/TokenValidationMethod.java @@ -1,15 +1,15 @@ package com.wavefront.agent.auth; -import com.beust.jcommander.IStringConverter; -import com.beust.jcommander.ParameterException; - /** * Auth validation methods supported. * * @author vasily@wavefront.com */ public enum TokenValidationMethod { - NONE, STATIC_TOKEN, HTTP_GET, OAUTH2; + NONE, + STATIC_TOKEN, + HTTP_GET, + OAUTH2; public static TokenValidationMethod fromString(String name) { for (TokenValidationMethod method : TokenValidationMethod.values()) { @@ -19,15 +19,4 @@ public static TokenValidationMethod fromString(String name) { } return null; } - - public class TokenValidationMethodConverter implements IStringConverter { - @Override - public TokenValidationMethod convert(String value) { - TokenValidationMethod convertedValue = TokenValidationMethod.fromString(value); - if (convertedValue == null) { - throw new ParameterException("Unknown token validation method value: " + value); - } - return convertedValue; - } - } } diff --git a/proxy/src/main/java/com/wavefront/agent/channel/CachingGraphiteHostAnnotator.java b/proxy/src/main/java/com/wavefront/agent/channel/CachingGraphiteHostAnnotator.java deleted file mode 100644 index 1e1bdb2e1..000000000 --- a/proxy/src/main/java/com/wavefront/agent/channel/CachingGraphiteHostAnnotator.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.wavefront.agent.channel; - -import com.google.common.collect.Lists; - -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.LoadingCache; -import com.wavefront.metrics.ExpectedAgentMetric; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Gauge; - -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import javax.annotation.Nullable; - -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import wavefront.report.ReportPoint; - -/** - * Given a raw Graphite/Wavefront line, look for any host tag, and add it if implicit. - * - * Differences from GraphiteHostAnnotator: - * - sharable - * - lazy load - does not proactively perform rDNS lookups unless needed - * - can be applied to HTTP payloads - * - * @author vasily@wavefront.com - */ -@ChannelHandler.Sharable -public class CachingGraphiteHostAnnotator { - private final LoadingCache rdnsCache; - private final boolean disableRdnsLookup; - private final List sourceTags; - - public CachingGraphiteHostAnnotator(@Nullable final List customSourceTags, boolean disableRdnsLookup) { - this.disableRdnsLookup = disableRdnsLookup; - this.sourceTags = Lists.newArrayListWithExpectedSize(customSourceTags == null ? 4 : customSourceTags.size() + 4); - this.sourceTags.add("source="); - this.sourceTags.add("source\"="); - this.sourceTags.add("host="); - this.sourceTags.add("host\"="); - if (customSourceTags != null) { - this.sourceTags.addAll(customSourceTags.stream().map(customTag -> customTag + "=").collect(Collectors.toList())); - } - - this.rdnsCache = disableRdnsLookup ? null : Caffeine.newBuilder() - .maximumSize(5000) - .refreshAfterWrite(5, TimeUnit.MINUTES) - .build(InetAddress::getHostName); - - Metrics.newGauge(ExpectedAgentMetric.RDNS_CACHE_SIZE.metricName, new Gauge() { - @Override - public Long value() { - return disableRdnsLookup ? 0 : rdnsCache.estimatedSize(); - } - }); - } - - public String apply(ChannelHandlerContext ctx, String msg) { - for (String tag : sourceTags) { - int strIndex = msg.indexOf(tag); - // if a source tags is found and is followed by a non-whitespace tag value, add without change - if (strIndex > -1 && msg.length() - strIndex - tag.length() > 0 && msg.charAt(strIndex + tag.length()) > ' ') { - return msg; - } - } - return msg + " source=\"" + getRemoteHost(ctx) + "\""; - } - - public String getRemoteHost(ChannelHandlerContext ctx) { - InetAddress remote = ((InetSocketAddress) ctx.channel().remoteAddress()).getAddress(); - return disableRdnsLookup ? remote.getHostAddress() : rdnsCache.get(remote); - } -} diff --git a/proxy/src/main/java/com/wavefront/agent/channel/CachingHostnameLookupResolver.java b/proxy/src/main/java/com/wavefront/agent/channel/CachingHostnameLookupResolver.java new file mode 100644 index 000000000..5ba2a7baa --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/channel/CachingHostnameLookupResolver.java @@ -0,0 +1,97 @@ +package com.wavefront.agent.channel; + +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.annotations.VisibleForTesting; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Gauge; +import com.yammer.metrics.core.MetricName; +import java.net.InetAddress; +import java.time.Duration; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Convert {@link InetAddress} to {@link String}, either by performing reverse DNS lookups (cached, + * as the name implies), or by converting IP addresses into their string representation. + * + * @author vasily@wavefront.com + */ +public class CachingHostnameLookupResolver implements Function { + + private final Function resolverFunc; + private final LoadingCache rdnsCache; + private final boolean disableRdnsLookup; + + /** + * Create a new instance with default cache settings: - max 5000 elements in the cache - 5 minutes + * refresh TTL - 1 hour expiry TTL + * + * @param disableRdnsLookup if true, simply return a string representation of the IP address + * @param metricName if specified, use this metric for the cache size gauge. + */ + public CachingHostnameLookupResolver(boolean disableRdnsLookup, @Nullable MetricName metricName) { + this(disableRdnsLookup, metricName, 5000, Duration.ofMinutes(5), Duration.ofHours(1)); + } + + /** + * Create a new instance with specific cache settings: + * + * @param disableRdnsLookup if true, simply return a string representation of the IP address. + * @param metricName if specified, use this metric for the cache size gauge. + * @param cacheSize max cache size. + * @param cacheRefreshTtl trigger cache refresh after specified duration + * @param cacheExpiryTtl expire items after specified duration + */ + public CachingHostnameLookupResolver( + boolean disableRdnsLookup, + @Nullable MetricName metricName, + int cacheSize, + Duration cacheRefreshTtl, + Duration cacheExpiryTtl) { + this( + InetAddress::getHostAddress, + disableRdnsLookup, + metricName, + cacheSize, + cacheRefreshTtl, + cacheExpiryTtl); + } + + @VisibleForTesting + CachingHostnameLookupResolver( + @Nonnull Function resolverFunc, + boolean disableRdnsLookup, + @Nullable MetricName metricName, + int cacheSize, + Duration cacheRefreshTtl, + Duration cacheExpiryTtl) { + this.resolverFunc = resolverFunc; + this.disableRdnsLookup = disableRdnsLookup; + this.rdnsCache = + disableRdnsLookup + ? null + : Caffeine.newBuilder() + .maximumSize(cacheSize) + .refreshAfterWrite(cacheRefreshTtl) + .expireAfterAccess(cacheExpiryTtl) + .build(key -> InetAddress.getByAddress(key.getAddress()).getHostName()); + + if (metricName != null) { + Metrics.newGauge( + metricName, + new Gauge() { + @Override + public Long value() { + return disableRdnsLookup ? 0 : rdnsCache.estimatedSize(); + } + }); + } + } + + @Override + public String apply(InetAddress addr) { + return disableRdnsLookup ? resolverFunc.apply(addr) : rdnsCache.get(addr); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/channel/ChannelUtils.java b/proxy/src/main/java/com/wavefront/agent/channel/ChannelUtils.java new file mode 100644 index 000000000..6bb9ea733 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/channel/ChannelUtils.java @@ -0,0 +1,244 @@ +package com.wavefront.agent.channel; + +import com.fasterxml.jackson.databind.JsonNode; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.base.Throwables; +import com.wavefront.agent.SharedMetricsRegistry; +import com.wavefront.common.TaggedMetricName; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Counter; +import com.yammer.metrics.core.MetricName; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; +import io.netty.handler.codec.http.HttpMessage; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.util.CharsetUtil; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * A collection of helper methods around Netty channels. + * + * @author vasily@wavefront.com + */ +public abstract class ChannelUtils { + + private static final Map> RESPONSE_STATUS_CACHES = + new ConcurrentHashMap<>(); + + /** + * Create a detailed error message from an exception, including current handle (port). + * + * @param message the error message + * @param e the exception (optional) that caused the error + * @param ctx ChannelHandlerContext (optional) to extract remote client ip + * @return formatted error message + */ + public static String formatErrorMessage( + final String message, + @Nullable final Throwable e, + @Nullable final ChannelHandlerContext ctx) { + StringBuilder errMsg = new StringBuilder(message); + if (ctx != null) { + errMsg.append("; remote: "); + errMsg.append(getRemoteName(ctx)); + } + if (e != null) { + errMsg.append("; "); + errMsg.append(errorMessageWithRootCause(e)); + } + return errMsg.toString(); + } + + /** + * Writes HTTP response back to client. + * + * @param ctx Channel handler context + * @param status HTTP status to return with the response + * @param contents Response body payload (JsonNode or CharSequence) + * @param request Incoming request (used to get keep-alive header) + */ + public static void writeHttpResponse( + final ChannelHandlerContext ctx, + final HttpResponseStatus status, + final Object /* JsonNode | CharSequence */ contents, + final HttpMessage request) { + writeHttpResponse(ctx, status, contents, HttpUtil.isKeepAlive(request)); + } + + /** + * Writes HTTP response back to client. + * + * @param ctx Channel handler context + * @param status HTTP status to return with the response + * @param contents Response body payload (JsonNode or CharSequence) + * @param keepAlive Keep-alive requested + */ + public static void writeHttpResponse( + final ChannelHandlerContext ctx, + final HttpResponseStatus status, + final Object /* JsonNode | CharSequence */ contents, + boolean keepAlive) { + writeHttpResponse(ctx, makeResponse(status, contents), keepAlive); + } + + /** + * Writes HTTP response back to client. + * + * @param ctx Channel handler context. + * @param response HTTP response object. + * @param request HTTP request object (to extract keep-alive flag). + */ + public static void writeHttpResponse( + final ChannelHandlerContext ctx, final HttpResponse response, final HttpMessage request) { + writeHttpResponse(ctx, response, HttpUtil.isKeepAlive(request)); + } + + /** + * Writes HTTP response back to client. + * + * @param ctx Channel handler context. + * @param response HTTP response object. + * @param keepAlive Keep-alive requested. + */ + public static void writeHttpResponse( + final ChannelHandlerContext ctx, final HttpResponse response, boolean keepAlive) { + getHttpStatusCounter(ctx, response.status().code()).inc(); + // Decide whether to close the connection or not. + if (keepAlive) { + // Add keep alive header as per: + // - http://www.w3.org/Protocols/HTTP/1.1/draft-ietf-http-v11-spec-01.html#Connection + response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE); + ctx.write(response); + } else { + ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE); + } + } + + /** + * Create {@link FullHttpResponse} based on provided status and body contents. + * + * @param status response status. + * @param contents response body. + * @return http response object + */ + public static HttpResponse makeResponse( + final HttpResponseStatus status, final Object /* JsonNode | CharSequence */ contents) { + final FullHttpResponse response; + if (contents instanceof JsonNode) { + response = + new DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, + status, + Unpooled.copiedBuffer(contents.toString(), CharsetUtil.UTF_8)); + response.headers().set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.APPLICATION_JSON); + } else if (contents instanceof CharSequence) { + response = + new DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, + status, + Unpooled.copiedBuffer((CharSequence) contents, CharsetUtil.UTF_8)); + response.headers().set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.TEXT_PLAIN); + } else { + throw new IllegalArgumentException( + "Unexpected response content type, JsonNode or " + + "CharSequence expected: " + + contents.getClass().getName()); + } + response.headers().set(HttpHeaderNames.CONTENT_LENGTH, response.content().readableBytes()); + return response; + } + + /** + * Write detailed exception text. + * + * @param e Exceptions thrown + * @return error message + */ + public static String errorMessageWithRootCause(@Nonnull final Throwable e) { + StringBuilder msg = new StringBuilder(); + final Throwable rootCause = Throwables.getRootCause(e); + msg.append("reason: \""); + msg.append(e.getMessage()); + msg.append("\""); + if (rootCause != null && rootCause != e && rootCause.getMessage() != null) { + msg.append(", root cause: \""); + msg.append(rootCause.getMessage()); + msg.append("\""); + } + return msg.toString(); + } + + /** + * Get remote client's address as string (without rDNS lookup) and local port + * + * @param ctx Channel handler context + * @return remote client's address in a string form + */ + @Nonnull + public static String getRemoteName(@Nullable final ChannelHandlerContext ctx) { + if (ctx != null) { + InetAddress remoteAddress = getRemoteAddress(ctx); + InetSocketAddress localAddress = (InetSocketAddress) ctx.channel().localAddress(); + if (remoteAddress != null && localAddress != null) { + return remoteAddress.getHostAddress() + " [" + localAddress.getPort() + "]"; + } + } + return ""; + } + + /** + * Get {@link InetAddress} for the current channel. + * + * @param ctx Channel handler's context. + * @return remote address + */ + public static InetAddress getRemoteAddress(@Nonnull ChannelHandlerContext ctx) { + InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress(); + return remoteAddress == null ? null : remoteAddress.getAddress(); + } + + /** + * Get a counter for ~proxy.listeners.http-requests.status.###.count metric for a specific status + * code, with port= point tag for added context. + * + * @param ctx channel handler context where a response is being sent. + * @param status response status code. + */ + public static Counter getHttpStatusCounter(ChannelHandlerContext ctx, int status) { + if (ctx != null && ctx.channel() != null) { + InetSocketAddress localAddress = (InetSocketAddress) ctx.channel().localAddress(); + if (localAddress != null) { + return RESPONSE_STATUS_CACHES + .computeIfAbsent( + localAddress.getPort(), + port -> + Caffeine.newBuilder() + .build( + statusCode -> + Metrics.newCounter( + new TaggedMetricName( + "listeners", + "http-requests.status." + statusCode + ".count", + "port", + String.valueOf(port))))) + .get(status); + } + } + // return a non-reportable counter otherwise + return SharedMetricsRegistry.getInstance().newCounter(new MetricName("", "", "dummy")); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/channel/ConnectionTrackingHandler.java b/proxy/src/main/java/com/wavefront/agent/channel/ConnectionTrackingHandler.java index ff57cbb34..4d6bf6117 100644 --- a/proxy/src/main/java/com/wavefront/agent/channel/ConnectionTrackingHandler.java +++ b/proxy/src/main/java/com/wavefront/agent/channel/ConnectionTrackingHandler.java @@ -1,15 +1,14 @@ package com.wavefront.agent.channel; import com.yammer.metrics.core.Counter; - -import javax.validation.constraints.NotNull; - import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; +import javax.annotation.Nonnull; /** - * Track the number of currently active connections and total count of accepted incoming connections. + * Track the number of currently active connections and total count of accepted incoming + * connections. * * @author vasily@wavefront.com */ @@ -19,8 +18,8 @@ public class ConnectionTrackingHandler extends ChannelInboundHandlerAdapter { private final Counter acceptedConnections; private final Counter activeConnections; - public ConnectionTrackingHandler(@NotNull Counter acceptedConnectionsCounter, - @NotNull Counter activeConnectionsCounter) { + public ConnectionTrackingHandler( + @Nonnull Counter acceptedConnectionsCounter, @Nonnull Counter activeConnectionsCounter) { this.acceptedConnections = acceptedConnectionsCounter; this.activeConnections = activeConnectionsCounter; } diff --git a/proxy/src/main/java/com/wavefront/agent/channel/DisableGZIPEncodingInterceptor.java b/proxy/src/main/java/com/wavefront/agent/channel/DisableGZIPEncodingInterceptor.java index 0a81ae809..8d3004d18 100644 --- a/proxy/src/main/java/com/wavefront/agent/channel/DisableGZIPEncodingInterceptor.java +++ b/proxy/src/main/java/com/wavefront/agent/channel/DisableGZIPEncodingInterceptor.java @@ -2,26 +2,26 @@ import java.io.IOException; import java.util.logging.Logger; - import javax.ws.rs.WebApplicationException; import javax.ws.rs.ext.WriterInterceptor; import javax.ws.rs.ext.WriterInterceptorContext; /** - * This RESTEasy interceptor allows disabling GZIP compression even for methods annotated with @GZIP by removing the - * Content-Encoding header. - * RESTEasy always adds "Content-Encoding: gzip" header when it encounters @GZIP annotation, but if the request body - * is actually sent uncompressed, it violates section 3.1.2.2 of RFC7231. + * This RESTEasy interceptor allows disabling GZIP compression even for methods annotated with @GZIP + * by removing the Content-Encoding header. RESTEasy always adds "Content-Encoding: gzip" header + * when it encounters @GZIP annotation, but if the request body is actually sent uncompressed, it + * violates section 3.1.2.2 of RFC7231. * - * Created by vasily@wavefront.com on 6/9/17. + *

Created by vasily@wavefront.com on 6/9/17. */ public class DisableGZIPEncodingInterceptor implements WriterInterceptor { - private static final Logger logger = Logger.getLogger(DisableGZIPEncodingInterceptor.class.getCanonicalName()); + private static final Logger logger = + Logger.getLogger(DisableGZIPEncodingInterceptor.class.getCanonicalName()); - public DisableGZIPEncodingInterceptor() { - } + public DisableGZIPEncodingInterceptor() {} - public void aroundWriteTo(WriterInterceptorContext context) throws IOException, WebApplicationException { + public void aroundWriteTo(WriterInterceptorContext context) + throws IOException, WebApplicationException { logger.fine("Interceptor : " + this.getClass().getName() + ", Method : aroundWriteTo"); Object encoding = context.getHeaders().getFirst("Content-Encoding"); if (encoding != null && encoding.toString().equalsIgnoreCase("gzip")) { diff --git a/proxy/src/main/java/com/wavefront/agent/channel/GZIPEncodingInterceptorWithVariableCompression.java b/proxy/src/main/java/com/wavefront/agent/channel/GZIPEncodingInterceptorWithVariableCompression.java new file mode 100644 index 000000000..9f723dd5f --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/channel/GZIPEncodingInterceptorWithVariableCompression.java @@ -0,0 +1,86 @@ +package com.wavefront.agent.channel; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.zip.GZIPOutputStream; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.ext.WriterInterceptor; +import javax.ws.rs.ext.WriterInterceptorContext; +import org.jboss.resteasy.util.CommitHeaderOutputStream; + +/** + * An alternative to {@link + * org.jboss.resteasy.plugins.interceptors.encoding.GZIPEncodingInterceptor} that allows changing + * the GZIP deflater's compression level. + * + * @author vasily@wavefront.com + * @author Bill Burke + */ +public class GZIPEncodingInterceptorWithVariableCompression implements WriterInterceptor { + private final int level; + + public GZIPEncodingInterceptorWithVariableCompression(int level) { + this.level = level; + } + + public static class EndableGZIPOutputStream extends GZIPOutputStream { + public EndableGZIPOutputStream(final OutputStream os, int level) throws IOException { + super(os); + this.def.setLevel(level); + } + + @Override + public void finish() throws IOException { + super.finish(); + def.end(); + } + } + + public static class CommittedGZIPOutputStream extends CommitHeaderOutputStream { + private final int level; + + protected CommittedGZIPOutputStream(final OutputStream delegate, int level) { + super(delegate, null); + this.level = level; + } + + protected GZIPOutputStream gzip; + + public GZIPOutputStream getGzip() { + return gzip; + } + + @Override + public synchronized void commit() { + if (isHeadersCommitted) return; + isHeadersCommitted = true; + try { + gzip = new EndableGZIPOutputStream(delegate, level); + delegate = gzip; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public void aroundWriteTo(WriterInterceptorContext context) + throws IOException, WebApplicationException { + Object encoding = context.getHeaders().getFirst(HttpHeaders.CONTENT_ENCODING); + if (encoding != null && encoding.toString().equalsIgnoreCase("gzip")) { + OutputStream old = context.getOutputStream(); + CommittedGZIPOutputStream gzipOutputStream = new CommittedGZIPOutputStream(old, level); + context.getHeaders().remove("Content-Length"); + context.setOutputStream(gzipOutputStream); + try { + context.proceed(); + } finally { + if (gzipOutputStream.getGzip() != null) gzipOutputStream.getGzip().finish(); + context.setOutputStream(old); + } + } else { + context.proceed(); + } + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/channel/HealthCheckManager.java b/proxy/src/main/java/com/wavefront/agent/channel/HealthCheckManager.java new file mode 100644 index 000000000..37bdb9d8f --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/channel/HealthCheckManager.java @@ -0,0 +1,29 @@ +package com.wavefront.agent.channel; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import java.net.URISyntaxException; +import javax.annotation.Nonnull; + +/** + * Centrally manages healthcheck statuses (for controlling load balancers). + * + * @author vasily@wavefront.com + */ +public interface HealthCheckManager { + HttpResponse getHealthCheckResponse(ChannelHandlerContext ctx, @Nonnull FullHttpRequest request) + throws URISyntaxException; + + boolean isHealthy(int port); + + void setHealthy(int port); + + void setUnhealthy(int port); + + void setAllHealthy(); + + void setAllUnhealthy(); + + void enableHealthcheck(int port); +} diff --git a/proxy/src/main/java/com/wavefront/agent/channel/HealthCheckManagerImpl.java b/proxy/src/main/java/com/wavefront/agent/channel/HealthCheckManagerImpl.java new file mode 100644 index 000000000..2c5071764 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/channel/HealthCheckManagerImpl.java @@ -0,0 +1,162 @@ +package com.wavefront.agent.channel; + +import com.google.common.annotations.VisibleForTesting; +import com.wavefront.agent.ProxyConfig; +import com.wavefront.common.TaggedMetricName; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.Gauge; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.util.CharsetUtil; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.commons.lang3.ObjectUtils; + +/** + * Centrally manages healthcheck statuses (for controlling load balancers). + * + * @author vasily@wavefront.com. + */ +public class HealthCheckManagerImpl implements HealthCheckManager { + private static final Logger log = Logger.getLogger(HealthCheckManager.class.getCanonicalName()); + + private final Map statusMap; + private final Set enabledPorts; + private final String path; + private final String contentType; + private final int passStatusCode; + private final String passResponseBody; + private final int failStatusCode; + private final String failResponseBody; + + /** @param config Proxy configuration */ + public HealthCheckManagerImpl(@Nonnull ProxyConfig config) { + this( + config.getHttpHealthCheckPath(), + config.getHttpHealthCheckResponseContentType(), + config.getHttpHealthCheckPassStatusCode(), + config.getHttpHealthCheckPassResponseBody(), + config.getHttpHealthCheckFailStatusCode(), + config.getHttpHealthCheckFailResponseBody()); + } + + /** + * @param path Health check's path. + * @param contentType Optional content-type of health check's response. + * @param passStatusCode HTTP status code for 'pass' health checks. + * @param passResponseBody Optional response body to return with 'pass' health checks. + * @param failStatusCode HTTP status code for 'fail' health checks. + * @param failResponseBody Optional response body to return with 'fail' health checks. + */ + @VisibleForTesting + HealthCheckManagerImpl( + @Nullable String path, + @Nullable String contentType, + int passStatusCode, + @Nullable String passResponseBody, + int failStatusCode, + @Nullable String failResponseBody) { + this.statusMap = new HashMap<>(); + this.enabledPorts = new HashSet<>(); + this.path = path; + this.contentType = contentType; + this.passStatusCode = passStatusCode; + this.passResponseBody = ObjectUtils.firstNonNull(passResponseBody, ""); + this.failStatusCode = failStatusCode; + this.failResponseBody = ObjectUtils.firstNonNull(failResponseBody, ""); + } + + @Override + public HttpResponse getHealthCheckResponse( + ChannelHandlerContext ctx, @Nonnull FullHttpRequest request) throws URISyntaxException { + int port = ((InetSocketAddress) ctx.channel().localAddress()).getPort(); + if (!enabledPorts.contains(port)) return null; + URI uri = new URI(request.uri()); + if (!(this.path == null || this.path.equals(uri.getPath()))) return null; + // it is a health check URL, now we need to determine current status and respond accordingly + final boolean ok = isHealthy(port); + Metrics.newGauge( + new TaggedMetricName("listeners", "healthcheck.status", "port", String.valueOf(port)), + new Gauge() { + @Override + public Integer value() { + return isHealthy(port) ? 1 : 0; + } + }); + final FullHttpResponse response = + new DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, + HttpResponseStatus.valueOf(ok ? passStatusCode : failStatusCode), + Unpooled.copiedBuffer(ok ? passResponseBody : failResponseBody, CharsetUtil.UTF_8)); + if (contentType != null) { + response.headers().set(HttpHeaderNames.CONTENT_TYPE, contentType); + } + if (HttpUtil.isKeepAlive(request)) { + response.headers().set(HttpHeaderNames.CONTENT_LENGTH, response.content().readableBytes()); + response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE); + } + Metrics.newCounter( + new TaggedMetricName( + "listeners", + "healthcheck.httpstatus." + (ok ? passStatusCode : failStatusCode) + ".count", + "port", + String.valueOf(port))) + .inc(); + return response; + } + + @Override + public boolean isHealthy(int port) { + return statusMap.getOrDefault(port, true); + } + + @Override + public void setHealthy(int port) { + statusMap.put(port, true); + } + + @Override + public void setUnhealthy(int port) { + statusMap.put(port, false); + } + + @Override + public void setAllHealthy() { + enabledPorts.forEach( + x -> { + setHealthy(x); + log.info("Port " + x + " was marked as healthy"); + }); + } + + @Override + public void setAllUnhealthy() { + enabledPorts.forEach( + x -> { + setUnhealthy(x); + log.info("Port " + x + " was marked as unhealthy"); + }); + } + + @Override + public void enableHealthcheck(int port) { + enabledPorts.add(port); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/channel/IdleStateEventHandler.java b/proxy/src/main/java/com/wavefront/agent/channel/IdleStateEventHandler.java index 49ac63cb7..d9af2a5ea 100644 --- a/proxy/src/main/java/com/wavefront/agent/channel/IdleStateEventHandler.java +++ b/proxy/src/main/java/com/wavefront/agent/channel/IdleStateEventHandler.java @@ -1,17 +1,14 @@ package com.wavefront.agent.channel; import com.yammer.metrics.core.Counter; - -import java.net.InetSocketAddress; -import java.util.logging.Logger; - -import javax.validation.constraints.NotNull; - import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.timeout.IdleState; import io.netty.handler.timeout.IdleStateEvent; +import java.net.InetSocketAddress; +import java.util.logging.Logger; +import javax.annotation.Nonnull; /** * Disconnect idle clients (handle READER_IDLE events triggered by IdleStateHandler) @@ -20,12 +17,12 @@ */ @ChannelHandler.Sharable public class IdleStateEventHandler extends ChannelInboundHandlerAdapter { - private static final Logger logger = Logger.getLogger( - IdleStateEventHandler.class.getCanonicalName()); + private static final Logger logger = + Logger.getLogger(IdleStateEventHandler.class.getCanonicalName()); - private Counter idleClosedConnections; + private final Counter idleClosedConnections; - public IdleStateEventHandler(@NotNull Counter idleClosedConnectionsCounter) { + public IdleStateEventHandler(@Nonnull Counter idleClosedConnectionsCounter) { this.idleClosedConnections = idleClosedConnectionsCounter; } @@ -35,8 +32,11 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { if (((IdleStateEvent) evt).state() == IdleState.READER_IDLE) { // close idle connections InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress(); InetSocketAddress localAddress = (InetSocketAddress) ctx.channel().localAddress(); - logger.info("Closing idle connection on port " + localAddress.getPort() + - ", remote address: " + remoteAddress.getAddress().getHostAddress()); + logger.info( + "Closing idle connection on port " + + localAddress.getPort() + + ", remote address: " + + remoteAddress.getAddress().getHostAddress()); idleClosedConnections.inc(); ctx.channel().close(); } diff --git a/proxy/src/main/java/com/wavefront/agent/channel/IncompleteLineDetectingLineBasedFrameDecoder.java b/proxy/src/main/java/com/wavefront/agent/channel/IncompleteLineDetectingLineBasedFrameDecoder.java new file mode 100644 index 000000000..31749db89 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/channel/IncompleteLineDetectingLineBasedFrameDecoder.java @@ -0,0 +1,46 @@ +package com.wavefront.agent.channel; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.LineBasedFrameDecoder; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.function.Consumer; +import javax.annotation.Nonnull; +import org.apache.commons.lang3.StringUtils; + +/** + * Line-delimited decoder that has the ability of detecting when clients have disconnected while + * leaving some data in the buffer. + * + * @author vasily@wavefront.com + */ +public class IncompleteLineDetectingLineBasedFrameDecoder extends LineBasedFrameDecoder { + private final Consumer warningMessageConsumer; + + IncompleteLineDetectingLineBasedFrameDecoder( + @Nonnull Consumer warningMessageConsumer, int maxLength) { + super(maxLength, true, false); + this.warningMessageConsumer = warningMessageConsumer; + } + + @Override + protected void decodeLast(ChannelHandlerContext ctx, ByteBuf in, List out) + throws Exception { + super.decodeLast(ctx, in, out); + int readableBytes = in.readableBytes(); + if (readableBytes > 0) { + String discardedData = in.readBytes(readableBytes).toString(StandardCharsets.UTF_8); + if (StringUtils.isNotBlank(discardedData)) { + warningMessageConsumer.accept( + "Client " + + ChannelUtils.getRemoteName(ctx) + + " disconnected, leaving unterminated string. Input (" + + readableBytes + + " bytes) discarded: \"" + + discardedData + + "\""); + } + } + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/channel/NoopHealthCheckManager.java b/proxy/src/main/java/com/wavefront/agent/channel/NoopHealthCheckManager.java new file mode 100644 index 000000000..a9c7a75eb --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/channel/NoopHealthCheckManager.java @@ -0,0 +1,39 @@ +package com.wavefront.agent.channel; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import javax.annotation.Nonnull; + +/** + * A no-op health check manager. + * + * @author vasily@wavefront.com. + */ +public class NoopHealthCheckManager implements HealthCheckManager { + @Override + public HttpResponse getHealthCheckResponse( + ChannelHandlerContext ctx, @Nonnull FullHttpRequest request) { + return null; + } + + @Override + public boolean isHealthy(int port) { + return true; + } + + @Override + public void setHealthy(int port) {} + + @Override + public void setUnhealthy(int port) {} + + @Override + public void setAllHealthy() {} + + @Override + public void setAllUnhealthy() {} + + @Override + public void enableHealthcheck(int port) {} +} diff --git a/proxy/src/main/java/com/wavefront/agent/channel/PlainTextOrHttpFrameDecoder.java b/proxy/src/main/java/com/wavefront/agent/channel/PlainTextOrHttpFrameDecoder.java index 5159cbb7a..15e7b396d 100644 --- a/proxy/src/main/java/com/wavefront/agent/channel/PlainTextOrHttpFrameDecoder.java +++ b/proxy/src/main/java/com/wavefront/agent/channel/PlainTextOrHttpFrameDecoder.java @@ -1,43 +1,44 @@ package com.wavefront.agent.channel; import com.google.common.base.Charsets; - import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPipeline; import io.netty.handler.codec.ByteToMessageDecoder; -import io.netty.handler.codec.LineBasedFrameDecoder; import io.netty.handler.codec.compression.ZlibCodecFactory; import io.netty.handler.codec.compression.ZlibWrapper; import io.netty.handler.codec.http.HttpContentDecompressor; -import io.netty.handler.codec.string.StringDecoder; -import io.netty.handler.codec.string.StringEncoder; -import io.netty.handler.codec.http.HttpObjectAggregator; import io.netty.handler.codec.http.HttpRequestDecoder; import io.netty.handler.codec.http.HttpResponseEncoder; - +import io.netty.handler.codec.http.cors.CorsConfig; +import io.netty.handler.codec.http.cors.CorsHandler; +import io.netty.handler.codec.string.StringDecoder; +import io.netty.handler.codec.string.StringEncoder; import java.util.List; import java.util.logging.Logger; +import javax.annotation.Nullable; /** - * This class handles 2 different protocols on a single port. Supported protocols include HTTP and - * a plain text protocol. It dynamically adds the appropriate encoder/decoder based on the detected + * This class handles 2 different protocols on a single port. Supported protocols include HTTP and a + * plain text protocol. It dynamically adds the appropriate encoder/decoder based on the detected * protocol. This class was largely adapted from the example provided with netty v4.0 * - * @see Netty - * Port Unification Example + * @see Netty + * Port Unification Example * @author Mike McLaughlin (mike@wavefront.com) */ public final class PlainTextOrHttpFrameDecoder extends ByteToMessageDecoder { - protected static final Logger logger = Logger.getLogger(PlainTextOrHttpFrameDecoder.class.getName()); + protected static final Logger logger = + Logger.getLogger(PlainTextOrHttpFrameDecoder.class.getName()); - /** - * The object for handling requests of either protocol - */ + /** The object for handling requests of either protocol */ private final ChannelHandler handler; + private final boolean detectGzip; + @Nullable private final CorsConfig corsConfig; private final int maxLengthPlaintext; private final int maxLengthHttp; @@ -45,26 +46,27 @@ public final class PlainTextOrHttpFrameDecoder extends ByteToMessageDecoder { private static final StringEncoder STRING_ENCODER = new StringEncoder(Charsets.UTF_8); /** - * Constructor with default input buffer limits (4KB for plaintext, 16MB for HTTP). - * - * @param handler the object responsible for handling the incoming messages or either protocol - */ - public PlainTextOrHttpFrameDecoder(final ChannelHandler handler) { - this(handler, 4096, 16 * 1024 * 1024, true); - } - - /** - * Constructor. - * - * @param handler the object responsible for handling the incoming messages or either protocol + * @param handler the object responsible for handling the incoming messages on either protocol. + * @param corsConfig enables CORS when {@link CorsConfig} is specified + * @param maxLengthPlaintext max allowed line length for line-delimiter protocol + * @param maxLengthHttp max allowed size for incoming HTTP requests */ - public PlainTextOrHttpFrameDecoder(final ChannelHandler handler, int maxLengthPlaintext, int maxLengthHttp) { - this(handler, maxLengthPlaintext, maxLengthHttp, true); + public PlainTextOrHttpFrameDecoder( + final ChannelHandler handler, + @Nullable final CorsConfig corsConfig, + int maxLengthPlaintext, + int maxLengthHttp) { + this(handler, corsConfig, maxLengthPlaintext, maxLengthHttp, true); } - private PlainTextOrHttpFrameDecoder(final ChannelHandler handler, int maxLengthPlaintext, - int maxLengthHttp, boolean detectGzip) { + private PlainTextOrHttpFrameDecoder( + final ChannelHandler handler, + @Nullable final CorsConfig corsConfig, + int maxLengthPlaintext, + int maxLengthHttp, + boolean detectGzip) { this.handler = handler; + this.corsConfig = corsConfig; this.maxLengthPlaintext = maxLengthPlaintext; this.maxLengthHttp = maxLengthHttp; this.detectGzip = detectGzip; @@ -75,10 +77,13 @@ private PlainTextOrHttpFrameDecoder(final ChannelHandler handler, int maxLengthP * protocol. */ @Override - protected void decode(final ChannelHandlerContext ctx, final ByteBuf buffer, List out) throws Exception { + protected void decode(final ChannelHandlerContext ctx, final ByteBuf buffer, List out) { // read the first 2 bytes to use for protocol detection if (buffer.readableBytes() < 2) { - logger.info("Inbound data from " + ctx.channel().remoteAddress()+ " has less that 2 readable bytes - ignoring "); + logger.info( + "Inbound data from " + + ctx.channel().remoteAddress() + + " has less that 2 readable bytes - ignoring"); return; } final int firstByte = buffer.getUnsignedByte(buffer.readerIndex()); @@ -92,23 +97,31 @@ protected void decode(final ChannelHandlerContext ctx, final ByteBuf buffer, Lis pipeline .addLast("gzipdeflater", ZlibCodecFactory.newZlibEncoder(ZlibWrapper.GZIP)) .addLast("gzipinflater", ZlibCodecFactory.newZlibDecoder(ZlibWrapper.GZIP)) - .addLast("unificationB", new PlainTextOrHttpFrameDecoder(handler, maxLengthPlaintext, maxLengthHttp, false)); + .addLast( + "unificationB", + new PlainTextOrHttpFrameDecoder( + handler, corsConfig, maxLengthPlaintext, maxLengthHttp, false)); } else if (isHttp(firstByte, secondByte)) { logger.fine("Switching to HTTP protocol"); pipeline .addLast("decoder", new HttpRequestDecoder()) .addLast("inflater", new HttpContentDecompressor()) .addLast("encoder", new HttpResponseEncoder()) - .addLast("aggregator", new HttpObjectAggregator(maxLengthHttp)) - .addLast("handler", this.handler); - } else { - logger.fine("Using TCP plaintext protocol"); - pipeline.addLast("line", new LineBasedFrameDecoder(maxLengthPlaintext)); - pipeline.addLast("decoder", STRING_DECODER); - pipeline.addLast("encoder", STRING_ENCODER); + .addLast("aggregator", new StatusTrackingHttpObjectAggregator(maxLengthHttp)); + if (corsConfig != null) { + pipeline.addLast("corsHandler", new CorsHandler(corsConfig)); + } pipeline.addLast("handler", this.handler); + } else { + logger.fine("Switching to plaintext TCP protocol"); + pipeline + .addLast( + "line", + new IncompleteLineDetectingLineBasedFrameDecoder(logger::warning, maxLengthPlaintext)) + .addLast("decoder", STRING_DECODER) + .addLast("encoder", STRING_ENCODER) + .addLast("handler", this.handler); } - pipeline.remove(this); } @@ -116,31 +129,39 @@ protected void decode(final ChannelHandlerContext ctx, final ByteBuf buffer, Lis * @param magic1 the first byte of the incoming message * @param magic2 the second byte of the incoming message * @return true if this is an HTTP message; false o/w - * @see Netty - * Port Unification Example + * @see Netty + * Port Unification Example */ private static boolean isHttp(int magic1, int magic2) { - return - ((magic1 == 'G' && magic2 == 'E') || // GET - (magic1 == 'P' && magic2 == 'O') || // POST - (magic1 == 'P' && magic2 == 'U') || // PUT - (magic1 == 'H' && magic2 == 'E') || // HEAD - (magic1 == 'O' && magic2 == 'P') || // OPTIONS - (magic1 == 'P' && magic2 == 'A') || // PATCH - (magic1 == 'D' && magic2 == 'E') || // DELETE - (magic1 == 'T' && magic2 == 'R') || // TRACE - (magic1 == 'C' && magic2 == 'O')); // CONNECT + return ((magic1 == 'G' && magic2 == 'E') + || // GET + (magic1 == 'P' && magic2 == 'O') + || // POST + (magic1 == 'P' && magic2 == 'U') + || // PUT + (magic1 == 'H' && magic2 == 'E') + || // HEAD + (magic1 == 'O' && magic2 == 'P') + || // OPTIONS + (magic1 == 'P' && magic2 == 'A') + || // PATCH + (magic1 == 'D' && magic2 == 'E') + || // DELETE + (magic1 == 'T' && magic2 == 'R') + || // TRACE + (magic1 == 'C' && magic2 == 'O')); // CONNECT } /** * @param magic1 the first byte of the incoming message * @param magic2 the second byte of the incoming message * @return true if this is a GZIP stream; false o/w - * @see Netty - * Port Unification Example + * @see Netty + * Port Unification Example */ private static boolean isGzip(int magic1, int magic2) { return magic1 == 31 && magic2 == 139; } - } diff --git a/proxy/src/main/java/com/wavefront/agent/channel/SharedGraphiteHostAnnotator.java b/proxy/src/main/java/com/wavefront/agent/channel/SharedGraphiteHostAnnotator.java new file mode 100644 index 000000000..7ace9dceb --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/channel/SharedGraphiteHostAnnotator.java @@ -0,0 +1,77 @@ +package com.wavefront.agent.channel; + +import static com.wavefront.agent.channel.ChannelUtils.getRemoteAddress; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Streams; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import java.net.InetAddress; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Given a raw Graphite/Wavefront line, look for any host tag, and add it if implicit. + * + *

Differences from GraphiteHostAnnotator: - sharable - lazy load - does not proactively perform + * rDNS lookups unless needed - can be applied to HTTP payloads + * + * @author vasily@wavefront.com + */ +@ChannelHandler.Sharable +public class SharedGraphiteHostAnnotator { + private static final List DEFAULT_SOURCE_TAGS = + ImmutableList.of("source", "host", "\"source\"", "\"host\""); + + private final Function hostnameResolver; + private final List sourceTags; + private final List sourceTagsJson; + + public SharedGraphiteHostAnnotator( + @Nullable List customSourceTags, + @Nonnull Function hostnameResolver) { + if (customSourceTags == null) { + customSourceTags = ImmutableList.of(); + } + this.hostnameResolver = hostnameResolver; + this.sourceTags = + Streams.concat(DEFAULT_SOURCE_TAGS.stream(), customSourceTags.stream()) + .map(customTag -> customTag + "=") + .collect(Collectors.toList()); + this.sourceTagsJson = + Streams.concat( + DEFAULT_SOURCE_TAGS.subList(2, 4).stream(), + customSourceTags.stream().map(customTag -> "\"" + customTag + "\"")) + .collect(Collectors.toList()); + } + + public String apply(ChannelHandlerContext ctx, String msg) { + return apply(ctx, msg, false); + } + + public String apply(ChannelHandlerContext ctx, String msg, boolean addAsJsonProperty) { + List defaultSourceTags = addAsJsonProperty ? sourceTagsJson : sourceTags; + for (int i = 0; i < defaultSourceTags.size(); i++) { + String tag = defaultSourceTags.get(i); + int strIndex = msg.indexOf(tag); + // if a source tags is found and is followed by a non-whitespace tag value, add without + // change + if (strIndex > -1 + && msg.length() - strIndex - tag.length() > 0 + && msg.charAt(strIndex + tag.length()) > ' ') { + return msg; + } + } + + String sourceValue = "\"" + hostnameResolver.apply(getRemoteAddress(ctx)) + "\""; + + if (addAsJsonProperty) { + return msg.replaceFirst("\\{", "{\"source\":" + sourceValue + ", "); + } else { + return msg + " source=" + sourceValue; + } + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/channel/StatusTrackingHttpObjectAggregator.java b/proxy/src/main/java/com/wavefront/agent/channel/StatusTrackingHttpObjectAggregator.java new file mode 100644 index 000000000..00e2d11a8 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/channel/StatusTrackingHttpObjectAggregator.java @@ -0,0 +1,28 @@ +package com.wavefront.agent.channel; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.HttpMessage; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpRequest; + +/** + * A {@link HttpObjectAggregator} that correctly tracks HTTP 413 returned for incoming payloads that + * are too large. + * + * @author vasily@wavefront.com + */ +public class StatusTrackingHttpObjectAggregator extends HttpObjectAggregator { + + public StatusTrackingHttpObjectAggregator(int maxContentLength) { + super(maxContentLength); + } + + @Override + protected void handleOversizedMessage(ChannelHandlerContext ctx, HttpMessage oversized) + throws Exception { + if (oversized instanceof HttpRequest) { + ChannelUtils.getHttpStatusCounter(ctx, 413).inc(); + } + super.handleOversizedMessage(ctx, oversized); + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/config/Categories.java b/proxy/src/main/java/com/wavefront/agent/config/Categories.java new file mode 100644 index 000000000..a0ea48d86 --- /dev/null +++ b/proxy/src/main/java/com/wavefront/agent/config/Categories.java @@ -0,0 +1,29 @@ +package com.wavefront.agent.config; + +import com.fasterxml.jackson.annotation.JsonValue; + +public enum Categories { + GENERAL("General", 1), + INPUT("Input", 2), + BUFFER("Buffering", 3), + OUTPUT("Output", 4), + TRACE("Trace", 5), + NA("Others", 9999); // for hided options + + private final String value; + private int order; + + Categories(String value, int order) { + this.value = value; + this.order = order; + } + + @JsonValue + public String getValue() { + return value; + } + + public int getOrder() { + return order; + } +} diff --git a/proxy/src/main/java/com/wavefront/agent/config/Configuration.java b/proxy/src/main/java/com/wavefront/agent/config/Configuration.java index 4d5811ba7..f0ad85088 100644 --- a/proxy/src/main/java/com/wavefront/agent/config/Configuration.java +++ b/proxy/src/main/java/com/wavefront/agent/config/Configuration.java @@ -3,9 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -/** - * @author Mori Bellamy (mori@wavefront.com) - */ +/** @author Mori Bellamy (mori@wavefront.com) */ public abstract class Configuration { protected void ensure(boolean b, String message) throws ConfigurationException { if (!b) { @@ -15,14 +13,14 @@ protected void ensure(boolean b, String message) throws ConfigurationException { public abstract void verifyAndInit() throws ConfigurationException; - private static ObjectMapper objectMapper = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @Override public String toString() { try { - return objectMapper.writeValueAsString(this); + return OBJECT_MAPPER.writeValueAsString(this); } catch (JsonProcessingException e) { - return super.toString(); + throw new RuntimeException(e); } } diff --git a/proxy/src/main/java/com/wavefront/agent/config/ConfigurationException.java b/proxy/src/main/java/com/wavefront/agent/config/ConfigurationException.java index 59e76c533..23ffa8422 100644 --- a/proxy/src/main/java/com/wavefront/agent/config/ConfigurationException.java +++ b/proxy/src/main/java/com/wavefront/agent/config/ConfigurationException.java @@ -1,8 +1,6 @@ package com.wavefront.agent.config; -/** - * @author Mori Bellamy (mori@wavefront.com) - */ +/** @author Mori Bellamy (mori@wavefront.com) */ public class ConfigurationException extends Exception { public ConfigurationException(String message) { super(message); diff --git a/proxy/src/main/java/com/wavefront/agent/config/LogsIngestionConfig.java b/proxy/src/main/java/com/wavefront/agent/config/LogsIngestionConfig.java index 6715d3187..e8479f175 100644 --- a/proxy/src/main/java/com/wavefront/agent/config/LogsIngestionConfig.java +++ b/proxy/src/main/java/com/wavefront/agent/config/LogsIngestionConfig.java @@ -1,42 +1,35 @@ package com.wavefront.agent.config; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; - import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import com.wavefront.agent.logsharvesting.FlushProcessor; import com.wavefront.agent.logsharvesting.FlushProcessorContext; import com.yammer.metrics.core.Histogram; import com.yammer.metrics.core.MetricName; - -import org.apache.commons.io.IOUtils; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintWriter; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; -import oi.thekraken.grok.api.Grok; -import oi.thekraken.grok.api.exception.GrokException; - /** - * Top level configuration object for ingesting log data into the Wavefront Proxy. To turn on logs ingestion, - * specify 'filebeatPort' and 'logsIngestionConfigFile' in the Wavefront Proxy Config File (typically - * /etc/wavefront/wavefront-proxy/wavefront.conf, or /opt/wavefront/wavefront-proxy/conf/wavefront.conf). + * Top level configuration object for ingesting log data into the Wavefront Proxy. To turn on logs + * ingestion, specify 'filebeatPort' and 'logsIngestionConfigFile' in the Wavefront Proxy Config + * File (typically /etc/wavefront/wavefront-proxy/wavefront.conf, or + * /opt/wavefront/wavefront-proxy/conf/wavefront.conf). * - * Every file with annotated with {@link JsonProperty} is parsed directly from your logsIngestionConfigFile, which is - * YAML. Below is a sample config file which shows the features of direct logs ingestion. The "counters" section - * corresponds to {@link #counters}, likewise for {@link #gauges} and {@link #histograms}. In each of these three - * groups, the pricipal entry is a {@link MetricMatcher}. See the patterns file - * here for - * help defining patterns, also various grok debug tools (e.g. this one, - * or use google) + *

Every file with annotated with {@link JsonProperty} is parsed directly from your + * logsIngestionConfigFile, which is YAML. Below is a sample config file which shows the features of + * direct logs ingestion. The "counters" section corresponds to {@link #counters}, likewise for + * {@link #gauges} and {@link #histograms}. In each of these three groups, the pricipal entry is a + * {@link MetricMatcher}. See the patterns file here + * for help defining patterns, also various grok debug tools (e.g. this one, or use google) * - * All metrics support dynamic naming with %{}. To see exactly what data we send as part of histograms, see - * {@link com.wavefront.agent.logsharvesting.FlushProcessor#processHistogram(MetricName, Histogram, FlushProcessorContext)}. + *

All metrics support dynamic naming with %{}. To see exactly what data we send as part of + * histograms, see {@link FlushProcessor#processHistogram(MetricName, Histogram, + * FlushProcessorContext)}. * * 

  * 
@@ -91,118 +84,82 @@
  *
  * @author Mori Bellamy (mori@wavefront.com)
  */
-
+@SuppressWarnings("CanBeFinal")
 public class LogsIngestionConfig extends Configuration {
   /**
-   * How often metrics are aggregated and sent to wavefront. Histograms are cleared every time they are sent,
-   * counters and gauges are not.
-   */
-  @JsonProperty
-  public Integer aggregationIntervalSeconds = 5;
-  /**
-   * Counters to ingest from incoming log data.
-   */
-  @JsonProperty
-  public List counters = ImmutableList.of();
-  /**
-   * Gauges to ingest from incoming log data.
-   */
-  @JsonProperty
-  public List gauges = ImmutableList.of();
-  /**
-   * Histograms to ingest from incoming log data.
+   * How often metrics are aggregated and sent to wavefront. Histograms are cleared every time they
+   * are sent, counters and gauges are not.
    */
-  @JsonProperty
-  public List histograms = ImmutableList.of();
-  /**
-   * Additional grok patterns to use in pattern matching for the above {@link MetricMatcher}s.
-   */
-  @JsonProperty
-  public List additionalPatterns = ImmutableList.of();
+  @JsonProperty public Integer aggregationIntervalSeconds = 60;
+
+  /** Counters to ingest from incoming log data. */
+  @JsonProperty public List counters = ImmutableList.of();
+
+  /** Gauges to ingest from incoming log data. */
+  @JsonProperty public List gauges = ImmutableList.of();
+
+  /** Histograms to ingest from incoming log data. */
+  @JsonProperty public List histograms = ImmutableList.of();
+
+  /** Additional grok patterns to use in pattern matching for the above {@link MetricMatcher}s. */
+  @JsonProperty public List additionalPatterns = ImmutableList.of();
+
   /**
-   * Metrics are cleared from memory (and so their aggregation state is lost) if a metric is not updated
-   * within this many milliseconds.
+   * Metrics are cleared from memory (and so their aggregation state is lost) if a metric is not
+   * updated within this many milliseconds. Applicable only if useDeltaCounters = false. Default:
+   * 3600000 (1 hour).
    */
-  @JsonProperty
-  public long expiryMillis = TimeUnit.HOURS.toMillis(1);
+  @JsonProperty public long expiryMillis = TimeUnit.HOURS.toMillis(1);
+
   /**
    * If true, use {@link com.yammer.metrics.core.WavefrontHistogram}s rather than {@link
-   * com.yammer.metrics.core.Histogram}s. Histogram ingestion must be enabled on wavefront to use this feature. When
-   * using Yammer histograms, the data is exploded into constituent metrics. See {@link
-   * com.wavefront.agent.logsharvesting.FlushProcessor#processHistogram(MetricName, Histogram, FlushProcessorContext)}.
+   * com.yammer.metrics.core.Histogram}s. Histogram ingestion must be enabled on wavefront to use
+   * this feature. When using Yammer histograms, the data is exploded into constituent metrics. See
+   * {@link FlushProcessor#processHistogram(MetricName, Histogram, FlushProcessorContext)}.
    */
-  @JsonProperty
-  public boolean useWavefrontHistograms = false;
+  @JsonProperty public boolean useWavefrontHistograms = false;
 
   /**
-   * If true (default), simulate Yammer histogram behavior (report all stats as zeroes when histogram is empty).
-   * Otherwise, only .count is reported with a zero value.
+   * If true (default), simulate Yammer histogram behavior (report all stats as zeroes when
+   * histogram is empty). Otherwise, only .count is reported with a zero value.
    */
-  @JsonProperty
-  public boolean reportEmptyHistogramStats = true;
+  @JsonProperty public boolean reportEmptyHistogramStats = true;
 
   /**
-   * How often to check this config file for updates.
+   * If true, use delta counters instead of regular counters to prevent metric collisions when
+   * multiple proxies are behind a load balancer. Default: true
    */
-  @JsonProperty
-  public int configReloadIntervalSeconds = 5;
-
-  private String patternsFile = null;
-  private Object patternsFileLock = new Object();
+  @JsonProperty public boolean useDeltaCounters = true;
 
-  /**
-   * @return The path to a temporary file (on disk) containing grok patterns, to be consumed by {@link Grok}.
-   */
-  public String patternsFile() {
-    if (patternsFile != null) return patternsFile;
-    synchronized (patternsFileLock) {
-      if (patternsFile != null) return patternsFile;
-      try {
-        File temp = File.createTempFile("patterns", ".tmp");
-        InputStream patternInputStream = getClass().getClassLoader().getResourceAsStream("patterns/patterns");
-        FileOutputStream fileOutputStream = new FileOutputStream(temp);
-        IOUtils.copy(patternInputStream, fileOutputStream);
-        PrintWriter printWriter = new PrintWriter(fileOutputStream);
-        for (String pattern : additionalPatterns) {
-          printWriter.write("\n" + pattern);
-        }
-        printWriter.close();
-        patternsFile = temp.getAbsolutePath();
-        return patternsFile;
-      } catch (IOException e) {
-        throw Throwables.propagate(e);
-      }
-    }
-  }
+  /** How often to check this config file for updates. */
+  @JsonProperty public int configReloadIntervalSeconds = 5;
 
   @Override
   public void verifyAndInit() throws ConfigurationException {
-    Grok grok = new Grok();
+    Map additionalPatternMap = Maps.newHashMap();
     for (String pattern : additionalPatterns) {
       String[] parts = pattern.split(" ");
       String name = parts[0];
       String regex = String.join(" ", Arrays.copyOfRange(parts, 1, parts.length));
-      try {
-        grok.addPattern(name, regex);
-      } catch (GrokException e) {
-        throw new ConfigurationException("bad grok pattern: " + pattern);
-      }
+      additionalPatternMap.put(name, regex);
     }
     ensure(aggregationIntervalSeconds > 0, "aggregationIntervalSeconds must be positive.");
     for (MetricMatcher p : counters) {
-      p.setPatternsFile(patternsFile());
+      p.setAdditionalPatterns(additionalPatternMap);
       p.verifyAndInit();
     }
     for (MetricMatcher p : gauges) {
-      p.setPatternsFile(patternsFile());
+      p.setAdditionalPatterns(additionalPatternMap);
       p.verifyAndInit();
-      ensure(p.hasCapture(p.getValueLabel()),
+      ensure(
+          p.hasCapture(p.getValueLabel()),
           "Must have a capture with label '" + p.getValueLabel() + "' for this gauge.");
     }
     for (MetricMatcher p : histograms) {
-      p.setPatternsFile(patternsFile());
+      p.setAdditionalPatterns(additionalPatternMap);
       p.verifyAndInit();
-      ensure(p.hasCapture(p.getValueLabel()),
+      ensure(
+          p.hasCapture(p.getValueLabel()),
           "Must have a capture with label '" + p.getValueLabel() + "' for this histogram.");
     }
   }
diff --git a/proxy/src/main/java/com/wavefront/agent/config/MetricMatcher.java b/proxy/src/main/java/com/wavefront/agent/config/MetricMatcher.java
index f6520cc98..752c79cc6 100644
--- a/proxy/src/main/java/com/wavefront/agent/config/MetricMatcher.java
+++ b/proxy/src/main/java/com/wavefront/agent/config/MetricMatcher.java
@@ -1,24 +1,21 @@
 package com.wavefront.agent.config;
 
-import com.google.common.base.Throwables;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.wavefront.data.Validation;
 import com.wavefront.agent.logsharvesting.LogsMessage;
-
-import org.apache.commons.lang3.StringUtils;
-
+import com.wavefront.data.Validation;
+import io.thekraken.grok.api.Grok;
+import io.thekraken.grok.api.Match;
+import io.thekraken.grok.api.exception.GrokException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-
-import oi.thekraken.grok.api.Grok;
-import oi.thekraken.grok.api.Match;
-import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.commons.lang3.StringUtils;
 import wavefront.report.TimeSeries;
 
 /**
@@ -26,58 +23,63 @@
  *
  * @author Mori Bellamy (mori@wavefront.com)
  */
+@SuppressWarnings("CanBeFinal")
 public class MetricMatcher extends Configuration {
   protected static final Logger logger = Logger.getLogger(MetricMatcher.class.getCanonicalName());
   private final Object grokLock = new Object();
+
   /**
    * A Logstash style grok pattern, see
-   * https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html and http://grokdebug.herokuapp.com/.
-   * If a log line matches this pattern, that log line will be transformed into a metric per the other fields
-   * in this object.
+   * https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html and
+   * http://grokdebug.herokuapp.com/. If a log line matches this pattern, that log line will be
+   * transformed into a metric per the other fields in this object.
    */
-  @JsonProperty
-  private String pattern = "";
+  @JsonProperty private String pattern = "";
+
   /**
-   * The metric name for the point we're creating from the current log line. may contain substitutions from
-   * {@link #pattern}. For example, if your pattern is "operation %{WORD:opName} ...",
-   * and your log line is "operation baz ..." then you can use the metric name "operations.%{opName}".
+   * The metric name for the point we're creating from the current log line. may contain
+   * substitutions from {@link #pattern}. For example, if your pattern is "operation %{WORD:opName}
+   * ...", and your log line is "operation baz ..." then you can use the metric name
+   * "operations.%{opName}".
    */
-  @JsonProperty
-  private String metricName = "";
+  @JsonProperty private String metricName = "";
+
   /**
-   * A list of tags for the point you are creating from the logLine. If you don't want any tags, leave empty. For
-   * example, could be ["myDatacenter", "myEnvironment"] Also see {@link #tagValueLabels}.
+   * Override the host name for the point we're creating from the current log line. May contain
+   * substitutions from {@link #pattern}, similar to metricName.
    */
-  @JsonProperty
-  private List tagKeys = ImmutableList.of();
+  @JsonProperty private String hostName = "";
+  /**
+   * A list of tags for the point you are creating from the logLine. If you don't want any tags,
+   * leave empty. For example, could be ["myDatacenter", "myEnvironment"] Also see {@link
+   * #tagValues}.
+   */
+  @JsonProperty private List tagKeys = ImmutableList.of();
   /**
    * Deprecated, use tagValues instead
    *
-   * A parallel array to {@link #tagKeys}. Each entry is a label you defined in {@link #pattern}. For example, you
-   * might use ["datacenter", "env"] if your pattern is
-   * "operation foo in %{WORD:datacenter}:%{WORD:env} succeeded in %{NUMBER:value} milliseconds", and your log line is
+   * 
A parallel array to {@link #tagKeys}. Each entry is a label you defined in {@link #pattern}.
+   * For example, you might use ["datacenter", "env"] if your pattern is "operation foo in
+   * %{WORD:datacenter}:%{WORD:env} succeeded in %{NUMBER:value} milliseconds", and your log line is
    * "operation foo in 2a:prod succeeded in 1234 milliseconds", then you would generate the point
    * "foo.latency 1234 myDataCenter=2a myEnvironment=prod"
    */
-  @Deprecated
-  @JsonProperty
-  private List tagValueLabels = ImmutableList.of();
-  /**
-   * A parallel array to {@link #tagKeys}. Each entry is a string value that will be used as a tag value,
-   * substituting %{...} placeholders with corresponding labels you defined in {@link #pattern}. For example, you
-   * might use ["%{datacenter}", "%{env}-environment", "staticTag"] if your pattern is
-   * "operation foo in %{WORD:datacenter}:%{WORD:env} succeeded in %{NUMBER:value} milliseconds", and your log line is
-   * "operation foo in 2a:prod succeeded in 1234 milliseconds", then you would generate the point
-   * "foo.latency 1234 myDataCenter=2a myEnvironment=prod-environment myStaticValue=staticTag"
-   */
-  @JsonProperty
-  private List tagValues = ImmutableList.of();
+  @Deprecated @JsonProperty private List tagValueLabels = ImmutableList.of();
   /**
-   * The label which is used to parse a telemetry datum from the log line.
+   * A parallel array to {@link #tagKeys}. Each entry is a string value that will be used as a tag
+   * value, substituting %{...} placeholders with corresponding labels you defined in {@link
+   * #pattern}. For example, you might use ["%{datacenter}", "%{env}-environment", "staticTag"] if
+   * your pattern is "operation foo in %{WORD:datacenter}:%{WORD:env} succeeded in %{NUMBER:value}
+   * milliseconds", and your log line is "operation foo in 2a:prod succeeded in 1234 milliseconds",
+   * then you would generate the point "foo.latency 1234 myDataCenter=2a
+   * myEnvironment=prod-environment myStaticValue=staticTag"
    */
-  @JsonProperty
-  private String valueLabel = "value";
+  @JsonProperty private List tagValues = ImmutableList.of();
+  /** The label which is used to parse a telemetry datum from the log line. */
+  @JsonProperty private String valueLabel = "value";
+
   private Grok grok = null;
+  private Map additionalPatterns = Maps.newHashMap();
 
   public String getValueLabel() {
     return valueLabel;
@@ -87,10 +89,8 @@ public String getPattern() {
     return pattern;
   }
 
-  private String patternsFile = null;
-
-  public void setPatternsFile(String patternsFile) {
-    this.patternsFile = patternsFile;
+  public void setAdditionalPatterns(Map additionalPatterns) {
+    this.additionalPatterns = additionalPatterns;
   }
 
   // Singleton grok for this pattern.
@@ -99,11 +99,25 @@ private Grok grok() {
     synchronized (grokLock) {
       if (grok != null) return grok;
       try {
-        grok = Grok.create(patternsFile);
+        grok = new Grok();
+        InputStream patternStream =
+            getClass().getClassLoader().getResourceAsStream("patterns/patterns");
+        if (patternStream != null) {
+          grok.addPatternFromReader(new InputStreamReader(patternStream));
+        }
+        additionalPatterns.forEach(
+            (key, value) -> {
+              try {
+                grok.addPattern(key, value);
+              } catch (GrokException e) {
+                logger.severe("Invalid grok pattern: " + pattern);
+                throw new RuntimeException(e);
+              }
+            });
         grok.compile(pattern);
       } catch (GrokException e) {
         logger.severe("Invalid grok pattern: " + pattern);
-        throw Throwables.propagate(e);
+        throw new RuntimeException(e);
       }
       return grok;
     }
@@ -118,7 +132,8 @@ private static String expandTemplate(String template, Map replac
           placeholders.appendReplacement(result, placeholders.group(0));
         } else {
           if (replacements.get(placeholders.group(1)) != null) {
-            placeholders.appendReplacement(result, (String)replacements.get(placeholders.group(1)));
+            placeholders.appendReplacement(
+                result, (String) replacements.get(placeholders.group(1)));
           } else {
             placeholders.appendReplacement(result, placeholders.group(0));
           }
@@ -133,58 +148,73 @@ private static String expandTemplate(String template, Map replac
   /**
    * Convert the given message to a timeSeries and a telemetry datum.
    *
-   * @param logsMessage     The message to convert.
-   * @param output          The telemetry parsed from the filebeat message.
+   * @param logsMessage The message to convert.
+   * @param output The telemetry parsed from the filebeat message.
    */
-  public TimeSeries timeSeries(LogsMessage logsMessage, Double[] output) throws NumberFormatException {
+  public TimeSeries timeSeries(LogsMessage logsMessage, Double[] output)
+      throws NumberFormatException {
     Match match = grok().match(logsMessage.getLogLine());
     match.captures();
     if (match.getEnd() == 0) return null;
+    Map matches = match.toMap();
     if (output != null) {
-      if (match.toMap().containsKey(valueLabel)) {
-        output[0] = Double.parseDouble((String) match.toMap().get(valueLabel));
+      if (matches.containsKey(valueLabel)) {
+        output[0] = Double.parseDouble((String) matches.get(valueLabel));
       } else {
         output[0] = null;
       }
     }
     TimeSeries.Builder builder = TimeSeries.newBuilder();
-    String dynamicName = expandTemplate(metricName, match.toMap());
-    // Important to use a tree map for tags, since we need a stable ordering for the serialization
+    String dynamicName = expandTemplate(metricName, matches);
+    String sourceName =
+        StringUtils.isBlank(hostName)
+            ? logsMessage.hostOrDefault("parsed-logs")
+            : expandTemplate(hostName, matches);
+    // Important to use a tree map for tags, since we need a stable ordering for the
+    // serialization
     // into the LogsIngester.metricsCache.
     Map tags = Maps.newTreeMap();
     for (int i = 0; i < tagKeys.size(); i++) {
       String tagKey = tagKeys.get(i);
       if (tagValues.size() > 0) {
-        tags.put(tagKey, expandTemplate(tagValues.get(i), match.toMap()));
+        String value = expandTemplate(tagValues.get(i), matches);
+        if (StringUtils.isNotBlank(value)) {
+          tags.put(tagKey, value);
+        }
       } else {
         String tagValueLabel = tagValueLabels.get(i);
-        if (!match.toMap().containsKey(tagValueLabel)) {
+        if (!matches.containsKey(tagValueLabel)) {
           // What happened? We shouldn't have had matchEnd != 0 above...
           logger.severe("Application error: unparsed tag key.");
           continue;
         }
-        String value = (String) match.toMap().get(tagValueLabel);
-        tags.put(tagKey, value);
+        String value = (String) matches.get(tagValueLabel);
+        if (StringUtils.isNotBlank(value)) {
+          tags.put(tagKey, value);
+        }
       }
     }
     builder.setAnnotations(tags);
-    return builder.setMetric(dynamicName).setHost(logsMessage.hostOrDefault("parsed-logs")).build();
+    return builder.setMetric(dynamicName).setHost(sourceName).build();
   }
 
   public boolean hasCapture(String label) {
-    return grok().getNamedRegexCollection().values().contains(label);
+    return grok().getNamedRegexCollection().containsValue(label);
   }
 
-
   @Override
   public void verifyAndInit() throws ConfigurationException {
     ensure(StringUtils.isNotBlank(pattern), "pattern must not be empty.");
     ensure(StringUtils.isNotBlank(metricName), "metric name must not be empty.");
     String fauxMetricName = metricName.replaceAll("%\\{.*\\}", "");
-    ensure(Validation.charactersAreValid(fauxMetricName), "Metric name has illegal characters: " + metricName);
-    ensure(!(tagValues.size() > 0 && tagValueLabels.size() > 0), "tagValues and tagValueLabels can't be used together");
-    ensure(tagKeys.size() == Math.max(tagValueLabels.size(), tagValues.size()),
+    ensure(
+        Validation.charactersAreValid(fauxMetricName),
+        "Metric name has illegal characters: " + metricName);
+    ensure(
+        !(tagValues.size() > 0 && tagValueLabels.size() > 0),
+        "tagValues and tagValueLabels can't be used together");
+    ensure(
+        tagKeys.size() == Math.max(tagValueLabels.size(), tagValues.size()),
         "tagKeys and tagValues/tagValueLabels must be parallel arrays.");
   }
-
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/config/ProxyConfigOption.java b/proxy/src/main/java/com/wavefront/agent/config/ProxyConfigOption.java
new file mode 100644
index 000000000..e207edd8f
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/config/ProxyConfigOption.java
@@ -0,0 +1,19 @@
+package com.wavefront.agent.config;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+@Retention(RUNTIME)
+@Target({FIELD})
+public @interface ProxyConfigOption {
+  Categories category() default Categories.NA;
+
+  SubCategories subCategory() default SubCategories.NA;
+
+  boolean hide() default false;
+
+  boolean secret() default false;
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/config/ReportableConfig.java b/proxy/src/main/java/com/wavefront/agent/config/ReportableConfig.java
index 63c588039..fc6a6cf91 100644
--- a/proxy/src/main/java/com/wavefront/agent/config/ReportableConfig.java
+++ b/proxy/src/main/java/com/wavefront/agent/config/ReportableConfig.java
@@ -1,116 +1,146 @@
 package com.wavefront.agent.config;
 
-import com.wavefront.common.TaggedMetricName;
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Gauge;
 import com.yammer.metrics.core.MetricName;
-
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.Properties;
 import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-
 import javax.annotation.Nullable;
 
 /**
- * Wrapper class to simplify access to .properties file + track values as metrics as they are retrieved
- *
- * @author vasily@wavefront.com
+ * Wrapper class to simplify access to .properties file + track values as metrics as they are
+ * retrieved
  */
-public class ReportableConfig {
+public class ReportableConfig extends Properties {
   private static final Logger logger = Logger.getLogger(ReportableConfig.class.getCanonicalName());
 
-  private Properties prop = new Properties();
+  public ReportableConfig(String fileName) throws IOException {
+    this.load(new FileInputStream(fileName));
+  }
+
+  public ReportableConfig() {}
 
-  public ReportableConfig(InputStream stream) throws IOException {
-    prop.load(stream);
+  /** Returns string value for the property without tracking it as a metric */
+  public String getRawProperty(String key, String defaultValue) {
+    return this.getProperty(key, defaultValue);
   }
 
-  public ReportableConfig(String fileName) throws IOException {
-    prop.load(new FileInputStream(fileName));
+  public int getInteger(String key, Number defaultValue) {
+    return getNumber(key, defaultValue).intValue();
   }
 
-  public ReportableConfig() {
+  public long getLong(String key, Number defaultValue) {
+    return getNumber(key, defaultValue).longValue();
   }
 
-  /**
-   * Returns string value for the property without tracking it as a metric
-   *
-   */
-  public String getRawProperty(String key, String defaultValue) {
-    return prop.getProperty(key, defaultValue);
+  public double getDouble(String key, Number defaultValue) {
+    return getNumber(key, defaultValue).doubleValue();
   }
 
   public Number getNumber(String key, Number defaultValue) {
     return getNumber(key, defaultValue, null, null);
   }
 
-  public Number getNumber(String key, @Nullable Number defaultValue, @Nullable Number clampMinValue,
-                          @Nullable Number clampMaxValue) {
-    String property = prop.getProperty(key);
+  public Number getNumber(
+      String key,
+      @Nullable Number defaultValue,
+      @Nullable Number clampMinValue,
+      @Nullable Number clampMaxValue) {
+    String property = this.getProperty(key);
     if (property == null && defaultValue == null) return null;
-    Long l;
+    double d;
     try {
-      l = property == null ? defaultValue.longValue() : Long.parseLong(property.trim());
+      d = property == null ? defaultValue.doubleValue() : Double.parseDouble(property.trim());
     } catch (NumberFormatException e) {
-      throw new NumberFormatException("Config setting \"" + key + "\": invalid number format \"" + property + "\"");
+      throw new NumberFormatException(
+          "Config setting \"" + key + "\": invalid number format \"" + property + "\"");
     }
-    if (clampMinValue != null && l < clampMinValue.longValue()) {
-      logger.log(Level.WARNING, key + " (" + l + ") is less than " + clampMinValue +
-          ", will default to " + clampMinValue);
-      reportGauge(clampMinValue, new MetricName("config", "", key));
+    if (clampMinValue != null && d < clampMinValue.longValue()) {
+      logger.log(
+          Level.WARNING,
+          key
+              + " ("
+              + d
+              + ") is less than "
+              + clampMinValue
+              + ", will default to "
+              + clampMinValue);
+      //      reportGauge(clampMinValue, new MetricName("config", "", key));
       return clampMinValue;
     }
-    if (clampMaxValue != null && l > clampMaxValue.longValue()) {
-      logger.log(Level.WARNING, key + " (" + l + ") is greater than " + clampMaxValue +
-          ", will default to " + clampMaxValue);
-      reportGauge(clampMaxValue, new MetricName("config", "", key));
+    if (clampMaxValue != null && d > clampMaxValue.longValue()) {
+      logger.log(
+          Level.WARNING,
+          key
+              + " ("
+              + d
+              + ") is greater than "
+              + clampMaxValue
+              + ", will default to "
+              + clampMaxValue);
+      //      reportGauge(clampMaxValue, new MetricName("config", "", key));
       return clampMaxValue;
     }
-    reportGauge(l, new MetricName("config", "", key));
-    return l;
+    //    reportGauge(d, new MetricName("config", "", key));
+    return d;
   }
 
   public String getString(String key, String defaultValue) {
     return getString(key, defaultValue, null);
   }
 
-  public String getString(String key, String defaultValue,
-                          @Nullable Function converter) {
-    String s = prop.getProperty(key, defaultValue);
-    if (s == null || s.trim().isEmpty()) {
-      reportGauge(0, new MetricName("config", "", key));
-    } else {
-      reportGauge(1, new TaggedMetricName("config", key, "value", converter == null ? s : converter.apply(s)));
-    }
+  public String getString(
+      String key, String defaultValue, @Nullable Function converter) {
+    String s = this.getProperty(key, defaultValue);
+    //    if (s == null || s.trim().isEmpty()) {
+    //      reportGauge(0, new MetricName("config", "", key));
+    //    } else {
+    //      reportGauge(
+    //          1,
+    //          new TaggedMetricName("config", key, "value", converter == null ? s :
+    // converter.apply(s)));
+    //    }
     return s;
   }
 
   public Boolean getBoolean(String key, Boolean defaultValue) {
-    Boolean b = Boolean.parseBoolean(prop.getProperty(key, String.valueOf(defaultValue)).trim());
-    reportGauge(b ? 1 : 0, new MetricName("config", "", key));
+    Boolean b = Boolean.parseBoolean(this.getProperty(key, String.valueOf(defaultValue)).trim());
+    //    reportGauge(b ? 1 : 0, new MetricName("config", "", key));
     return b;
   }
 
   public Boolean isDefined(String key) {
-    return prop.getProperty(key) != null;
+    return this.getProperty(key) != null;
+  }
+
+  public static void reportSettingAsGauge(Supplier numberSupplier, String key) {
+    reportGauge(numberSupplier, new MetricName("config", "", key));
   }
 
-  public void reportSettingAsGauge(Number number, String key) {
-    reportGauge(number, new MetricName("config", "", key));
+  public static void reportGauge(Supplier numberSupplier, MetricName metricName) {
+    Metrics.newGauge(
+        metricName,
+        new Gauge() {
+          @Override
+          public Double value() {
+            return numberSupplier.get().doubleValue();
+          }
+        });
   }
 
-  public void reportGauge(Number number, MetricName metricName) {
-    Metrics.newGauge(metricName,
+  public static void reportGauge(Number number, MetricName metricName) {
+    Metrics.newGauge(
+        metricName,
         new Gauge() {
           @Override
           public Double value() {
             return number.doubleValue();
           }
-        }
-    );
+        });
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/config/SubCategories.java b/proxy/src/main/java/com/wavefront/agent/config/SubCategories.java
new file mode 100644
index 000000000..3b8fcccd1
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/config/SubCategories.java
@@ -0,0 +1,47 @@
+package com.wavefront.agent.config;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public enum SubCategories {
+  METRICS("Metrics", 1),
+  LOGS("Logs", 2),
+  HISTO("Histograms", 3),
+  TRACES("Traces", 4),
+  EVENTS("Events", 5),
+  SOURCETAGS("Source Tags", 6),
+  OTHER("Other", 6),
+  MEMORY("Disk buffer", 1),
+  DISK("Disk buffer", 2),
+  GRAPHITE("Graphite", 5),
+  JSON("JSON", 6),
+  DDOG("DataDog", 7),
+  TLS("HTTPS TLS", 1),
+  CORS("CORS", 2),
+  SQS("External SQS", 3),
+  CONF("Configuration", 0),
+  HTTPPROXY("HTTP/S Proxy", 3),
+  NA("Others", 9999),
+  OPENTEL("Open Telemetry", 8),
+  FILEB("Filebeat logs", 10),
+  RAWLOGS("Raw logs", 11),
+  TSDB("OpenTSDB", 12),
+  TRACES_JAEGER("Jaeger", 13),
+  TRACES_ZIPKIN("Zipkin", 14); // for hided options
+
+  private final String value;
+  private int order;
+
+  SubCategories(String value, int order) {
+    this.value = value;
+    this.order = order;
+  }
+
+  @JsonValue
+  public String getValue() {
+    return value;
+  }
+
+  public int getOrder() {
+    return order;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/AbstractDataSubmissionTask.java b/proxy/src/main/java/com/wavefront/agent/data/AbstractDataSubmissionTask.java
new file mode 100644
index 000000000..be644beaf
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/AbstractDataSubmissionTask.java
@@ -0,0 +1,294 @@
+package com.wavefront.agent.data;
+
+import static com.wavefront.common.Utils.isWavefrontResponse;
+import static java.lang.Boolean.TRUE;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Throwables;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.common.TaggedMetricName;
+import com.wavefront.common.logger.MessageDedupingLogger;
+import com.wavefront.data.ReportableEntityType;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.TimerContext;
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import javax.net.ssl.SSLHandshakeException;
+import javax.ws.rs.ProcessingException;
+import javax.ws.rs.core.Response;
+
+/**
+ * A base class for data submission tasks.
+ *
+ * @param  task type
+ * @author vasily@wavefront.com.
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+abstract class AbstractDataSubmissionTask>
+    implements DataSubmissionTask {
+  private static final int MAX_RETRIES = 15;
+  private static final Logger log =
+      new MessageDedupingLogger(
+          Logger.getLogger(AbstractDataSubmissionTask.class.getCanonicalName()), 1000, 1);
+
+  @JsonProperty protected long enqueuedTimeMillis = Long.MAX_VALUE;
+  @JsonProperty protected int attempts = 0;
+  @JsonProperty protected int serverErrors = 0;
+  @JsonProperty protected String handle;
+  @JsonProperty protected ReportableEntityType entityType;
+  @JsonProperty protected Boolean limitRetries = null;
+
+  protected transient Histogram timeSpentInQueue;
+  protected transient Supplier timeProvider;
+  protected transient EntityProperties properties;
+  protected transient TaskQueue backlog;
+
+  AbstractDataSubmissionTask() {}
+
+  /**
+   * @param properties entity-specific wrapper for runtime properties.
+   * @param backlog backing queue.
+   * @param handle port/handle
+   * @param entityType entity type
+   * @param timeProvider time provider (in millis)
+   */
+  AbstractDataSubmissionTask(
+      EntityProperties properties,
+      TaskQueue backlog,
+      String handle,
+      ReportableEntityType entityType,
+      @Nullable Supplier timeProvider) {
+    this.properties = properties;
+    this.backlog = backlog;
+    this.handle = handle;
+    this.entityType = entityType;
+    this.timeProvider = MoreObjects.firstNonNull(timeProvider, System::currentTimeMillis);
+  }
+
+  @Override
+  public long getEnqueuedMillis() {
+    return enqueuedTimeMillis;
+  }
+
+  @Override
+  public ReportableEntityType getEntityType() {
+    return entityType;
+  }
+
+  abstract Response doExecute() throws DataSubmissionException;
+
+  public TaskResult execute() {
+    if (enqueuedTimeMillis < Long.MAX_VALUE) {
+      if (timeSpentInQueue == null) {
+        timeSpentInQueue =
+            Metrics.newHistogram(
+                new TaggedMetricName(
+                    "buffer", "queue-time", "port", handle, "content", entityType.toString()));
+      }
+      timeSpentInQueue.update(timeProvider.get() - enqueuedTimeMillis);
+    }
+    attempts += 1;
+    TimerContext timer =
+        Metrics.newTimer(
+                new MetricName("push." + handle, "", "duration"),
+                TimeUnit.MILLISECONDS,
+                TimeUnit.MINUTES)
+            .time();
+    try (Response response = doExecute()) {
+      Metrics.newCounter(
+              new TaggedMetricName("push", handle + ".http." + response.getStatus() + ".count"))
+          .inc();
+      if (response.getStatus() >= 200 && response.getStatus() < 300) {
+        Metrics.newCounter(new MetricName(entityType + "." + handle, "", "delivered"))
+            .inc(this.weight());
+        return TaskResult.DELIVERED;
+      }
+      switch (response.getStatus()) {
+        case 406:
+        case 429:
+          return handleStatus429();
+        case 401:
+        case 403:
+          log.warning(
+              "["
+                  + handle
+                  + "] HTTP "
+                  + response.getStatus()
+                  + ": "
+                  + "Please verify that \""
+                  + entityType
+                  + "\" is enabled for your account!");
+          return checkStatusAndQueue(QueueingReason.AUTH, false);
+        case 407:
+        case 408:
+          if (isWavefrontResponse(response)) {
+            log.warning(
+                "["
+                    + handle
+                    + "] HTTP "
+                    + response.getStatus()
+                    + " (Unregistered proxy) "
+                    + "received while sending data to Wavefront - please verify that your token is "
+                    + "valid and has Proxy Management permissions!");
+          } else {
+            log.warning(
+                "["
+                    + handle
+                    + "] HTTP "
+                    + response.getStatus()
+                    + " "
+                    + "received while sending data to Wavefront - please verify your network/HTTP proxy"
+                    + " settings!");
+          }
+          return checkStatusAndQueue(QueueingReason.RETRY, false);
+        case 413:
+          splitTask(1, properties.getDataPerBatch())
+              .forEach(
+                  x ->
+                      x.enqueue(
+                          enqueuedTimeMillis == Long.MAX_VALUE ? QueueingReason.SPLIT : null));
+          return TaskResult.PERSISTED_RETRY;
+        default:
+          serverErrors += 1;
+          if (serverErrors > MAX_RETRIES && TRUE.equals(limitRetries)) {
+            log.info(
+                "["
+                    + handle
+                    + "] HTTP "
+                    + response.getStatus()
+                    + " received while sending "
+                    + "data to Wavefront, max retries reached");
+            return TaskResult.DELIVERED;
+          } else {
+            log.info(
+                "["
+                    + handle
+                    + "] HTTP "
+                    + response.getStatus()
+                    + " received while sending "
+                    + "data to Wavefront, retrying");
+            return checkStatusAndQueue(QueueingReason.RETRY, true);
+          }
+      }
+    } catch (DataSubmissionException ex) {
+      if (ex instanceof IgnoreStatusCodeException) {
+        Metrics.newCounter(new TaggedMetricName("push", handle + ".http.404.count")).inc();
+        Metrics.newCounter(new MetricName(entityType + "." + handle, "", "delivered"))
+            .inc(this.weight());
+        return TaskResult.DELIVERED;
+      }
+      throw new RuntimeException("Unhandled DataSubmissionException", ex);
+    } catch (ProcessingException ex) {
+      Throwable rootCause = Throwables.getRootCause(ex);
+      if (rootCause instanceof UnknownHostException) {
+        log.warning(
+            "["
+                + handle
+                + "] Error sending data to Wavefront: Unknown host "
+                + rootCause.getMessage()
+                + ", please check your network!");
+      } else if (rootCause instanceof ConnectException
+          || rootCause instanceof SocketTimeoutException) {
+        log.warning(
+            "["
+                + handle
+                + "] Error sending data to Wavefront: "
+                + rootCause.getMessage()
+                + ", please verify your network/HTTP proxy settings!");
+      } else if (ex.getCause() instanceof SSLHandshakeException) {
+        log.warning(
+            "["
+                + handle
+                + "] Error sending data to Wavefront: "
+                + ex.getCause()
+                + ", please verify that your environment has up-to-date root certificates!");
+      } else {
+        log.warning("[" + handle + "] Error sending data to Wavefront: " + rootCause);
+      }
+      if (log.isLoggable(Level.FINE)) {
+        log.log(Level.FINE, "Full stacktrace: ", ex);
+      }
+      return checkStatusAndQueue(QueueingReason.RETRY, false);
+    } catch (Exception ex) {
+      log.warning(
+          "[" + handle + "] Error sending data to Wavefront: " + Throwables.getRootCause(ex));
+      if (log.isLoggable(Level.FINE)) {
+        log.log(Level.FINE, "Full stacktrace: ", ex);
+      }
+      return checkStatusAndQueue(QueueingReason.RETRY, true);
+    } finally {
+      timer.stop();
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public void enqueue(@Nullable QueueingReason reason) {
+    enqueuedTimeMillis = timeProvider.get();
+    try {
+      backlog.add((T) this);
+      if (reason != null) {
+        Metrics.newCounter(
+                new TaggedMetricName(
+                    entityType + "." + handle, "queued", "reason", reason.toString()))
+            .inc(this.weight());
+      }
+    } catch (IOException e) {
+      Metrics.newCounter(new TaggedMetricName("buffer", "failures", "port", handle)).inc();
+      log.severe(
+          "["
+              + handle
+              + "] CRITICAL (Losing data): WF-1: Error adding task to the queue: "
+              + e.getMessage());
+    }
+  }
+
+  private TaskResult checkStatusAndQueue(QueueingReason reason, boolean requeue) {
+    if (reason == QueueingReason.AUTH) return TaskResult.REMOVED;
+    if (enqueuedTimeMillis == Long.MAX_VALUE) {
+      if (properties.getTaskQueueLevel().isLessThan(TaskQueueLevel.ANY_ERROR)) {
+        return TaskResult.RETRY_LATER;
+      }
+      enqueue(reason);
+      return TaskResult.PERSISTED;
+    }
+    if (requeue) {
+      enqueue(null);
+      return TaskResult.PERSISTED_RETRY;
+    } else {
+      return TaskResult.RETRY_LATER;
+    }
+  }
+
+  protected TaskResult handleStatus429() {
+    if (enqueuedTimeMillis == Long.MAX_VALUE) {
+      if (properties.getTaskQueueLevel().isLessThan(TaskQueueLevel.PUSHBACK)) {
+        return TaskResult.RETRY_LATER;
+      }
+      enqueue(QueueingReason.PUSHBACK);
+      return TaskResult.PERSISTED;
+    }
+    if (properties.isSplitPushWhenRateLimited()) {
+      List splitTasks =
+          splitTask(properties.getMinBatchSplitSize(), properties.getDataPerBatch());
+      if (splitTasks.size() == 1) return TaskResult.RETRY_LATER;
+      splitTasks.forEach(x -> x.enqueue(null));
+      return TaskResult.PERSISTED;
+    }
+    return TaskResult.RETRY_LATER;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/DataSubmissionException.java b/proxy/src/main/java/com/wavefront/agent/data/DataSubmissionException.java
new file mode 100644
index 000000000..5fa2f3e38
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/DataSubmissionException.java
@@ -0,0 +1,12 @@
+package com.wavefront.agent.data;
+
+/**
+ * Exception to bypass standard handling for response status codes.
+ *
+ * @author vasily@wavefront.com
+ */
+public abstract class DataSubmissionException extends Exception {
+  public DataSubmissionException(String message) {
+    super(message);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/DataSubmissionTask.java b/proxy/src/main/java/com/wavefront/agent/data/DataSubmissionTask.java
new file mode 100644
index 000000000..9e4a0a1a4
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/DataSubmissionTask.java
@@ -0,0 +1,61 @@
+package com.wavefront.agent.data;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.wavefront.data.ReportableEntityType;
+import java.io.Serializable;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * A serializable data submission task.
+ *
+ * @param  task type
+ * @author vasily@wavefront.com
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "__CLASS")
+public interface DataSubmissionTask> extends Serializable {
+
+  /**
+   * Returns a task weight.
+   *
+   * @return task weight
+   */
+  int weight();
+
+  /**
+   * Returns task enqueue time in milliseconds.
+   *
+   * @return enqueue time in milliseconds
+   */
+  long getEnqueuedMillis();
+
+  /**
+   * Execute this task
+   *
+   * @return operation result
+   */
+  TaskResult execute();
+
+  /**
+   * Persist task in the queue
+   *
+   * @param reason reason for queueing. used to increment metrics, if specified.
+   */
+  void enqueue(@Nullable QueueingReason reason);
+
+  /**
+   * Returns entity type handled.
+   *
+   * @return entity type
+   */
+  ReportableEntityType getEntityType();
+
+  /**
+   * Split the task into smaller tasks.
+   *
+   * @param minSplitSize Don't split the task if its weight is smaller than this number.
+   * @param maxSplitSize Split tasks size cap.
+   * @return tasks
+   */
+  List splitTask(int minSplitSize, int maxSplitSize);
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/EntityProperties.java b/proxy/src/main/java/com/wavefront/agent/data/EntityProperties.java
new file mode 100644
index 000000000..6d60f81cc
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/EntityProperties.java
@@ -0,0 +1,158 @@
+package com.wavefront.agent.data;
+
+import com.google.common.util.concurrent.RecyclableRateLimiter;
+import javax.annotation.Nullable;
+
+/**
+ * Unified interface for dynamic entity-specific dynamic properties, that may change at runtime
+ *
+ * @author vasily@wavefront.com
+ */
+public interface EntityProperties {
+  // what we consider "unlimited"
+  int NO_RATE_LIMIT = 10_000_000;
+  int NO_RATE_LIMIT_BYTES = 1_000_000_000;
+
+  // default values for dynamic properties
+  boolean DEFAULT_SPLIT_PUSH_WHEN_RATE_LIMITED = false;
+  double DEFAULT_RETRY_BACKOFF_BASE_SECONDS = 2.0d;
+  int DEFAULT_FLUSH_INTERVAL = 1000;
+  int DEFAULT_MAX_BURST_SECONDS = 10;
+  int DEFAULT_BATCH_SIZE = 40000;
+  int DEFAULT_BATCH_SIZE_HISTOGRAMS = 10000;
+  int DEFAULT_BATCH_SIZE_SOURCE_TAGS = 50;
+  int DEFAULT_BATCH_SIZE_SPANS = 5000;
+  int DEFAULT_BATCH_SIZE_SPAN_LOGS = 1000;
+  int DEFAULT_BATCH_SIZE_EVENTS = 50;
+  int DEFAULT_MIN_SPLIT_BATCH_SIZE = 100;
+  int DEFAULT_FLUSH_THREADS_SOURCE_TAGS = 2;
+  int DEFAULT_FLUSH_THREADS_EVENTS = 2;
+
+  // the maximum batch size for logs is set between 1 and 5 mb, with a default of 4mb
+  int DEFAULT_MIN_SPLIT_BATCH_SIZE_LOGS_PAYLOAD = 1024 * 1024;
+  int DEFAULT_BATCH_SIZE_LOGS_PAYLOAD = 4 * 1024 * 1024;
+  int MAX_BATCH_SIZE_LOGS_PAYLOAD = 5 * 1024 * 1024;
+
+  /**
+   * Get initially configured batch size.
+   *
+   * @return batch size
+   */
+  int getDataPerBatchOriginal();
+
+  /**
+   * Whether we should split batches into smaller ones after getting HTTP 406 response from server.
+   *
+   * @return true if we should split on pushback
+   */
+  boolean isSplitPushWhenRateLimited();
+
+  /**
+   * Get initially configured rate limit (per second).
+   *
+   * @return rate limit
+   */
+  double getRateLimit();
+
+  /**
+   * Get max number of burst seconds to allow when rate limiting to smooth out uneven traffic.
+   *
+   * @return number of seconds
+   */
+  int getRateLimitMaxBurstSeconds();
+
+  /**
+   * Get specific {@link RecyclableRateLimiter} instance.
+   *
+   * @return rate limiter
+   */
+  RecyclableRateLimiter getRateLimiter();
+
+  /**
+   * Get the number of worker threads.
+   *
+   * @return number of threads
+   */
+  int getFlushThreads();
+
+  /**
+   * Get interval between batches (in milliseconds)
+   *
+   * @return interval between batches
+   */
+  int getPushFlushInterval();
+
+  /**
+   * Get the maximum allowed number of items per single flush.
+   *
+   * @return batch size
+   */
+  int getDataPerBatch();
+
+  /**
+   * Sets the maximum allowed number of items per single flush.
+   *
+   * @param dataPerBatch batch size. if null is provided, reverts to originally configured value.
+   */
+  void setDataPerBatch(@Nullable Integer dataPerBatch);
+
+  /**
+   * Do not split the batch if its size is less than this value. Only applicable when {@link
+   * #isSplitPushWhenRateLimited()} is true.
+   *
+   * @return smallest allowed batch size
+   */
+  int getMinBatchSplitSize();
+
+  /**
+   * Max number of items that can stay in memory buffers before spooling to disk. Defaults to 16 *
+   * {@link #getDataPerBatch()}, minimum size: {@link #getDataPerBatch()}. Setting this value lower
+   * than default reduces memory usage, but will force the proxy to spool to disk more frequently if
+   * you have points arriving at the proxy in short bursts, and/or your network latency is on the
+   * higher side.
+   *
+   * @return memory buffer limit
+   */
+  int getMemoryBufferLimit();
+
+  /**
+   * Get current queueing behavior - defines conditions that trigger queueing.
+   *
+   * @return queueing behavior level
+   */
+  TaskQueueLevel getTaskQueueLevel();
+
+  /**
+   * Checks whether data flow for this entity type is disabled.
+   *
+   * @return true if data flow is disabled
+   */
+  boolean isFeatureDisabled();
+
+  /**
+   * Sets the flag value for "feature disabled" flag.
+   *
+   * @param featureDisabled if "true", data flow for this entity type is disabled.
+   */
+  void setFeatureDisabled(boolean featureDisabled);
+
+  /**
+   * Get aggregated backlog size across all ports for this entity type.
+   *
+   * @return backlog size
+   */
+  int getTotalBacklogSize();
+
+  /** Updates backlog size for specific port. */
+  void reportBacklogSize(String handle, int backlogSize);
+
+  /**
+   * Get aggregated received rate across all ports for this entity type.
+   *
+   * @return received rate
+   */
+  long getTotalReceivedRate();
+
+  /** Updates received rate for specific port. */
+  void reportReceivedRate(String handle, long receivedRate);
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/EntityPropertiesFactory.java b/proxy/src/main/java/com/wavefront/agent/data/EntityPropertiesFactory.java
new file mode 100644
index 000000000..2597000e7
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/EntityPropertiesFactory.java
@@ -0,0 +1,26 @@
+package com.wavefront.agent.data;
+
+import com.wavefront.data.ReportableEntityType;
+
+/**
+ * Generates entity-specific wrappers for dynamic proxy settings.
+ *
+ * @author vasily@wavefront.com
+ */
+public interface EntityPropertiesFactory {
+
+  /**
+   * Get an entity-specific wrapper for proxy runtime properties.
+   *
+   * @param entityType entity type to get wrapper for
+   * @return EntityProperties wrapper
+   */
+  EntityProperties get(ReportableEntityType entityType);
+
+  /**
+   * Returns a container with properties shared across all entity types
+   *
+   * @return global properties container
+   */
+  GlobalProperties getGlobalProperties();
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/EntityPropertiesFactoryImpl.java b/proxy/src/main/java/com/wavefront/agent/data/EntityPropertiesFactoryImpl.java
new file mode 100644
index 000000000..245504842
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/EntityPropertiesFactoryImpl.java
@@ -0,0 +1,395 @@
+package com.wavefront.agent.data;
+
+import static com.wavefront.agent.config.ReportableConfig.reportSettingAsGauge;
+import static org.apache.commons.lang3.ObjectUtils.firstNonNull;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.RecyclableRateLimiter;
+import com.google.common.util.concurrent.RecyclableRateLimiterImpl;
+import com.google.common.util.concurrent.RecyclableRateLimiterWithMetrics;
+import com.wavefront.agent.ProxyConfig;
+import com.wavefront.data.ReportableEntityType;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import javax.annotation.Nullable;
+
+/**
+ * Generates entity-specific wrappers for dynamic proxy settings.
+ *
+ * @author vasily@wavefront.com
+ */
+public class EntityPropertiesFactoryImpl implements EntityPropertiesFactory {
+
+  private final Map wrappers;
+  private final GlobalProperties global;
+
+  /** @param proxyConfig proxy settings container */
+  public EntityPropertiesFactoryImpl(ProxyConfig proxyConfig) {
+    global = new GlobalPropertiesImpl(proxyConfig);
+    EntityProperties pointProperties = new PointsProperties(proxyConfig);
+    wrappers =
+        ImmutableMap.builder()
+            .put(ReportableEntityType.POINT, pointProperties)
+            .put(ReportableEntityType.DELTA_COUNTER, pointProperties)
+            .put(ReportableEntityType.HISTOGRAM, new HistogramsProperties(proxyConfig))
+            .put(ReportableEntityType.SOURCE_TAG, new SourceTagsProperties(proxyConfig))
+            .put(ReportableEntityType.TRACE, new SpansProperties(proxyConfig))
+            .put(ReportableEntityType.TRACE_SPAN_LOGS, new SpanLogsProperties(proxyConfig))
+            .put(ReportableEntityType.EVENT, new EventsProperties(proxyConfig))
+            .put(ReportableEntityType.LOGS, new LogsProperties(proxyConfig))
+            .build();
+  }
+
+  @Override
+  public EntityProperties get(ReportableEntityType entityType) {
+    return wrappers.get(entityType);
+  }
+
+  @Override
+  public GlobalProperties getGlobalProperties() {
+    return global;
+  }
+
+  /** Common base for all wrappers (to avoid code duplication) */
+  private abstract static class AbstractEntityProperties implements EntityProperties {
+    private Integer dataPerBatch = null;
+    protected final ProxyConfig wrapped;
+    private final RecyclableRateLimiter rateLimiter;
+    private final LoadingCache backlogSizeCache =
+        Caffeine.newBuilder()
+            .expireAfterAccess(10, TimeUnit.SECONDS)
+            .build(x -> new AtomicInteger());
+    private final LoadingCache receivedRateCache =
+        Caffeine.newBuilder().expireAfterAccess(10, TimeUnit.SECONDS).build(x -> new AtomicLong());
+
+    public AbstractEntityProperties(ProxyConfig wrapped) {
+      this.wrapped = wrapped;
+      this.rateLimiter =
+          getRateLimit() > 0
+              ? new RecyclableRateLimiterWithMetrics(
+                  RecyclableRateLimiterImpl.create(getRateLimit(), getRateLimitMaxBurstSeconds()),
+                  getRateLimiterName())
+              : null;
+
+      reportSettingAsGauge(this::getPushFlushInterval, "dynamic.pushFlushInterval");
+    }
+
+    @Override
+    public int getDataPerBatch() {
+      return firstNonNull(dataPerBatch, getDataPerBatchOriginal());
+    }
+
+    @Override
+    public void setDataPerBatch(@Nullable Integer dataPerBatch) {
+      this.dataPerBatch = dataPerBatch;
+    }
+
+    @Override
+    public boolean isSplitPushWhenRateLimited() {
+      return wrapped.isSplitPushWhenRateLimited();
+    }
+
+    @Override
+    public int getRateLimitMaxBurstSeconds() {
+      return wrapped.getPushRateLimitMaxBurstSeconds();
+    }
+
+    @Override
+    public RecyclableRateLimiter getRateLimiter() {
+      return rateLimiter;
+    }
+
+    protected abstract String getRateLimiterName();
+
+    @Override
+    public int getFlushThreads() {
+      return wrapped.getFlushThreads();
+    }
+
+    @Override
+    public int getPushFlushInterval() {
+      return wrapped.getPushFlushInterval();
+    }
+
+    @Override
+    public int getMinBatchSplitSize() {
+      return DEFAULT_MIN_SPLIT_BATCH_SIZE;
+    }
+
+    @Override
+    public int getMemoryBufferLimit() {
+      return wrapped.getPushMemoryBufferLimit();
+    }
+
+    @Override
+    public TaskQueueLevel getTaskQueueLevel() {
+      return wrapped.getTaskQueueLevel();
+    }
+
+    @Override
+    public int getTotalBacklogSize() {
+      return backlogSizeCache.asMap().values().stream().mapToInt(AtomicInteger::get).sum();
+    }
+
+    @Override
+    public void reportBacklogSize(String handle, int backlogSize) {
+      backlogSizeCache.get(handle).set(backlogSize);
+    }
+
+    @Override
+    public long getTotalReceivedRate() {
+      return receivedRateCache.asMap().values().stream().mapToLong(AtomicLong::get).sum();
+    }
+
+    @Override
+    public void reportReceivedRate(String handle, long receivedRate) {
+      receivedRateCache.get(handle).set(receivedRate);
+    }
+  }
+
+  /** Base class for entity types that do not require separate subscriptions. */
+  private abstract static class CoreEntityProperties extends AbstractEntityProperties {
+    public CoreEntityProperties(ProxyConfig wrapped) {
+      super(wrapped);
+    }
+
+    @Override
+    public boolean isFeatureDisabled() {
+      return false;
+    }
+
+    @Override
+    public void setFeatureDisabled(boolean featureDisabledFlag) {
+      throw new UnsupportedOperationException("Can't disable this feature");
+    }
+  }
+
+  /**
+   * Base class for entity types that do require a separate subscription and can be controlled
+   * remotely.
+   */
+  private abstract static class SubscriptionBasedEntityProperties extends AbstractEntityProperties {
+    private boolean featureDisabled = false;
+
+    public SubscriptionBasedEntityProperties(ProxyConfig wrapped) {
+      super(wrapped);
+    }
+
+    @Override
+    public boolean isFeatureDisabled() {
+      return featureDisabled;
+    }
+
+    @Override
+    public void setFeatureDisabled(boolean featureDisabledFlag) {
+      this.featureDisabled = featureDisabledFlag;
+    }
+  }
+
+  /** Runtime properties wrapper for points */
+  private static final class PointsProperties extends CoreEntityProperties {
+    public PointsProperties(ProxyConfig wrapped) {
+      super(wrapped);
+      reportSettingAsGauge(this::getDataPerBatch, "dynamic.pushFlushMaxPoints");
+      reportSettingAsGauge(this::getMemoryBufferLimit, "dynamic.pushMemoryBufferLimit");
+    }
+
+    @Override
+    protected String getRateLimiterName() {
+      return "limiter";
+    }
+
+    @Override
+    public int getDataPerBatchOriginal() {
+      return wrapped.getPushFlushMaxPoints();
+    }
+
+    @Override
+    public double getRateLimit() {
+      return wrapped.getPushRateLimit();
+    }
+  }
+
+  /** Runtime properties wrapper for histograms */
+  private static final class HistogramsProperties extends SubscriptionBasedEntityProperties {
+    public HistogramsProperties(ProxyConfig wrapped) {
+      super(wrapped);
+      reportSettingAsGauge(this::getDataPerBatch, "dynamic.pushFlushMaxHistograms");
+      reportSettingAsGauge(this::getMemoryBufferLimit, "dynamic.pushMemoryBufferLimit");
+    }
+
+    @Override
+    protected String getRateLimiterName() {
+      return "limiter.histograms";
+    }
+
+    @Override
+    public int getDataPerBatchOriginal() {
+      return wrapped.getPushFlushMaxHistograms();
+    }
+
+    @Override
+    public double getRateLimit() {
+      return wrapped.getPushRateLimitHistograms();
+    }
+  }
+
+  /** Runtime properties wrapper for source tags */
+  private static final class SourceTagsProperties extends CoreEntityProperties {
+    public SourceTagsProperties(ProxyConfig wrapped) {
+      super(wrapped);
+      reportSettingAsGauge(this::getDataPerBatch, "dynamic.pushFlushMaxSourceTags");
+      reportSettingAsGauge(this::getMemoryBufferLimit, "dynamic.pushMemoryBufferLimitSourceTags");
+    }
+
+    @Override
+    protected String getRateLimiterName() {
+      return "limiter.sourceTags";
+    }
+
+    @Override
+    public int getDataPerBatchOriginal() {
+      return wrapped.getPushFlushMaxSourceTags();
+    }
+
+    @Override
+    public double getRateLimit() {
+      return wrapped.getPushRateLimitSourceTags();
+    }
+
+    @Override
+    public int getMemoryBufferLimit() {
+      return 16 * wrapped.getPushFlushMaxSourceTags();
+    }
+
+    @Override
+    public int getFlushThreads() {
+      return wrapped.getFlushThreadsSourceTags();
+    }
+  }
+
+  /** Runtime properties wrapper for spans */
+  private static final class SpansProperties extends SubscriptionBasedEntityProperties {
+    public SpansProperties(ProxyConfig wrapped) {
+      super(wrapped);
+      reportSettingAsGauge(this::getDataPerBatch, "dynamic.pushFlushMaxSpans");
+      reportSettingAsGauge(this::getMemoryBufferLimit, "dynamic.pushMemoryBufferLimit");
+    }
+
+    @Override
+    protected String getRateLimiterName() {
+      return "limiter.spans";
+    }
+
+    @Override
+    public int getDataPerBatchOriginal() {
+      return wrapped.getPushFlushMaxSpans();
+    }
+
+    @Override
+    public double getRateLimit() {
+      return wrapped.getPushRateLimitSpans();
+    }
+  }
+
+  /** Runtime properties wrapper for span logs */
+  private static final class SpanLogsProperties extends SubscriptionBasedEntityProperties {
+    public SpanLogsProperties(ProxyConfig wrapped) {
+      super(wrapped);
+      reportSettingAsGauge(this::getDataPerBatch, "dynamic.pushFlushMaxSpanLogs");
+      reportSettingAsGauge(this::getMemoryBufferLimit, "dynamic.pushMemoryBufferLimit");
+    }
+
+    @Override
+    protected String getRateLimiterName() {
+      return "limiter.spanLogs";
+    }
+
+    @Override
+    public int getDataPerBatchOriginal() {
+      return wrapped.getPushFlushMaxSpanLogs();
+    }
+
+    @Override
+    public double getRateLimit() {
+      return wrapped.getPushRateLimitSpanLogs();
+    }
+  }
+
+  /** Runtime properties wrapper for events */
+  private static final class EventsProperties extends CoreEntityProperties {
+    public EventsProperties(ProxyConfig wrapped) {
+      super(wrapped);
+      reportSettingAsGauge(this::getDataPerBatch, "dynamic.pushFlushMaxEvents");
+      reportSettingAsGauge(this::getMemoryBufferLimit, "dynamic.pushMemoryBufferLimitEvents");
+    }
+
+    @Override
+    protected String getRateLimiterName() {
+      return "limiter.events";
+    }
+
+    @Override
+    public int getDataPerBatchOriginal() {
+      return wrapped.getPushFlushMaxEvents();
+    }
+
+    @Override
+    public double getRateLimit() {
+      return wrapped.getPushRateLimitEvents();
+    }
+
+    @Override
+    public int getMemoryBufferLimit() {
+      return 16 * wrapped.getPushFlushMaxEvents();
+    }
+
+    @Override
+    public int getFlushThreads() {
+      return wrapped.getFlushThreadsEvents();
+    }
+  }
+
+  /** Runtime properties wrapper for logs */
+  private static final class LogsProperties extends SubscriptionBasedEntityProperties {
+    public LogsProperties(ProxyConfig wrapped) {
+      super(wrapped);
+      reportSettingAsGauge(this::getDataPerBatch, "dynamic.pushFlushMaxLogs");
+      reportSettingAsGauge(this::getMemoryBufferLimit, "dynamic.pushMemoryBufferLimitLogs");
+    }
+
+    @Override
+    protected String getRateLimiterName() {
+      return "limiter.logs";
+    }
+
+    @Override
+    public int getDataPerBatchOriginal() {
+      return wrapped.getPushFlushMaxLogs();
+    }
+
+    @Override
+    public int getMemoryBufferLimit() {
+      return wrapped.getPushMemoryBufferLimitLogs();
+    }
+
+    @Override
+    public double getRateLimit() {
+      return wrapped.getPushRateLimitLogs();
+    }
+
+    @Override
+    public int getFlushThreads() {
+      return wrapped.getFlushThreadsLogs();
+    }
+
+    @Override
+    public int getPushFlushInterval() {
+      return wrapped.getPushFlushIntervalLogs();
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/EventDataSubmissionTask.java b/proxy/src/main/java/com/wavefront/agent/data/EventDataSubmissionTask.java
new file mode 100644
index 000000000..5a9c13a32
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/EventDataSubmissionTask.java
@@ -0,0 +1,105 @@
+package com.wavefront.agent.data;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.google.common.collect.ImmutableList;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.api.EventAPI;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.dto.Event;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.function.Supplier;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.ws.rs.core.Response;
+
+/**
+ * A {@link DataSubmissionTask} that handles event payloads.
+ *
+ * @author vasily@wavefront.com
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "__CLASS")
+public class EventDataSubmissionTask extends AbstractDataSubmissionTask {
+  private transient EventAPI api;
+  private transient UUID proxyId;
+
+  @JsonProperty private List events;
+
+  @SuppressWarnings("unused")
+  EventDataSubmissionTask() {}
+
+  /**
+   * @param api API endpoint.
+   * @param proxyId Proxy identifier. Used to authenticate proxy with the API.
+   * @param properties entity-specific wrapper over mutable proxy settings' container.
+   * @param backlog task queue.
+   * @param handle Handle (usually port number) of the pipeline where the data came from.
+   * @param events Data payload.
+   * @param timeProvider Time provider (in millis).
+   */
+  public EventDataSubmissionTask(
+      EventAPI api,
+      UUID proxyId,
+      EntityProperties properties,
+      TaskQueue backlog,
+      String handle,
+      @Nonnull List events,
+      @Nullable Supplier timeProvider) {
+    super(properties, backlog, handle, ReportableEntityType.EVENT, timeProvider);
+    this.api = api;
+    this.proxyId = proxyId;
+    this.events = new ArrayList<>(events);
+  }
+
+  @Override
+  public Response doExecute() {
+    return api.proxyEvents(proxyId, events);
+  }
+
+  public List splitTask(int minSplitSize, int maxSplitSize) {
+    if (events.size() > Math.max(1, minSplitSize)) {
+      List result = new ArrayList<>();
+      int stride = Math.min(maxSplitSize, (int) Math.ceil((float) events.size() / 2.0));
+      int endingIndex = 0;
+      for (int startingIndex = 0; endingIndex < events.size() - 1; startingIndex += stride) {
+        endingIndex = Math.min(events.size(), startingIndex + stride) - 1;
+        result.add(
+            new EventDataSubmissionTask(
+                api,
+                proxyId,
+                properties,
+                backlog,
+                handle,
+                events.subList(startingIndex, endingIndex + 1),
+                timeProvider));
+      }
+      return result;
+    }
+    return ImmutableList.of(this);
+  }
+
+  public List payload() {
+    return events;
+  }
+
+  @Override
+  public int weight() {
+    return events.size();
+  }
+
+  public void injectMembers(
+      EventAPI api,
+      UUID proxyId,
+      EntityProperties properties,
+      TaskQueue backlog) {
+    this.api = api;
+    this.proxyId = proxyId;
+    this.properties = properties;
+    this.backlog = backlog;
+    this.timeProvider = System::currentTimeMillis;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/GlobalProperties.java b/proxy/src/main/java/com/wavefront/agent/data/GlobalProperties.java
new file mode 100644
index 000000000..05ef682b7
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/GlobalProperties.java
@@ -0,0 +1,87 @@
+package com.wavefront.agent.data;
+
+import com.wavefront.api.agent.SpanSamplingPolicy;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * Unified interface for non-entity specific dynamic properties, that may change at runtime.
+ *
+ * @author vasily@wavefront.com
+ */
+public interface GlobalProperties {
+  /**
+   * Get base in seconds for retry thread exponential backoff.
+   *
+   * @return exponential backoff base value
+   */
+  double getRetryBackoffBaseSeconds();
+
+  /**
+   * Sets base in seconds for retry thread exponential backoff.
+   *
+   * @param retryBackoffBaseSeconds new value for exponential backoff base value. if null is
+   *     provided, reverts to originally configured value.
+   */
+  void setRetryBackoffBaseSeconds(@Nullable Double retryBackoffBaseSeconds);
+
+  /**
+   * Get histogram storage accuracy, as specified by the back-end.
+   *
+   * @return histogram storage accuracy
+   */
+  short getHistogramStorageAccuracy();
+
+  /**
+   * Sets histogram storage accuracy.
+   *
+   * @param histogramStorageAccuracy storage accuracy
+   */
+  void setHistogramStorageAccuracy(short histogramStorageAccuracy);
+
+  /**
+   * Get the sampling rate for tracing spans.
+   *
+   * @return sampling rate for tracing spans.
+   */
+  double getTraceSamplingRate();
+
+  /**
+   * Sets the sampling rate for tracing spans.
+   *
+   * @param traceSamplingRate sampling rate for tracing spans
+   */
+  void setTraceSamplingRate(@Nullable Double traceSamplingRate);
+
+  /**
+   * Get the maximum acceptable duration between now and the end of a span to be accepted for
+   * reporting to Wavefront, beyond which they are dropped.
+   *
+   * @return delay threshold for dropping spans in minutes.
+   */
+  @Nullable
+  Integer getDropSpansDelayedMinutes();
+
+  /**
+   * Set the maximum acceptable duration between now and the end of a span to be accepted for
+   * reporting to Wavefront, beyond which they are dropped.
+   *
+   * @param dropSpansDelayedMinutes delay threshold for dropping spans in minutes.
+   */
+  void setDropSpansDelayedMinutes(@Nullable Integer dropSpansDelayedMinutes);
+
+  /**
+   * Get active span sampling policies for policy based sampling.
+   *
+   * @return list of span sampling policies.
+   */
+  @Nullable
+  List getActiveSpanSamplingPolicies();
+
+  /**
+   * Set active span sampling policies for policy based sampling.
+   *
+   * @param activeSpanSamplingPolicies list of span sampling policies.
+   */
+  void setActiveSpanSamplingPolicies(@Nullable List activeSpanSamplingPolicies);
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/GlobalPropertiesImpl.java b/proxy/src/main/java/com/wavefront/agent/data/GlobalPropertiesImpl.java
new file mode 100644
index 000000000..2c2bf8185
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/GlobalPropertiesImpl.java
@@ -0,0 +1,83 @@
+package com.wavefront.agent.data;
+
+import static com.wavefront.agent.config.ReportableConfig.reportSettingAsGauge;
+import static org.apache.commons.lang3.ObjectUtils.firstNonNull;
+
+import com.wavefront.agent.ProxyConfig;
+import com.wavefront.api.agent.SpanSamplingPolicy;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * Dynamic non-entity specific properties, that may change at runtime.
+ *
+ * @author vasily@wavefront.com
+ */
+public final class GlobalPropertiesImpl implements GlobalProperties {
+  private final ProxyConfig wrapped;
+  private Double retryBackoffBaseSeconds = null;
+  private short histogramStorageAccuracy = 32;
+  private Double traceSamplingRate = null;
+  private Integer dropSpansDelayedMinutes = null;
+  private List activeSpanSamplingPolicies;
+
+  public GlobalPropertiesImpl(ProxyConfig wrapped) {
+    this.wrapped = wrapped;
+    reportSettingAsGauge(this::getRetryBackoffBaseSeconds, "dynamic.retryBackoffBaseSeconds");
+  }
+
+  @Override
+  public double getRetryBackoffBaseSeconds() {
+    return firstNonNull(retryBackoffBaseSeconds, wrapped.getRetryBackoffBaseSeconds());
+  }
+
+  @Override
+  public void setRetryBackoffBaseSeconds(@Nullable Double retryBackoffBaseSeconds) {
+    this.retryBackoffBaseSeconds = retryBackoffBaseSeconds;
+  }
+
+  @Override
+  public short getHistogramStorageAccuracy() {
+    return histogramStorageAccuracy;
+  }
+
+  @Override
+  public void setHistogramStorageAccuracy(short histogramStorageAccuracy) {
+    this.histogramStorageAccuracy = histogramStorageAccuracy;
+  }
+
+  @Override
+  public double getTraceSamplingRate() {
+    if (traceSamplingRate != null) {
+      // use the minimum of backend provided and local proxy configured sampling rates.
+      return Math.min(traceSamplingRate, wrapped.getTraceSamplingRate());
+    } else {
+      return wrapped.getTraceSamplingRate();
+    }
+  }
+
+  public void setTraceSamplingRate(@Nullable Double traceSamplingRate) {
+    this.traceSamplingRate = traceSamplingRate;
+  }
+
+  @Override
+  public Integer getDropSpansDelayedMinutes() {
+    return dropSpansDelayedMinutes;
+  }
+
+  @Override
+  public void setDropSpansDelayedMinutes(@Nullable Integer dropSpansDelayedMinutes) {
+    this.dropSpansDelayedMinutes = dropSpansDelayedMinutes;
+  }
+
+  @Override
+  public List getActiveSpanSamplingPolicies() {
+    return activeSpanSamplingPolicies;
+  }
+
+  @Override
+  public void setActiveSpanSamplingPolicies(
+      @Nullable List activeSpanSamplingPolicies) {
+    this.activeSpanSamplingPolicies = activeSpanSamplingPolicies;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/IgnoreStatusCodeException.java b/proxy/src/main/java/com/wavefront/agent/data/IgnoreStatusCodeException.java
new file mode 100644
index 000000000..4661d0596
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/IgnoreStatusCodeException.java
@@ -0,0 +1,12 @@
+package com.wavefront.agent.data;
+
+/**
+ * Exception used to ignore 404s for DELETE API calls for sourceTags.
+ *
+ * @author vasily@wavefront.com
+ */
+public class IgnoreStatusCodeException extends DataSubmissionException {
+  public IgnoreStatusCodeException(String message) {
+    super(message);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/LineDelimitedDataSubmissionTask.java b/proxy/src/main/java/com/wavefront/agent/data/LineDelimitedDataSubmissionTask.java
new file mode 100644
index 000000000..58b2a0f5c
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/LineDelimitedDataSubmissionTask.java
@@ -0,0 +1,117 @@
+package com.wavefront.agent.data;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.wavefront.agent.handlers.LineDelimitedUtils;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.api.ProxyV2API;
+import com.wavefront.data.ReportableEntityType;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.function.Supplier;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.ws.rs.core.Response;
+
+/**
+ * A {@link DataSubmissionTask} that handles plaintext payloads in the newline-delimited format.
+ *
+ * @author vasily@wavefront.com
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "__CLASS")
+public class LineDelimitedDataSubmissionTask
+    extends AbstractDataSubmissionTask {
+
+  private transient ProxyV2API api;
+  private transient UUID proxyId;
+
+  @JsonProperty private String format;
+  @VisibleForTesting @JsonProperty protected List payload;
+
+  @SuppressWarnings("unused")
+  LineDelimitedDataSubmissionTask() {}
+
+  /**
+   * @param api API endpoint
+   * @param proxyId Proxy identifier. Used to authenticate proxy with the API.
+   * @param properties entity-specific wrapper over mutable proxy settings' container.
+   * @param backlog task queue.
+   * @param format Data format (passed as an argument to the API)
+   * @param entityType Entity type handled
+   * @param handle Handle (usually port number) of the pipeline where the data came from.
+   * @param payload Data payload
+   * @param timeProvider Time provider (in millis)
+   */
+  public LineDelimitedDataSubmissionTask(
+      ProxyV2API api,
+      UUID proxyId,
+      EntityProperties properties,
+      TaskQueue backlog,
+      String format,
+      ReportableEntityType entityType,
+      String handle,
+      @Nonnull List payload,
+      @Nullable Supplier timeProvider) {
+    super(properties, backlog, handle, entityType, timeProvider);
+    this.api = api;
+    this.proxyId = proxyId;
+    this.format = format;
+    this.payload = new ArrayList<>(payload);
+  }
+
+  @Override
+  Response doExecute() {
+    return api.proxyReport(proxyId, format, LineDelimitedUtils.joinPushData(payload));
+  }
+
+  @Override
+  public int weight() {
+    return this.payload.size();
+  }
+
+  @Override
+  public List splitTask(int minSplitSize, int maxSplitSize) {
+    if (payload.size() > Math.max(1, minSplitSize)) {
+      List result = new ArrayList<>();
+      int stride = Math.min(maxSplitSize, (int) Math.ceil((float) payload.size() / 2.0));
+      int endingIndex = 0;
+      for (int startingIndex = 0; endingIndex < payload.size() - 1; startingIndex += stride) {
+        endingIndex = Math.min(payload.size(), startingIndex + stride) - 1;
+        result.add(
+            new LineDelimitedDataSubmissionTask(
+                api,
+                proxyId,
+                properties,
+                backlog,
+                format,
+                getEntityType(),
+                handle,
+                payload.subList(startingIndex, endingIndex + 1),
+                timeProvider));
+      }
+      return result;
+    }
+    return ImmutableList.of(this);
+  }
+
+  public List payload() {
+    return payload;
+  }
+
+  public void injectMembers(
+      ProxyV2API api,
+      UUID proxyId,
+      EntityProperties properties,
+      TaskQueue backlog) {
+    this.api = api;
+    this.proxyId = proxyId;
+    this.properties = properties;
+    this.backlog = backlog;
+    this.timeProvider = System::currentTimeMillis;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/LogDataSubmissionTask.java b/proxy/src/main/java/com/wavefront/agent/data/LogDataSubmissionTask.java
new file mode 100644
index 000000000..a34092b34
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/LogDataSubmissionTask.java
@@ -0,0 +1,127 @@
+package com.wavefront.agent.data;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.google.common.collect.ImmutableList;
+import com.google.gson.Gson;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.api.LogAPI;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.dto.Log;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.MetricName;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.ws.rs.core.Response;
+
+/**
+ * A {@link DataSubmissionTask} that handles log payloads.
+ *
+ * @author amitw@vmware.com
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "__CLASS")
+public class LogDataSubmissionTask extends AbstractDataSubmissionTask {
+  private static final Logger LOGGER = Logger.getLogger("LogDataSubmission");
+  public static final String AGENT_PREFIX = "WF-PROXY-AGENT-";
+  private transient LogAPI api;
+  private transient UUID proxyId;
+
+  @JsonProperty private List logs;
+  @JsonProperty private int weight;
+
+  @SuppressWarnings("unused")
+  LogDataSubmissionTask() {}
+
+  /**
+   * @param api API endpoint.
+   * @param proxyId Proxy identifier
+   * @param properties entity-specific wrapper over mutable proxy settings' container.
+   * @param backlog task queue.
+   * @param handle Handle (usually port number) of the pipeline where the data came from.
+   * @param logs Data payload.
+   * @param timeProvider Time provider (in millis).
+   */
+  public LogDataSubmissionTask(
+      LogAPI api,
+      UUID proxyId,
+      EntityProperties properties,
+      TaskQueue backlog,
+      String handle,
+      @Nonnull List logs,
+      @Nullable Supplier timeProvider) {
+    super(properties, backlog, handle, ReportableEntityType.LOGS, timeProvider);
+    this.api = api;
+    this.proxyId = proxyId;
+    this.logs = new ArrayList<>(logs);
+    for (Log l : logs) {
+      weight += l.getDataSize();
+    }
+  }
+
+  @Override
+  Response doExecute() {
+    try {
+      LOGGER.finest(() -> ("Logs batch sent to vRLIC: " + new Gson().toJson(logs)));
+    } catch (Exception e) {
+      LOGGER.log(
+          Level.WARNING, "Error occurred while logging the batch sent to vRLIC: " + e.getMessage());
+    }
+    return api.proxyLogs(AGENT_PREFIX + proxyId.toString(), logs);
+  }
+
+  @Override
+  protected TaskResult handleStatus429() {
+    Metrics.newCounter(
+            new MetricName(entityType + "." + handle, "", "failed" + ".ingestion_limit_reached"))
+        .inc(this.weight());
+    return TaskResult.REMOVED;
+  }
+
+  @Override
+  public int weight() {
+    return weight;
+  }
+
+  @Override
+  public List splitTask(int minSplitSize, int maxSplitSize) {
+    if (logs.size() > Math.max(1, minSplitSize)) {
+      List result = new ArrayList<>();
+      int stride = Math.min(maxSplitSize, (int) Math.ceil((float) logs.size() / 2.0));
+      int endingIndex = 0;
+      for (int startingIndex = 0; endingIndex < logs.size() - 1; startingIndex += stride) {
+        endingIndex = Math.min(logs.size(), startingIndex + stride) - 1;
+        result.add(
+            new LogDataSubmissionTask(
+                api,
+                proxyId,
+                properties,
+                backlog,
+                handle,
+                logs.subList(startingIndex, endingIndex + 1),
+                timeProvider));
+      }
+      return result;
+    }
+    return ImmutableList.of(this);
+  }
+
+  public void injectMembers(
+      LogAPI api,
+      UUID proxyId,
+      EntityProperties properties,
+      TaskQueue backlog) {
+    this.api = api;
+    this.proxyId = proxyId;
+    this.properties = properties;
+    this.backlog = backlog;
+    this.timeProvider = System::currentTimeMillis;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/QueueingReason.java b/proxy/src/main/java/com/wavefront/agent/data/QueueingReason.java
new file mode 100644
index 000000000..3d5d69315
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/QueueingReason.java
@@ -0,0 +1,26 @@
+package com.wavefront.agent.data;
+
+/**
+ * Additional context to help understand why a certain batch was queued.
+ *
+ * @author vasily@wavefront.com
+ */
+public enum QueueingReason {
+  PUSHBACK("pushback"), // server pushback
+  AUTH("auth"), // feature not enabled or auth error
+  SPLIT("split"), // splitting batches
+  RETRY("retry"), // all other errors (http error codes or network errors)
+  BUFFER_SIZE("bufferSize"), // buffer size threshold exceeded
+  MEMORY_PRESSURE("memoryPressure"), // heap memory limits exceeded
+  DURABILITY("durability"); // force-flush for maximum durability (for future use)
+
+  private final String name;
+
+  QueueingReason(String name) {
+    this.name = name;
+  }
+
+  public String toString() {
+    return this.name;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/SourceTagSubmissionTask.java b/proxy/src/main/java/com/wavefront/agent/data/SourceTagSubmissionTask.java
new file mode 100644
index 000000000..4d3f986ee
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/SourceTagSubmissionTask.java
@@ -0,0 +1,120 @@
+package com.wavefront.agent.data;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.google.common.collect.ImmutableList;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.api.SourceTagAPI;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.dto.SourceTag;
+import java.util.List;
+import java.util.function.Supplier;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.ws.rs.core.Response;
+
+/**
+ * A {@link DataSubmissionTask} that handles source tag payloads.
+ *
+ * @author vasily@wavefront.com
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "__CLASS")
+public class SourceTagSubmissionTask extends AbstractDataSubmissionTask {
+  private transient SourceTagAPI api;
+
+  @JsonProperty private SourceTag sourceTag;
+
+  @SuppressWarnings("unused")
+  SourceTagSubmissionTask() {}
+
+  /**
+   * @param api API endpoint.
+   * @param properties container for mutable proxy settings.
+   * @param backlog backing queue.
+   * @param handle Handle (usually port number) of the pipeline where the data came from.
+   * @param sourceTag source tag operation
+   * @param timeProvider Time provider (in millis).
+   */
+  public SourceTagSubmissionTask(
+      SourceTagAPI api,
+      EntityProperties properties,
+      TaskQueue backlog,
+      String handle,
+      @Nonnull SourceTag sourceTag,
+      @Nullable Supplier timeProvider) {
+    super(properties, backlog, handle, ReportableEntityType.SOURCE_TAG, timeProvider);
+    this.api = api;
+    this.sourceTag = sourceTag;
+    this.limitRetries = true;
+  }
+
+  @Nullable
+  Response doExecute() throws DataSubmissionException {
+    switch (sourceTag.getOperation()) {
+      case SOURCE_DESCRIPTION:
+        switch (sourceTag.getAction()) {
+          case DELETE:
+            Response resp = api.removeDescription(sourceTag.getSource());
+            if (resp.getStatus() == 404) {
+              throw new IgnoreStatusCodeException(
+                  "Attempting to delete description for "
+                      + "a non-existent source  "
+                      + sourceTag.getSource()
+                      + ", ignoring");
+            }
+            return resp;
+          case SAVE:
+          case ADD:
+            return api.setDescription(sourceTag.getSource(), sourceTag.getAnnotations().get(0));
+          default:
+            throw new IllegalArgumentException("Invalid acton: " + sourceTag.getAction());
+        }
+      case SOURCE_TAG:
+        switch (sourceTag.getAction()) {
+          case ADD:
+            return api.appendTag(sourceTag.getSource(), sourceTag.getAnnotations().get(0));
+          case DELETE:
+            String tag = sourceTag.getAnnotations().get(0);
+            Response resp = api.removeTag(sourceTag.getSource(), tag);
+            if (resp.getStatus() == 404) {
+              throw new IgnoreStatusCodeException(
+                  "Attempting to delete non-existing tag "
+                      + tag
+                      + " for source "
+                      + sourceTag.getSource()
+                      + ", ignoring");
+            }
+            return resp;
+          case SAVE:
+            return api.setTags(sourceTag.getSource(), sourceTag.getAnnotations());
+          default:
+            throw new IllegalArgumentException("Invalid acton: " + sourceTag.getAction());
+        }
+      default:
+        throw new IllegalArgumentException(
+            "Invalid source tag operation: " + sourceTag.getOperation());
+    }
+  }
+
+  public SourceTag payload() {
+    return sourceTag;
+  }
+
+  @Override
+  public int weight() {
+    return 1;
+  }
+
+  @Override
+  public List splitTask(int minSplitSize, int maxSplitSize) {
+    return ImmutableList.of(this);
+  }
+
+  public void injectMembers(
+      SourceTagAPI api, EntityProperties properties, TaskQueue backlog) {
+    this.api = api;
+    this.properties = properties;
+    this.backlog = backlog;
+    this.timeProvider = System::currentTimeMillis;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/TaskInjector.java b/proxy/src/main/java/com/wavefront/agent/data/TaskInjector.java
new file mode 100644
index 000000000..946ff8e29
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/TaskInjector.java
@@ -0,0 +1,16 @@
+package com.wavefront.agent.data;
+
+/**
+ * Class to inject non-serializable members into a {@link DataSubmissionTask} before execution
+ *
+ * @author vasily@wavefront.com
+ */
+public interface TaskInjector> {
+
+  /**
+   * Inject members into specified task.
+   *
+   * @param task task to inject
+   */
+  void inject(T task);
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/TaskQueueLevel.java b/proxy/src/main/java/com/wavefront/agent/data/TaskQueueLevel.java
new file mode 100644
index 000000000..8043a67d0
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/TaskQueueLevel.java
@@ -0,0 +1,33 @@
+package com.wavefront.agent.data;
+
+/**
+ * Controls conditions under which proxy would actually queue data.
+ *
+ * @author vasily@wavefront.com
+ */
+public enum TaskQueueLevel {
+  NEVER(0), // never queue (not used, placeholder for future use)
+  MEMORY(1), // queue on memory pressure (heap threshold or pushMemoryBufferLimit exceeded)
+  PUSHBACK(2), // queue on pushback + memory pressure
+  ANY_ERROR(3), // queue on any errors, pushback or memory pressure
+  ALWAYS(4); // queue before send attempts (maximum durability - placeholder for future use)
+
+  private final int level;
+
+  TaskQueueLevel(int level) {
+    this.level = level;
+  }
+
+  public boolean isLessThan(TaskQueueLevel other) {
+    return this.level < other.level;
+  }
+
+  public static TaskQueueLevel fromString(String name) {
+    for (TaskQueueLevel level : TaskQueueLevel.values()) {
+      if (level.toString().equalsIgnoreCase(name)) {
+        return level;
+      }
+    }
+    return null;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/data/TaskResult.java b/proxy/src/main/java/com/wavefront/agent/data/TaskResult.java
new file mode 100644
index 000000000..7efec8272
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/data/TaskResult.java
@@ -0,0 +1,14 @@
+package com.wavefront.agent.data;
+
+/**
+ * Possible outcomes of {@link DataSubmissionTask} execution
+ *
+ * @author vasily@wavefront.com
+ */
+public enum TaskResult {
+  DELIVERED, // success
+  REMOVED, // data is removed from queue, due to feature disabled or auth error
+  PERSISTED, // data is persisted in the queue, start back-off process
+  PERSISTED_RETRY, // data is persisted in the queue, ok to continue processing backlog
+  RETRY_LATER // data needs to be returned to the pool and retried later
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/formatter/DataFormat.java b/proxy/src/main/java/com/wavefront/agent/formatter/DataFormat.java
new file mode 100644
index 000000000..27e21498a
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/formatter/DataFormat.java
@@ -0,0 +1,73 @@
+package com.wavefront.agent.formatter;
+
+import com.wavefront.api.agent.Constants;
+import com.wavefront.ingester.AbstractIngesterFormatter;
+import javax.annotation.Nullable;
+
+/**
+ * Best-effort data format auto-detection.
+ *
+ * @author vasily@wavefront.com
+ */
+public enum DataFormat {
+  DEFAULT,
+  WAVEFRONT,
+  HISTOGRAM,
+  SOURCE_TAG,
+  EVENT,
+  SPAN,
+  SPAN_LOG,
+  LOGS_JSON_ARR,
+  LOGS_JSON_LINES,
+  LOGS_JSON_CLOUDWATCH;
+
+  public static DataFormat autodetect(final String input) {
+    if (input.length() < 2) return DEFAULT;
+    char firstChar = input.charAt(0);
+    switch (firstChar) {
+      case '@':
+        if (input.startsWith(AbstractIngesterFormatter.SOURCE_TAG_LITERAL)
+            || input.startsWith(AbstractIngesterFormatter.SOURCE_DESCRIPTION_LITERAL)) {
+          return SOURCE_TAG;
+        }
+        if (input.startsWith(AbstractIngesterFormatter.EVENT_LITERAL)) return EVENT;
+        break;
+      case '{':
+        if (input.charAt(input.length() - 1) == '}') return SPAN_LOG;
+        break;
+      case '!':
+        if (input.startsWith("!M ") || input.startsWith("!H ") || input.startsWith("!D ")) {
+          return HISTOGRAM;
+        }
+        break;
+      case '[':
+        if (input.charAt(input.length() - 1) == ']') return LOGS_JSON_ARR;
+        break;
+    }
+    return DEFAULT;
+  }
+
+  @Nullable
+  public static DataFormat parse(String format) {
+    if (format == null) return null;
+    switch (format) {
+      case Constants.PUSH_FORMAT_WAVEFRONT:
+      case Constants.PUSH_FORMAT_GRAPHITE_V2:
+        return DataFormat.WAVEFRONT;
+      case Constants.PUSH_FORMAT_HISTOGRAM:
+        return DataFormat.HISTOGRAM;
+      case Constants.PUSH_FORMAT_TRACING:
+        return DataFormat.SPAN;
+      case Constants.PUSH_FORMAT_TRACING_SPAN_LOGS:
+        return DataFormat.SPAN_LOG;
+      case Constants.PUSH_FORMAT_LOGS_JSON_ARR:
+        return DataFormat.LOGS_JSON_ARR;
+      case Constants.PUSH_FORMAT_LOGS_JSON_LINES:
+        return DataFormat.LOGS_JSON_LINES;
+      case Constants.PUSH_FORMAT_LOGS_JSON_CLOUDWATCH:
+        return DataFormat.LOGS_JSON_CLOUDWATCH;
+      default:
+        return null;
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/formatter/GraphiteFormatter.java b/proxy/src/main/java/com/wavefront/agent/formatter/GraphiteFormatter.java
index 91e7f6119..f30ae654f 100644
--- a/proxy/src/main/java/com/wavefront/agent/formatter/GraphiteFormatter.java
+++ b/proxy/src/main/java/com/wavefront/agent/formatter/GraphiteFormatter.java
@@ -2,13 +2,8 @@
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
 import com.wavefront.common.MetricMangler;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * Specific formatter for the graphite/collectd world of metric-munged names.
@@ -42,7 +37,7 @@ public String apply(String mesg) {
     // 1. Extract fields
     String[] regions = mesg.trim().split(" ");
     final MetricMangler.MetricComponents components = metricMangler.extractComponents(regions[0]);
-    
+
     finalMesg.append(components.metric);
     finalMesg.append(" ");
 
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/AbstractReportableEntityHandler.java b/proxy/src/main/java/com/wavefront/agent/handlers/AbstractReportableEntityHandler.java
index 464f5e793..b3d0cb49c 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/AbstractReportableEntityHandler.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/AbstractReportableEntityHandler.java
@@ -1,209 +1,240 @@
 package com.wavefront.agent.handlers;
 
 import com.google.common.util.concurrent.RateLimiter;
-
-import com.wavefront.agent.SharedMetricsRegistry;
-import com.wavefront.data.ReportableEntityType;
+import com.wavefront.agent.formatter.DataFormat;
+import com.wavefront.common.NamedThreadFactory;
 import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.Gauge;
-import com.yammer.metrics.core.Histogram;
-import com.yammer.metrics.core.Meter;
-import com.yammer.metrics.core.MetricName;
-
-import java.lang.reflect.ParameterizedType;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
+import com.yammer.metrics.core.*;
+import java.util.*;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
 
 /**
  * Base class for all {@link ReportableEntityHandler} implementations.
  *
  * @author vasily@wavefront.com
- *
  * @param  the type of input objects handled
+ * @param  the type of the output object as handled by {@link SenderTask}
  */
-abstract class AbstractReportableEntityHandler implements ReportableEntityHandler {
-  private static final Logger logger = Logger.getLogger(AbstractReportableEntityHandler.class.getCanonicalName());
-
-  @SuppressWarnings("unchecked")
-  private Class type = (Class) ((ParameterizedType) getClass().getGenericSuperclass())
-      .getActualTypeArguments()[0];
-
-  private static SharedMetricsRegistry metricsRegistry = SharedMetricsRegistry.getInstance();
+abstract class AbstractReportableEntityHandler implements ReportableEntityHandler {
+  private static final Logger logger =
+      Logger.getLogger(AbstractReportableEntityHandler.class.getCanonicalName());
+  protected static final MetricsRegistry LOCAL_REGISTRY = new MetricsRegistry();
+  protected static final String MULTICASTING_TENANT_TAG_KEY = "multicastingTenantName";
 
-  ScheduledExecutorService statisticOutputExecutor = Executors.newSingleThreadScheduledExecutor();
   private final Logger blockedItemsLogger;
 
-  String handle;
-  Counter receivedCounter;
-  Counter blockedCounter;
-  Counter rejectedCounter;
-  final RateLimiter blockedItemsLimiter;
-  final Function serializer;
-  List> senderTasks;
+  final HandlerKey handlerKey;
+  protected final Counter receivedCounter;
+  protected final Counter attemptedCounter;
+  protected Counter blockedCounter;
+  protected Counter rejectedCounter;
 
-  final ArrayList receivedStats = new ArrayList<>(Collections.nCopies(300, 0L));
-  private final Histogram receivedBurstRateHistogram;
-  private long receivedPrevious;
-  long receivedBurstRateCurrent;
+  @SuppressWarnings("UnstableApiUsage")
+  final RateLimiter blockedItemsLimiter;
 
-  Function serializerFunc;
+  final Function serializer;
+  final Map>> senderTaskMap;
+  protected final boolean isMulticastingActive;
+  final boolean reportReceivedStats;
+  final String rateUnit;
 
+  final BurstRateTrackingCounter receivedStats;
+  final BurstRateTrackingCounter deliveredStats;
+  private final ScheduledThreadPoolExecutor timer;
   private final AtomicLong roundRobinCounter = new AtomicLong();
+  protected final MetricsRegistry registry;
+  protected final String metricPrefix;
+
+  @SuppressWarnings("UnstableApiUsage")
+  private final RateLimiter noDataStatsRateLimiter = RateLimiter.create(1.0d / 60);
 
   /**
-   * Base constructor.
-   *
-   * @param entityType           entity type that dictates the data flow.
-   * @param handle               handle (usually port number), that serves as an identifier for the metrics pipeline.
-   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into the main log file.
-   * @param serializer           helper function to convert objects to string. Used when writing blocked points to logs.
-   * @param senderTasks          tasks actually handling data transfer to the Wavefront endpoint.
+   * @param handlerKey metrics pipeline key (entity type + port number)
+   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into
+   *     the main log file.
+   * @param serializer helper function to convert objects to string. Used when writing blocked
+   *     points to logs.
+   * @param senderTaskMap map of tenant name and tasks actually handling data transfer to the
+   *     Wavefront endpoint corresponding to the tenant name
+   * @param reportReceivedStats Whether we should report a .received counter metric.
+   * @param receivedRateSink Where to report received rate (tenant specific).
+   * @param blockedItemsLogger a {@link Logger} instance for blocked items
    */
-  @SuppressWarnings("unchecked")
-  AbstractReportableEntityHandler(ReportableEntityType entityType,
-                                  @NotNull String handle,
-                                  final int blockedItemsPerBatch,
-                                  Function serializer,
-                                  @NotNull Collection senderTasks) {
-    String strEntityType = entityType.toString();
-    this.blockedItemsLogger = Logger.getLogger("RawBlocked" + strEntityType.substring(0, 1).toUpperCase() +
-        strEntityType.substring(1));
-    this.handle = handle;
-    this.receivedCounter = Metrics.newCounter(new MetricName(strEntityType + "." + handle, "", "received"));
-    this.blockedCounter = Metrics.newCounter(new MetricName(strEntityType + "." + handle, "", "blocked"));
-    this.rejectedCounter = Metrics.newCounter(new MetricName(strEntityType + "." + handle, "", "rejected"));
-    this.blockedItemsLimiter = blockedItemsPerBatch == 0 ? null : RateLimiter.create(blockedItemsPerBatch / 10d);
+  AbstractReportableEntityHandler(
+      HandlerKey handlerKey,
+      final int blockedItemsPerBatch,
+      final Function serializer,
+      @Nullable final Map>> senderTaskMap,
+      boolean reportReceivedStats,
+      @Nullable final BiConsumer receivedRateSink,
+      @Nullable final Logger blockedItemsLogger) {
+    this.handlerKey = handlerKey;
+    //noinspection UnstableApiUsage
+    this.blockedItemsLimiter =
+        blockedItemsPerBatch == 0 ? null : RateLimiter.create(blockedItemsPerBatch / 10d);
     this.serializer = serializer;
-    this.serializerFunc = obj -> {
-      if (type.isInstance(obj)) {
-        return serializer.apply(type.cast(obj));
-      } else {
-        return null;
-      }
-    };
-    this.senderTasks = new ArrayList<>();
-    for (SenderTask task : senderTasks) {
-      this.senderTasks.add((SenderTask) task);
+    this.senderTaskMap = senderTaskMap == null ? new HashMap<>() : new HashMap<>(senderTaskMap);
+    this.isMulticastingActive = this.senderTaskMap.size() > 1;
+    this.reportReceivedStats = reportReceivedStats;
+    this.rateUnit = handlerKey.getEntityType().getRateUnit();
+    this.blockedItemsLogger = blockedItemsLogger;
+    this.registry = reportReceivedStats ? Metrics.defaultRegistry() : LOCAL_REGISTRY;
+    this.metricPrefix = handlerKey.toString();
+    MetricName receivedMetricName = new MetricName(metricPrefix, "", "received");
+    MetricName deliveredMetricName = new MetricName(metricPrefix, "", "delivered");
+    this.receivedCounter = registry.newCounter(receivedMetricName);
+    this.attemptedCounter = Metrics.newCounter(new MetricName(metricPrefix, "", "sent"));
+    this.receivedStats = new BurstRateTrackingCounter(receivedMetricName, registry, 1000);
+    this.deliveredStats = new BurstRateTrackingCounter(deliveredMetricName, registry, 1000);
+    registry.newGauge(
+        new MetricName(metricPrefix + ".received", "", "max-burst-rate"),
+        new Gauge() {
+          @Override
+          public Double value() {
+            return receivedStats.getMaxBurstRateAndClear();
+          }
+        });
+    this.timer = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("stats-output"));
+    if (receivedRateSink != null) {
+      timer.scheduleAtFixedRate(
+          () -> {
+            try {
+              for (String tenantName : senderTaskMap.keySet()) {
+                receivedRateSink.accept(tenantName, receivedStats.getCurrentRate());
+              }
+            } catch (Throwable e) {
+              logger.log(Level.WARNING, "receivedRateSink", e);
+            }
+          },
+          1,
+          1,
+          TimeUnit.SECONDS);
     }
-    this.receivedBurstRateHistogram = metricsRegistry.newHistogram(AbstractReportableEntityHandler.class,
-        "received-" + strEntityType + ".burst-rate." + handle);
-    Metrics.newGauge(new MetricName(strEntityType + "." + handle + ".received", "",
-        "max-burst-rate"), new Gauge() {
-      @Override
-      public Double value() {
-        Double maxValue = receivedBurstRateHistogram.max();
-        receivedBurstRateHistogram.clear();
-        return maxValue;
-      }
-    });
-
-    this.receivedPrevious = 0;
-    this.receivedBurstRateCurrent = 0;
-    statisticOutputExecutor.scheduleAtFixedRate(() -> {
-      long received = this.receivedCounter.count();
-      this.receivedBurstRateCurrent = received - this.receivedPrevious;
-      this.receivedBurstRateHistogram.update(this.receivedBurstRateCurrent);
-      this.receivedPrevious = received;
-      receivedStats.remove(0);
-      receivedStats.add(this.receivedBurstRateCurrent);
-    }, 1, 1, TimeUnit.SECONDS);
-  }
 
-  @Override
-  public void reject(T item) {
-    blockedCounter.inc();
-    rejectedCounter.inc();
-    if (item != null) {
-      blockedItemsLogger.warning(serializer.apply(item));
-    }
-    if (blockedItemsLimiter != null && blockedItemsLimiter.tryAcquire()) {
-      logger.info("[" + handle + "] blocked input: [" + serializer.apply(item) + "]");
+    timer.scheduleAtFixedRate(
+        () -> {
+          try {
+            printStats();
+          } catch (Throwable e) {
+            logger.log(Level.WARNING, "printStats", e);
+          }
+        },
+        10,
+        10,
+        TimeUnit.SECONDS);
+
+    if (reportReceivedStats) {
+      timer.scheduleAtFixedRate(
+          () -> {
+            try {
+              printTotal();
+            } catch (Throwable e) {
+              logger.log(Level.WARNING, "printTotal", e);
+            }
+          },
+          1,
+          1,
+          TimeUnit.MINUTES);
     }
   }
 
+  protected void initializeCounters() {
+    this.blockedCounter = registry.newCounter(new MetricName(metricPrefix, "", "blocked"));
+    this.rejectedCounter = registry.newCounter(new MetricName(metricPrefix, "", "rejected"));
+  }
+
   @Override
   public void reject(@Nullable T item, @Nullable String message) {
     blockedCounter.inc();
     rejectedCounter.inc();
-    if (item != null) {
+    if (item != null && blockedItemsLogger != null) {
       blockedItemsLogger.warning(serializer.apply(item));
     }
+    //noinspection UnstableApiUsage
     if (message != null && blockedItemsLimiter != null && blockedItemsLimiter.tryAcquire()) {
-      logger.info("[" + handle + "] blocked input: [" + message + "]");
+      logger.info("[" + handlerKey.getHandle() + "] blocked input: [" + message + "]");
     }
   }
 
   @Override
-  public void reject(@NotNull String line, @Nullable String message) {
+  public void reject(@Nonnull String line, @Nullable String message) {
     blockedCounter.inc();
     rejectedCounter.inc();
-    blockedItemsLogger.warning(line);
+    if (blockedItemsLogger != null) blockedItemsLogger.warning(line);
+    //noinspection UnstableApiUsage
     if (message != null && blockedItemsLimiter != null && blockedItemsLimiter.tryAcquire()) {
-      logger.info("[" + handle + "] blocked input: [" + message + "]");
+      logger.info("[" + handlerKey.getHandle() + "] blocked input: [" + message + "]");
     }
   }
 
   @Override
   public void block(T item) {
     blockedCounter.inc();
-    blockedItemsLogger.info(serializer.apply(item));
+    if (blockedItemsLogger != null) {
+      blockedItemsLogger.info(serializer.apply(item));
+    }
   }
 
   @Override
   public void block(@Nullable T item, @Nullable String message) {
     blockedCounter.inc();
-    if (item != null) {
+    if (item != null && blockedItemsLogger != null) {
       blockedItemsLogger.info(serializer.apply(item));
     }
-    if (message != null) {
+    if (message != null && blockedItemsLogger != null) {
       blockedItemsLogger.info(message);
     }
   }
 
   @Override
   public void report(T item) {
-    report(item, item, serializerFunc);
-  }
-
-  @Override
-  public void report(T item, @Nullable Object messageObject,
-                     @NotNull Function messageSerializer) {
     try {
+      attemptedCounter.inc();
       reportInternal(item);
     } catch (IllegalArgumentException e) {
-      this.reject(item, e.getMessage() + " (" + messageSerializer.apply(messageObject) + ")");
+      this.reject(item, e.getMessage() + " (" + serializer.apply(item) + ")");
     } catch (Exception ex) {
-      logger.log(Level.SEVERE, "WF-500 Uncaught exception when handling input (" +
-          messageSerializer.apply(messageObject) + ")", ex);
+      logger.log(
+          Level.SEVERE,
+          "WF-500 Uncaught exception when handling input (" + serializer.apply(item) + ")",
+          ex);
     }
   }
 
-  abstract void reportInternal(T item);
+  @Override
+  public void shutdown() {
+    if (this.timer != null) timer.shutdown();
+  }
 
-  protected long getReceivedOneMinuteRate() {
-    return receivedStats.subList(240, 300).stream().mapToLong(i -> i).sum() / 60;
+  @Override
+  public void setLogFormat(DataFormat format) {
+    throw new UnsupportedOperationException();
   }
 
-  protected long getReceivedFiveMinuteRate() {
-    return receivedStats.stream().mapToLong(i -> i).sum() / 300;
+  abstract void reportInternal(T item);
+
+  protected Counter getReceivedCounter() {
+    return receivedCounter;
   }
 
-  protected SenderTask getTask() {
+  protected SenderTask getTask(String tenantName) {
+    if (senderTaskMap == null) {
+      throw new IllegalStateException("getTask() cannot be called on null senderTasks");
+    }
+    if (!senderTaskMap.containsKey(tenantName)) {
+      return null;
+    }
+    List> senderTasks = new ArrayList<>(senderTaskMap.get(tenantName));
     // roundrobin all tasks, skipping the worst one (usually with the highest number of points)
-    int nextTaskId = (int)(roundRobinCounter.getAndIncrement() % senderTasks.size());
+    int nextTaskId = (int) (roundRobinCounter.getAndIncrement() % senderTasks.size());
     long worstScore = 0L;
     int worstTaskId = 0;
     for (int i = 0; i < senderTasks.size(); i++) {
@@ -214,8 +245,62 @@ protected SenderTask getTask() {
       }
     }
     if (nextTaskId == worstTaskId) {
-      nextTaskId = (int)(roundRobinCounter.getAndIncrement() % senderTasks.size());
+      nextTaskId = (int) (roundRobinCounter.getAndIncrement() % senderTasks.size());
     }
     return senderTasks.get(nextTaskId);
   }
+
+  protected void printStats() {
+    // if we received no data over the last 5 minutes, only print stats once a minute
+    //noinspection UnstableApiUsage
+    if (receivedStats.getFiveMinuteCount() == 0 && !noDataStatsRateLimiter.tryAcquire()) return;
+    if (reportReceivedStats) {
+      logger.info(
+          "["
+              + handlerKey.getHandle()
+              + "] "
+              + handlerKey.getEntityType().toCapitalizedString()
+              + " received rate: "
+              + receivedStats.getOneMinutePrintableRate()
+              + " "
+              + rateUnit
+              + " (1 min), "
+              + receivedStats.getFiveMinutePrintableRate()
+              + " "
+              + rateUnit
+              + " (5 min), "
+              + receivedStats.getCurrentRate()
+              + " "
+              + rateUnit
+              + " (current).");
+    }
+    if (deliveredStats.getFiveMinuteCount() == 0) return;
+    logger.info(
+        "["
+            + handlerKey.getHandle()
+            + "] "
+            + handlerKey.getEntityType().toCapitalizedString()
+            + " delivered rate: "
+            + deliveredStats.getOneMinutePrintableRate()
+            + " "
+            + rateUnit
+            + " (1 min), "
+            + deliveredStats.getFiveMinutePrintableRate()
+            + " "
+            + rateUnit
+            + " (5 min)");
+    // we are not going to display current delivered rate because it _will_ be misinterpreted.
+  }
+
+  protected void printTotal() {
+    logger.info(
+        "["
+            + handlerKey.getHandle()
+            + "] "
+            + handlerKey.getEntityType().toCapitalizedString()
+            + " processed since start: "
+            + this.attemptedCounter.count()
+            + "; blocked: "
+            + this.blockedCounter.count());
+  }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/AbstractSenderTask.java b/proxy/src/main/java/com/wavefront/agent/handlers/AbstractSenderTask.java
index 82e84df8d..1781fb40c 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/AbstractSenderTask.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/AbstractSenderTask.java
@@ -1,120 +1,203 @@
 package com.wavefront.agent.handlers;
 
 import com.google.common.util.concurrent.RateLimiter;
-
+import com.google.common.util.concurrent.RecyclableRateLimiter;
+import com.wavefront.agent.data.EntityProperties;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.agent.data.TaskResult;
 import com.wavefront.common.NamedThreadFactory;
 import com.wavefront.common.TaggedMetricName;
+import com.wavefront.common.logger.SharedRateLimitingLogger;
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Histogram;
 import com.yammer.metrics.core.MetricName;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-
 import javax.annotation.Nullable;
 
 /**
  * Base class for all {@link SenderTask} implementations.
  *
- * @author vasily@wavefront.com
- *
  * @param  the type of input objects handled.
  */
 abstract class AbstractSenderTask implements SenderTask, Runnable {
-  private static final Logger logger = Logger.getLogger(AbstractSenderTask.class.getCanonicalName());
+  private static final Logger logger =
+      Logger.getLogger(AbstractSenderTask.class.getCanonicalName());
+
+  /** Warn about exceeding the rate limit no more than once every 5 seconds */
+  protected final Logger throttledLogger;
 
   List datum = new ArrayList<>();
+  int datumSize;
   final Object mutex = new Object();
   final ScheduledExecutorService scheduler;
   private final ExecutorService flushExecutor;
 
-  final String entityType;
-  protected final String handle;
+  final HandlerKey handlerKey;
   final int threadId;
+  final EntityProperties properties;
+  final RecyclableRateLimiter rateLimiter;
 
-  final AtomicInteger itemsPerBatch;
-  final AtomicInteger memoryBufferLimit;
-
-  final Counter receivedCounter;
   final Counter attemptedCounter;
-  final Counter queuedCounter;
   final Counter blockedCounter;
   final Counter bufferFlushCounter;
+  final Counter bufferCompletedFlushCounter;
+  private final Histogram metricSize;
 
-  boolean isBuffering = false;
-  boolean isSending = false;
+  private final AtomicBoolean isRunning = new AtomicBoolean(false);
+  final AtomicBoolean isBuffering = new AtomicBoolean(false);
+  volatile boolean isSending = false;
 
   /**
-   * Attempt to schedule drainBuffersToQueueTask no more than once every 100ms to reduce
-   * scheduler overhead under memory pressure.
+   * Attempt to schedule drainBuffersToQueueTask no more than once every 100ms to reduce scheduler
+   * overhead under memory pressure.
    */
+  @SuppressWarnings("UnstableApiUsage")
   private final RateLimiter drainBuffersRateLimiter = RateLimiter.create(10);
 
-
   /**
    * Base constructor.
    *
-   * @param entityType        entity type that dictates the data processing flow.
-   * @param handle            handle (usually port number), that serves as an identifier for the metrics pipeline.
-   * @param threadId          thread number
-   * @param itemsPerBatch     max points per flush.
-   * @param memoryBufferLimit max points in task's memory buffer before queueing.
+   * @param handlerKey pipeline handler key that dictates the data processing flow.
+   * @param threadId thread number
+   * @param properties runtime properties container
+   * @param scheduler executor service for running this task
    */
-  AbstractSenderTask(String entityType, String handle, int threadId,
-                     @Nullable final AtomicInteger itemsPerBatch,
-                     @Nullable final AtomicInteger memoryBufferLimit) {
-    this.entityType = entityType;
-    this.handle = handle;
+  AbstractSenderTask(
+      HandlerKey handlerKey,
+      int threadId,
+      EntityProperties properties,
+      ScheduledExecutorService scheduler) {
+    this.handlerKey = handlerKey;
     this.threadId = threadId;
-    this.itemsPerBatch = itemsPerBatch == null ? new AtomicInteger(40000) : itemsPerBatch;
-    this.memoryBufferLimit = memoryBufferLimit == null ? new AtomicInteger(32 * 40000) : memoryBufferLimit;
-    this.scheduler = Executors.newScheduledThreadPool(1,
-        new NamedThreadFactory("submitter-" + entityType + "-" + handle + "-" + String.valueOf(threadId)));
-    this.flushExecutor = new ThreadPoolExecutor(1, 1, 60L, TimeUnit.MINUTES, new SynchronousQueue<>(),
-        new NamedThreadFactory("flush-" + entityType + "-" + handle + "-" + String.valueOf(threadId)));
-
-    this.attemptedCounter = Metrics.newCounter(new MetricName(entityType + "." + handle, "", "sent"));
-    this.queuedCounter = Metrics.newCounter(new MetricName(entityType + "." + handle, "", "queued"));
-    this.blockedCounter = Metrics.newCounter(new MetricName(entityType + "." + handle, "", "blocked"));
-    this.receivedCounter = Metrics.newCounter(new MetricName(entityType + "." + handle, "", "received"));
-    this.bufferFlushCounter = Metrics.newCounter(new TaggedMetricName("buffer", "flush-count", "port", handle));
+    this.properties = properties;
+    this.rateLimiter = properties.getRateLimiter();
+    this.scheduler = scheduler;
+    this.throttledLogger = new SharedRateLimitingLogger(logger, "rateLimit-" + handlerKey, 0.2);
+    this.flushExecutor =
+        new ThreadPoolExecutor(
+            1,
+            1,
+            60L,
+            TimeUnit.MINUTES,
+            new SynchronousQueue<>(),
+            new NamedThreadFactory("flush-" + handlerKey.toString() + "-" + threadId));
+
+    this.attemptedCounter = Metrics.newCounter(new MetricName(handlerKey.toString(), "", "sent"));
+    this.blockedCounter = Metrics.newCounter(new MetricName(handlerKey.toString(), "", "blocked"));
+    this.bufferFlushCounter =
+        Metrics.newCounter(
+            new TaggedMetricName("buffer", "flush-count", "port", handlerKey.getHandle()));
+    this.bufferCompletedFlushCounter =
+        Metrics.newCounter(
+            new TaggedMetricName(
+                "buffer", "completed-flush-count", "port", handlerKey.getHandle()));
 
+    this.metricSize =
+        Metrics.newHistogram(
+            new MetricName(handlerKey.toString() + "." + threadId, "", "metric_length"));
+    Metrics.newGauge(
+        new MetricName(handlerKey.toString() + "." + threadId, "", "size"),
+        new Gauge() {
+          @Override
+          public Integer value() {
+            return datumSize;
+          }
+        });
   }
 
-  /**
-   * Shut down the scheduler for this task (prevent future scheduled runs)
-   */
+  abstract TaskResult processSingleBatch(List batch);
+
   @Override
-  public void shutdown() {
+  public void run() {
+    if (!isRunning.get()) return;
+    long nextRunMillis = properties.getPushFlushInterval();
+    isSending = true;
     try {
-      scheduler.shutdownNow();
-      scheduler.awaitTermination(1000L, TimeUnit.MILLISECONDS);
+      List current = createBatch();
+      int currentBatchSize = getDataSize(current);
+      if (currentBatchSize == 0) return;
+      if (rateLimiter == null || rateLimiter.tryAcquire(currentBatchSize)) {
+        TaskResult result = processSingleBatch(current);
+        this.attemptedCounter.inc(currentBatchSize);
+        switch (result) {
+          case DELIVERED:
+            break;
+          case PERSISTED:
+          case PERSISTED_RETRY:
+            if (rateLimiter != null) rateLimiter.recyclePermits(currentBatchSize);
+            break;
+          case RETRY_LATER:
+            undoBatch(current);
+            if (rateLimiter != null) rateLimiter.recyclePermits(currentBatchSize);
+          default:
+        }
+      } else {
+        // if proxy rate limit exceeded, try again in 1/4..1/2 of flush interval
+        // to introduce some degree of fairness.
+        nextRunMillis = nextRunMillis / 4 + (int) (Math.random() * nextRunMillis / 4);
+        final long willRetryIn = nextRunMillis;
+        throttledLogger.log(
+            Level.INFO,
+            () ->
+                "["
+                    + handlerKey.getHandle()
+                    + " thread "
+                    + threadId
+                    + "]: WF-4 Proxy rate limiter active (pending "
+                    + handlerKey.getEntityType()
+                    + ": "
+                    + datumSize
+                    + "), will retry in "
+                    + willRetryIn
+                    + "ms");
+        undoBatch(current);
+      }
     } catch (Throwable t) {
-      logger.log(Level.SEVERE, "Error during shutdown", t);
+      logger.log(Level.SEVERE, "Unexpected error in flush loop", t);
+    } finally {
+      isSending = false;
+      if (isRunning.get()) {
+        scheduler.schedule(this, nextRunMillis, TimeUnit.MILLISECONDS);
+      }
     }
   }
 
   @Override
-  public void add(T metricString) {
-    synchronized (mutex) {
-      this.datum.add(metricString);
+  public void start() {
+    if (isRunning.compareAndSet(false, true)) {
+      this.scheduler.schedule(this, properties.getPushFlushInterval(), TimeUnit.MILLISECONDS);
     }
-    this.enforceBufferLimits();
   }
 
+  @Override
+  public void stop() {
+    isRunning.set(false);
+    flushExecutor.shutdown();
+  }
 
-  void enforceBufferLimits() {
-    if (datum.size() >= memoryBufferLimit.get() && drainBuffersRateLimiter.tryAcquire()) {
+  @Override
+  public void add(T metricString) {
+    metricSize.update(metricString.toString().length());
+    synchronized (mutex) {
+      this.datum.add(metricString);
+      datumSize += getObjectSize(metricString);
+    }
+    //noinspection UnstableApiUsage
+    if (datumSize >= properties.getMemoryBufferLimit()
+        && !isBuffering.get()
+        && drainBuffersRateLimiter.tryAcquire()) {
       try {
         flushExecutor.submit(drainBuffersToQueueTask);
       } catch (RejectedExecutionException e) {
@@ -123,50 +206,142 @@ void enforceBufferLimits() {
     }
   }
 
-  List createBatch() {
+  protected List createBatch() {
     List current;
     int blockSize;
     synchronized (mutex) {
-      blockSize = Math.min(datum.size(), itemsPerBatch.get());
+      blockSize = getBlockSize(datum, (int) rateLimiter.getRate(), properties.getDataPerBatch());
       current = datum.subList(0, blockSize);
+      datumSize -= getDataSize(current);
       datum = new ArrayList<>(datum.subList(blockSize, datum.size()));
     }
-    logger.fine("[" + handle + "] (DETAILED): sending " + current.size() + " valid " + entityType +
-        "; in memory: " + this.datum.size() +
-        "; total attempted: " + this.attemptedCounter.count() +
-        "; total blocked: " + this.blockedCounter.count() +
-        "; total queued: " + this.queuedCounter.count());
+    logger.fine(
+        "["
+            + handlerKey.getHandle()
+            + "] (DETAILED): sending "
+            + current.size()
+            + " valid "
+            + handlerKey.getEntityType()
+            + "; in memory: "
+            + datumSize
+            + "; total attempted: "
+            + this.attemptedCounter.count()
+            + "; total blocked: "
+            + this.blockedCounter.count());
     return current;
   }
 
-  private Runnable drainBuffersToQueueTask = new Runnable() {
-    @Override
-    public void run() {
-      if (datum.size() > memoryBufferLimit.get()) {
-        // there are going to be too many points to be able to flush w/o the agent blowing up
-        // drain the leftovers straight to the retry queue (i.e. to disk)
-        // don't let anyone add any more to points while we're draining it.
-        logger.warning("[" + handle + " thread " + threadId + "]: WF-3 Too many pending " + entityType +
-            " (" + datum.size() + "), block size: " + itemsPerBatch.get() + ". flushing to retry queue");
-        try {
-          isBuffering = true;
-          drainBuffersToQueue();
-        } finally {
-          isBuffering = false;
-          bufferFlushCounter.inc();
+  protected void undoBatch(List batch) {
+    synchronized (mutex) {
+      datum.addAll(0, batch);
+      datumSize += getDataSize(batch);
+    }
+  }
+
+  private final Runnable drainBuffersToQueueTask =
+      new Runnable() {
+        @Override
+        public void run() {
+          if (datumSize > properties.getMemoryBufferLimit()) {
+            // there are going to be too many points to be able to flush w/o the agent
+            // blowing up
+            // drain the leftovers straight to the retry queue (i.e. to disk)
+            // don't let anyone add any more to points while we're draining it.
+            logger.warning(
+                "["
+                    + handlerKey.getHandle()
+                    + " thread "
+                    + threadId
+                    + "]: WF-3 Too many pending "
+                    + handlerKey.getEntityType()
+                    + " ("
+                    + datumSize
+                    + "), block size: "
+                    + properties.getDataPerBatch()
+                    + ". flushing to retry queue");
+            drainBuffersToQueue(QueueingReason.BUFFER_SIZE);
+            logger.info(
+                "["
+                    + handlerKey.getHandle()
+                    + " thread "
+                    + threadId
+                    + "]: flushing to retry queue complete. Pending "
+                    + handlerKey.getEntityType()
+                    + ": "
+                    + datumSize);
+          }
         }
-        logger.info("[" + handle + " thread " + threadId + "]: flushing to retry queue complete. " +
-            "Pending " + entityType + ": " + datum.size());
+      };
+
+  abstract void flushSingleBatch(List batch, @Nullable QueueingReason reason);
+
+  public void drainBuffersToQueue(@Nullable QueueingReason reason) {
+    if (isBuffering.compareAndSet(false, true)) {
+      bufferFlushCounter.inc();
+      try {
+        int lastBatchSize = Integer.MIN_VALUE;
+        // roughly limit number of items to flush to the the current buffer size (+1
+        // blockSize max)
+        // if too many points arrive at the proxy while it's draining,
+        // they will be taken care of in the next run
+        int toFlush = datum.size();
+        while (toFlush > 0) {
+          List batch = createBatch();
+          int batchSize = batch.size();
+          if (batchSize > 0) {
+            flushSingleBatch(batch, reason);
+            // update the counters as if this was a failed call to the API
+            this.attemptedCounter.inc(batchSize);
+            toFlush -= batchSize;
+            // stop draining buffers if the batch is smaller than the previous one
+            if (batchSize < lastBatchSize) {
+              break;
+            }
+            lastBatchSize = batchSize;
+          } else {
+            break;
+          }
+        }
+      } finally {
+        isBuffering.set(false);
+        bufferCompletedFlushCounter.inc();
       }
     }
-  };
-
-  public abstract void drainBuffersToQueue();
+  }
 
   @Override
   public long getTaskRelativeScore() {
-    return datum.size() + (isBuffering ? memoryBufferLimit.get() : (isSending ? itemsPerBatch.get() / 2 : 0));
+    return datumSize
+        + (isBuffering.get()
+            ? properties.getMemoryBufferLimit()
+            : (isSending ? properties.getDataPerBatch() / 2 : 0));
+  }
+
+  /**
+   * @param datum list from which to calculate the sub-list
+   * @param ratelimit the rate limit
+   * @param batchSize the size of the batch
+   * @return size of sublist such that datum[0:i) falls within the rate limit
+   */
+  protected int getBlockSize(List datum, int ratelimit, int batchSize) {
+    return Math.min(Math.min(getDataSize(datum), ratelimit), batchSize);
   }
 
+  /**
+   * @param data the data to get the size of
+   * @return the size of the data in regard to the rate limiter
+   */
+  protected int getDataSize(List data) {
+    return data.size();
+  }
 
+  /***
+   * returns the size of the object in relation to the scale we care about
+   * default each object = 1
+   * @param object object to size
+   * @return size of object
+   */
+  protected int getObjectSize(T object) {
+    return 1;
+  }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/DelegatingReportableEntityHandlerFactoryImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/DelegatingReportableEntityHandlerFactoryImpl.java
new file mode 100644
index 000000000..0cbe1b7c0
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/DelegatingReportableEntityHandlerFactoryImpl.java
@@ -0,0 +1,28 @@
+package com.wavefront.agent.handlers;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Wrapper for {@link ReportableEntityHandlerFactory} to allow partial overrides for the {@code
+ * getHandler} method.
+ *
+ * @author vasily@wavefront.com
+ */
+public class DelegatingReportableEntityHandlerFactoryImpl
+    implements ReportableEntityHandlerFactory {
+  protected final ReportableEntityHandlerFactory delegate;
+
+  public DelegatingReportableEntityHandlerFactoryImpl(ReportableEntityHandlerFactory delegate) {
+    this.delegate = delegate;
+  }
+
+  @Override
+  public  ReportableEntityHandler getHandler(HandlerKey handlerKey) {
+    return delegate.getHandler(handlerKey);
+  }
+
+  @Override
+  public void shutdown(@Nonnull String handle) {
+    delegate.shutdown(handle);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/DeltaCounterAccumulationHandlerImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/DeltaCounterAccumulationHandlerImpl.java
new file mode 100644
index 000000000..9be041ff4
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/DeltaCounterAccumulationHandlerImpl.java
@@ -0,0 +1,221 @@
+package com.wavefront.agent.handlers;
+
+import static com.wavefront.data.Validation.validatePoint;
+import static com.wavefront.sdk.common.Utils.metricToLineData;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.RemovalListener;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.AtomicDouble;
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.api.agent.ValidationConfiguration;
+import com.wavefront.common.Clock;
+import com.wavefront.common.HostMetricTagsPair;
+import com.wavefront.common.Utils;
+import com.wavefront.data.DeltaCounterValueException;
+import com.wavefront.ingester.ReportPointSerializer;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.BurstRateTrackingCounter;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.DeltaCounter;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.MetricName;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import wavefront.report.ReportPoint;
+
+/**
+ * Handler that processes incoming DeltaCounter objects, aggregates them and hands it over to one of
+ * the {@link SenderTask} threads according to deltaCountersAggregationIntervalSeconds or before
+ * cache expires.
+ *
+ * @author djia@vmware.com
+ */
+public class DeltaCounterAccumulationHandlerImpl
+    extends AbstractReportableEntityHandler {
+
+  private final ValidationConfiguration validationConfig;
+  private final Logger validItemsLogger;
+  final Histogram receivedPointLag;
+  private final BurstRateTrackingCounter reportedStats;
+  private final Supplier discardedCounterSupplier;
+  private final Cache aggregatedDeltas;
+  private final ScheduledExecutorService reporter = Executors.newSingleThreadScheduledExecutor();
+  private final Timer receivedRateTimer;
+
+  /**
+   * @param handlerKey metrics pipeline key.
+   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into
+   *     the main log file.
+   * @param senderTaskMap map of tenant name and tasks actually handling data transfer to the
+   *     Wavefront endpoint corresponding to the tenant name
+   * @param validationConfig validation configuration.
+   * @param aggregationIntervalSeconds aggregation interval for delta counters.
+   * @param receivedRateSink where to report received rate.
+   * @param blockedItemLogger logger for blocked items.
+   * @param validItemsLogger logger for valid items.
+   */
+  public DeltaCounterAccumulationHandlerImpl(
+      final HandlerKey handlerKey,
+      final int blockedItemsPerBatch,
+      @Nullable final Map>> senderTaskMap,
+      @Nonnull final ValidationConfiguration validationConfig,
+      long aggregationIntervalSeconds,
+      @Nullable final BiConsumer receivedRateSink,
+      @Nullable final Logger blockedItemLogger,
+      @Nullable final Logger validItemsLogger) {
+    super(
+        handlerKey,
+        blockedItemsPerBatch,
+        new ReportPointSerializer(),
+        senderTaskMap,
+        true,
+        null,
+        blockedItemLogger);
+    super.initializeCounters();
+    this.validationConfig = validationConfig;
+    this.validItemsLogger = validItemsLogger;
+
+    this.aggregatedDeltas =
+        Caffeine.newBuilder()
+            .expireAfterAccess(5 * aggregationIntervalSeconds, TimeUnit.SECONDS)
+            .removalListener(
+                (RemovalListener)
+                    (metric, value, reason) -> this.reportAggregatedDeltaValue(metric, value))
+            .build();
+
+    this.receivedPointLag =
+        Metrics.newHistogram(
+            new MetricName("points." + handlerKey.getHandle() + ".received", "", "lag"), false);
+
+    reporter.scheduleWithFixedDelay(
+        this::flushDeltaCounters,
+        aggregationIntervalSeconds,
+        aggregationIntervalSeconds,
+        TimeUnit.SECONDS);
+
+    String metricPrefix = handlerKey.toString();
+    this.reportedStats =
+        new BurstRateTrackingCounter(
+            new MetricName(metricPrefix, "", "sent"), Metrics.defaultRegistry(), 1000);
+    this.discardedCounterSupplier =
+        Utils.lazySupplier(() -> Metrics.newCounter(new MetricName(metricPrefix, "", "discarded")));
+    Metrics.newGauge(
+        new MetricName(metricPrefix, "", "accumulator.size"),
+        new Gauge() {
+          @Override
+          public Long value() {
+            return aggregatedDeltas.estimatedSize();
+          }
+        });
+    if (receivedRateSink == null) {
+      this.receivedRateTimer = null;
+    } else {
+      this.receivedRateTimer = new Timer("delta-counter-timer-" + handlerKey.getHandle());
+      this.receivedRateTimer.scheduleAtFixedRate(
+          new TimerTask() {
+            @Override
+            public void run() {
+              for (String tenantName : senderTaskMap.keySet()) {
+                receivedRateSink.accept(tenantName, receivedStats.getCurrentRate());
+              }
+            }
+          },
+          1000,
+          1000);
+    }
+  }
+
+  @VisibleForTesting
+  public void flushDeltaCounters() {
+    this.aggregatedDeltas.asMap().forEach(this::reportAggregatedDeltaValue);
+  }
+
+  private void reportAggregatedDeltaValue(
+      @Nullable HostMetricTagsPair hostMetricTagsPair, @Nullable AtomicDouble value) {
+    if (value == null || hostMetricTagsPair == null) {
+      return;
+    }
+    this.reportedStats.inc();
+    double reportedValue = value.getAndSet(0);
+    if (reportedValue == 0) return;
+    String strPoint =
+        metricToLineData(
+            hostMetricTagsPair.metric,
+            reportedValue,
+            Clock.now(),
+            hostMetricTagsPair.getHost(),
+            hostMetricTagsPair.getTags(),
+            "wavefront-proxy");
+    getTask(APIContainer.CENTRAL_TENANT_NAME).add(strPoint);
+    // check if delta tag contains the tag key indicating this delta point should be multicasted
+    if (isMulticastingActive
+        && hostMetricTagsPair.getTags() != null
+        && hostMetricTagsPair.getTags().containsKey(MULTICASTING_TENANT_TAG_KEY)) {
+      String[] multicastingTenantNames =
+          hostMetricTagsPair.getTags().get(MULTICASTING_TENANT_TAG_KEY).trim().split(",");
+      hostMetricTagsPair.getTags().remove(MULTICASTING_TENANT_TAG_KEY);
+      for (String multicastingTenantName : multicastingTenantNames) {
+        // if the tenant name indicated in delta point tag is not configured, just ignore
+        if (getTask(multicastingTenantName) != null) {
+          getTask(multicastingTenantName)
+              .add(
+                  metricToLineData(
+                      hostMetricTagsPair.metric,
+                      reportedValue,
+                      Clock.now(),
+                      hostMetricTagsPair.getHost(),
+                      hostMetricTagsPair.getTags(),
+                      "wavefront-proxy"));
+        }
+      }
+    }
+  }
+
+  @Override
+  void reportInternal(ReportPoint point) {
+    if (DeltaCounter.isDelta(point.getMetric())) {
+      try {
+        validatePoint(point, validationConfig);
+      } catch (DeltaCounterValueException e) {
+        discardedCounterSupplier.get().inc();
+        return;
+      }
+      getReceivedCounter().inc();
+      double deltaValue = (double) point.getValue();
+      receivedPointLag.update(Clock.now() - point.getTimestamp());
+      HostMetricTagsPair hostMetricTagsPair =
+          new HostMetricTagsPair(point.getHost(), point.getMetric(), point.getAnnotations());
+      Objects.requireNonNull(aggregatedDeltas.get(hostMetricTagsPair, key -> new AtomicDouble(0)))
+          .getAndAdd(deltaValue);
+      if (validItemsLogger != null && validItemsLogger.isLoggable(Level.FINEST)) {
+        validItemsLogger.info(serializer.apply(point));
+      }
+    } else {
+      reject(point, "Port is not configured to accept non-delta counter data!");
+    }
+  }
+
+  @Override
+  public void shutdown() {
+    super.shutdown();
+    reporter.shutdown();
+    if (receivedRateTimer != null) {
+      receivedRateTimer.cancel();
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/EventHandlerImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/EventHandlerImpl.java
new file mode 100644
index 000000000..3deef9501
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/EventHandlerImpl.java
@@ -0,0 +1,97 @@
+package com.wavefront.agent.handlers;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.data.Validation;
+import com.wavefront.dto.Event;
+import java.util.Collection;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import wavefront.report.ReportEvent;
+
+/**
+ * This class will validate parsed events and distribute them among SenderTask threads.
+ *
+ * @author vasily@wavefront.com
+ */
+public class EventHandlerImpl extends AbstractReportableEntityHandler {
+  private static final Logger logger =
+      Logger.getLogger(AbstractReportableEntityHandler.class.getCanonicalName());
+  private static final Function EVENT_SERIALIZER =
+      value -> new Event(value).toString();
+
+  private final Logger validItemsLogger;
+
+  /**
+   * @param handlerKey pipeline key.
+   * @param blockedItemsPerBatch number of blocked items that are allowed to be written into the
+   *     main log.
+   * @param senderTaskMap map of tenant name and tasks actually handling data transfer to the
+   *     Wavefront endpoint corresponding to the tenant name
+   * @param receivedRateSink where to report received rate.
+   * @param blockedEventsLogger logger for blocked events.
+   * @param validEventsLogger logger for valid events.
+   */
+  public EventHandlerImpl(
+      final HandlerKey handlerKey,
+      final int blockedItemsPerBatch,
+      @Nullable final Map>> senderTaskMap,
+      @Nullable final BiConsumer receivedRateSink,
+      @Nullable final Logger blockedEventsLogger,
+      @Nullable final Logger validEventsLogger) {
+    super(
+        handlerKey,
+        blockedItemsPerBatch,
+        EVENT_SERIALIZER,
+        senderTaskMap,
+        true,
+        receivedRateSink,
+        blockedEventsLogger);
+    super.initializeCounters();
+    this.validItemsLogger = validEventsLogger;
+  }
+
+  @Override
+  protected void reportInternal(ReportEvent event) {
+    if (!annotationKeysAreValid(event)) {
+      throw new IllegalArgumentException("WF-401: Event annotation key has illegal characters.");
+    }
+    Event eventToAdd = new Event(event);
+    getTask(APIContainer.CENTRAL_TENANT_NAME).add(eventToAdd);
+    getReceivedCounter().inc();
+    // check if event annotations contains the tag key indicating this event should be
+    // multicasted
+    if (isMulticastingActive
+        && event.getAnnotations() != null
+        && event.getAnnotations().containsKey(MULTICASTING_TENANT_TAG_KEY)) {
+      String[] multicastingTenantNames =
+          event.getAnnotations().get(MULTICASTING_TENANT_TAG_KEY).trim().split(",");
+      event.getAnnotations().remove(MULTICASTING_TENANT_TAG_KEY);
+      for (String multicastingTenantName : multicastingTenantNames) {
+        // if the tenant name indicated in event tag is not configured, just ignore
+        if (getTask(multicastingTenantName) != null) {
+          getTask(multicastingTenantName).add(new Event(event));
+        }
+      }
+    }
+    if (validItemsLogger != null && validItemsLogger.isLoggable(Level.FINEST)) {
+      validItemsLogger.info(EVENT_SERIALIZER.apply(event));
+    }
+  }
+
+  @VisibleForTesting
+  static boolean annotationKeysAreValid(ReportEvent event) {
+    if (event.getAnnotations() != null) {
+      for (String key : event.getAnnotations().keySet()) {
+        if (!Validation.charactersAreValid(key)) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/EventSenderTask.java b/proxy/src/main/java/com/wavefront/agent/handlers/EventSenderTask.java
new file mode 100644
index 000000000..b97550113
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/EventSenderTask.java
@@ -0,0 +1,65 @@
+package com.wavefront.agent.handlers;
+
+import com.wavefront.agent.data.EntityProperties;
+import com.wavefront.agent.data.EventDataSubmissionTask;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.agent.data.TaskResult;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.api.EventAPI;
+import com.wavefront.dto.Event;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ScheduledExecutorService;
+import javax.annotation.Nullable;
+
+/**
+ * This class is responsible for accumulating events and sending them batch. This class is similar
+ * to PostPushDataTimedTask.
+ *
+ * @author vasily@wavefront.com
+ */
+class EventSenderTask extends AbstractSenderTask {
+
+  private final EventAPI proxyAPI;
+  private final UUID proxyId;
+  private final TaskQueue backlog;
+
+  /**
+   * @param handlerKey handler key, that serves as an identifier of the metrics pipeline.
+   * @param proxyAPI handles interaction with Wavefront servers as well as queueing.
+   * @param proxyId id of the proxy.
+   * @param threadId thread number.
+   * @param properties container for mutable proxy settings.
+   * @param scheduler executor service for running this task
+   * @param backlog backing queue
+   */
+  EventSenderTask(
+      HandlerKey handlerKey,
+      EventAPI proxyAPI,
+      UUID proxyId,
+      int threadId,
+      EntityProperties properties,
+      ScheduledExecutorService scheduler,
+      TaskQueue backlog) {
+    super(handlerKey, threadId, properties, scheduler);
+    this.proxyAPI = proxyAPI;
+    this.proxyId = proxyId;
+    this.backlog = backlog;
+  }
+
+  @Override
+  TaskResult processSingleBatch(List batch) {
+    EventDataSubmissionTask task =
+        new EventDataSubmissionTask(
+            proxyAPI, proxyId, properties, backlog, handlerKey.getHandle(), batch, null);
+    return task.execute();
+  }
+
+  @Override
+  public void flushSingleBatch(List batch, @Nullable QueueingReason reason) {
+    EventDataSubmissionTask task =
+        new EventDataSubmissionTask(
+            proxyAPI, proxyId, properties, backlog, handlerKey.getHandle(), batch, null);
+    task.enqueue(reason);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/HandlerKey.java b/proxy/src/main/java/com/wavefront/agent/handlers/HandlerKey.java
index 857a63130..f2b3a0a54 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/HandlerKey.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/HandlerKey.java
@@ -1,34 +1,59 @@
 package com.wavefront.agent.handlers;
 
 import com.wavefront.data.ReportableEntityType;
-
-import javax.validation.constraints.NotNull;
+import java.util.Objects;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 /**
- * An immutable unique identifier for a handler pipeline (type of objects handled + port/handle name)
+ * An immutable unique identifier for a handler pipeline (type of objects handled + port/handle name
+ * + tenant name)
  *
  * @author vasily@wavefront.com
  */
 public class HandlerKey {
   private final ReportableEntityType entityType;
-  @NotNull
-  private final String handle;
+  @Nonnull private final String handle;
+  @Nullable private final String tenantName;
 
-  private HandlerKey(ReportableEntityType entityType, @NotNull String handle) {
+  private HandlerKey(
+      ReportableEntityType entityType, @Nonnull String handle, @Nullable String tenantName) {
     this.entityType = entityType;
     this.handle = handle;
+    this.tenantName = tenantName;
+  }
+
+  public static String generateTenantSpecificHandle(String handle, @Nonnull String tenantName) {
+    return handle + "." + tenantName;
   }
 
   public ReportableEntityType getEntityType() {
     return entityType;
   }
 
+  @Nonnull
   public String getHandle() {
-    return handle;
+    return handle + (this.tenantName == null ? "" : "." + this.tenantName);
+  }
+
+  public String getTenantName() {
+    return this.tenantName;
+  }
+
+  public static HandlerKey of(ReportableEntityType entityType, @Nonnull String handle) {
+    return new HandlerKey(entityType, handle, null);
+  }
+
+  public static HandlerKey of(
+      ReportableEntityType entityType, @Nonnull String handle, @Nonnull String tenantName) {
+    return new HandlerKey(entityType, handle, tenantName);
   }
 
-  public static HandlerKey of(ReportableEntityType entityType, @NotNull String handle) {
-    return new HandlerKey(entityType, handle);
+  @Override
+  public int hashCode() {
+    return 31 * 31 * entityType.hashCode()
+        + 31 * handle.hashCode()
+        + (this.tenantName == null ? 0 : this.tenantName.hashCode());
   }
 
   @Override
@@ -37,8 +62,16 @@ public boolean equals(Object o) {
     if (o == null || getClass() != o.getClass()) return false;
     HandlerKey that = (HandlerKey) o;
     if (!entityType.equals(that.entityType)) return false;
-    if (handle != null ? !handle.equals(that.handle) : that.handle != null) return false;
+    if (!Objects.equals(handle, that.handle)) return false;
+    if (!Objects.equals(tenantName, that.tenantName)) return false;
     return true;
   }
 
+  @Override
+  public String toString() {
+    return this.entityType
+        + "."
+        + this.handle
+        + (this.tenantName == null ? "" : "." + this.tenantName);
+  }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/HistogramAccumulationHandlerImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/HistogramAccumulationHandlerImpl.java
new file mode 100644
index 000000000..b3482f8c8
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/HistogramAccumulationHandlerImpl.java
@@ -0,0 +1,144 @@
+package com.wavefront.agent.handlers;
+
+import static com.wavefront.agent.histogram.HistogramUtils.granularityToString;
+import static com.wavefront.common.Utils.lazySupplier;
+import static com.wavefront.data.Validation.validatePoint;
+
+import com.wavefront.agent.histogram.Granularity;
+import com.wavefront.agent.histogram.HistogramKey;
+import com.wavefront.agent.histogram.HistogramUtils;
+import com.wavefront.agent.histogram.accumulator.Accumulator;
+import com.wavefront.api.agent.ValidationConfiguration;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import java.util.function.BiConsumer;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import wavefront.report.Histogram;
+import wavefront.report.ReportPoint;
+
+/**
+ * A ReportPointHandler that ships parsed points to a histogram accumulator instead of forwarding
+ * them to SenderTask.
+ *
+ * @author vasily@wavefront.com
+ */
+public class HistogramAccumulationHandlerImpl extends ReportPointHandlerImpl {
+  private final Accumulator digests;
+  private final Granularity granularity;
+  // Metrics
+  private final Supplier pointCounter;
+  private final Supplier pointRejectedCounter;
+  private final Supplier histogramCounter;
+  private final Supplier histogramRejectedCounter;
+  private final Supplier histogramBinCount;
+  private final Supplier histogramSampleCount;
+
+  /**
+   * Creates a new instance
+   *
+   * @param handlerKey pipeline handler key
+   * @param digests accumulator for storing digests
+   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into
+   *     the main log file.
+   * @param granularity granularity level
+   * @param validationConfig Supplier for the ValidationConfiguration
+   * @param isHistogramInput Whether expected input data for this handler is histograms.
+   * @param receivedRateSink Where to report received rate.
+   */
+  public HistogramAccumulationHandlerImpl(
+      final HandlerKey handlerKey,
+      final Accumulator digests,
+      final int blockedItemsPerBatch,
+      @Nullable Granularity granularity,
+      @Nonnull final ValidationConfiguration validationConfig,
+      boolean isHistogramInput,
+      @Nullable final BiConsumer receivedRateSink,
+      @Nullable final Logger blockedItemLogger,
+      @Nullable final Logger validItemsLogger) {
+    super(
+        handlerKey,
+        blockedItemsPerBatch,
+        null,
+        validationConfig,
+        !isHistogramInput,
+        receivedRateSink,
+        blockedItemLogger,
+        validItemsLogger,
+        null);
+    super.initializeCounters();
+    this.digests = digests;
+    this.granularity = granularity;
+    String metricNamespace = "histogram.accumulator." + granularityToString(granularity);
+    pointCounter =
+        lazySupplier(() -> Metrics.newCounter(new MetricName(metricNamespace, "", "sample_added")));
+    pointRejectedCounter =
+        lazySupplier(
+            () -> Metrics.newCounter(new MetricName(metricNamespace, "", "sample_rejected")));
+    histogramCounter =
+        lazySupplier(
+            () -> Metrics.newCounter(new MetricName(metricNamespace, "", "histogram_added")));
+    histogramRejectedCounter =
+        lazySupplier(
+            () -> Metrics.newCounter(new MetricName(metricNamespace, "", "histogram_rejected")));
+    histogramBinCount =
+        lazySupplier(
+            () -> Metrics.newHistogram(new MetricName(metricNamespace, "", "histogram_bins")));
+    histogramSampleCount =
+        lazySupplier(
+            () -> Metrics.newHistogram(new MetricName(metricNamespace, "", "histogram_samples")));
+  }
+
+  @Override
+  protected void reportInternal(ReportPoint point) {
+    validatePoint(point, validationConfig);
+
+    if (point.getValue() instanceof Double) {
+      if (granularity == null) {
+        pointRejectedCounter.get().inc();
+        reject(point, "Wavefront data format is not supported on distribution ports!");
+        return;
+      }
+      // Get key
+      HistogramKey histogramKey = HistogramUtils.makeKey(point, granularity);
+      double value = (Double) point.getValue();
+      pointCounter.get().inc();
+
+      // atomic update
+      digests.put(histogramKey, value);
+    } else if (point.getValue() instanceof Histogram) {
+      Histogram value = (Histogram) point.getValue();
+      Granularity pointGranularity = Granularity.fromMillis(value.getDuration());
+      if (granularity != null && pointGranularity.getInMillis() > granularity.getInMillis()) {
+        reject(
+            point,
+            "Attempting to send coarser granularity ("
+                + granularityToString(pointGranularity)
+                + ") distribution to a finer granularity ("
+                + granularityToString(granularity)
+                + ") port");
+        histogramRejectedCounter.get().inc();
+        return;
+      }
+
+      histogramBinCount.get().update(value.getCounts().size());
+      histogramSampleCount.get().update(value.getCounts().stream().mapToLong(x -> x).sum());
+
+      // Key
+      HistogramKey histogramKey =
+          HistogramUtils.makeKey(point, granularity == null ? pointGranularity : granularity);
+      histogramCounter.get().inc();
+
+      // atomic update
+      digests.put(histogramKey, value);
+    }
+
+    if (validItemsLogger != null && validItemsLogger.isLoggable(Level.FINEST)) {
+      validItemsLogger.info(serializer.apply(point));
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/InternalProxyWavefrontClient.java b/proxy/src/main/java/com/wavefront/agent/handlers/InternalProxyWavefrontClient.java
new file mode 100644
index 000000000..d60127515
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/InternalProxyWavefrontClient.java
@@ -0,0 +1,173 @@
+package com.wavefront.agent.handlers;
+
+import static com.wavefront.common.Utils.lazySupplier;
+
+import com.wavefront.common.Clock;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.entities.histograms.HistogramGranularity;
+import com.wavefront.sdk.entities.tracing.SpanLog;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import wavefront.report.Histogram;
+import wavefront.report.HistogramType;
+import wavefront.report.ReportPoint;
+
+public class InternalProxyWavefrontClient implements WavefrontSender {
+  private final Supplier> pointHandlerSupplier;
+  private final Supplier> histogramHandlerSupplier;
+  private final String clientId;
+
+  public InternalProxyWavefrontClient(
+      ReportableEntityHandlerFactory handlerFactory, String handle) {
+    this.pointHandlerSupplier =
+        lazySupplier(
+            () -> handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle)));
+    this.histogramHandlerSupplier =
+        lazySupplier(
+            () -> handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM, handle)));
+    this.clientId = handle;
+  }
+
+  @Override
+  public void flush() {
+    // noop
+  }
+
+  @Override
+  public int getFailureCount() {
+    return 0;
+  }
+
+  @Override
+  public void sendDistribution(
+      String name,
+      List> centroids,
+      Set histogramGranularities,
+      Long timestamp,
+      String source,
+      Map tags) {
+    final List bins = centroids.stream().map(x -> x._1).collect(Collectors.toList());
+    final List counts = centroids.stream().map(x -> x._2).collect(Collectors.toList());
+    for (HistogramGranularity granularity : histogramGranularities) {
+      int duration;
+      switch (granularity) {
+        case MINUTE:
+          duration = 60000;
+          break;
+        case HOUR:
+          duration = 3600000;
+          break;
+        case DAY:
+          duration = 86400000;
+          break;
+        default:
+          throw new IllegalArgumentException("Unknown granularity: " + granularity);
+      }
+      Histogram histogram =
+          Histogram.newBuilder()
+              .setType(HistogramType.TDIGEST)
+              .setBins(bins)
+              .setCounts(counts)
+              .setDuration(duration)
+              .build();
+      ReportPoint point =
+          ReportPoint.newBuilder()
+              .setTable("unknown")
+              .setMetric(name)
+              .setValue(histogram)
+              .setTimestamp(timestamp)
+              .setHost(source)
+              .setAnnotations(tags)
+              .build();
+      histogramHandlerSupplier.get().report(point);
+    }
+  }
+
+  @Override
+  public void sendMetric(
+      String name, double value, Long timestamp, String source, Map tags) {
+    // default to millis
+    long timestampMillis = 0;
+    timestamp = timestamp == null ? Clock.now() : timestamp;
+    if (timestamp < 10_000_000_000L) {
+      // seconds
+      timestampMillis = timestamp * 1000;
+    } else if (timestamp < 10_000_000_000_000L) {
+      // millis
+      timestampMillis = timestamp;
+    } else if (timestamp < 10_000_000_000_000_000L) {
+      // micros
+      timestampMillis = timestamp / 1000;
+    } else if (timestamp <= 999_999_999_999_999_999L) {
+      // nanos
+      timestampMillis = timestamp / 1000_000;
+    }
+
+    final ReportPoint point =
+        ReportPoint.newBuilder()
+            .setTable("unknown")
+            .setMetric(name)
+            .setValue(value)
+            .setTimestamp(timestampMillis)
+            .setHost(source)
+            .setAnnotations(tags)
+            .build();
+    pointHandlerSupplier.get().report(point);
+  }
+
+  public void sendFormattedMetric(String s) {
+    throw new UnsupportedOperationException("Not applicable");
+  }
+
+  @Override
+  public void sendSpan(
+      String name,
+      long startMillis,
+      long durationMillis,
+      String source,
+      UUID traceId,
+      UUID spanId,
+      List parents,
+      List followsFrom,
+      List> tags,
+      @Nullable List spanLogs) {
+    throw new UnsupportedOperationException("Not applicable");
+  }
+
+  @Override
+  public void close() {
+    // noop
+  }
+
+  @Override
+  public String getClientId() {
+    return clientId;
+  }
+
+  @Override
+  public void sendEvent(
+      String name,
+      long startMillis,
+      long endMillis,
+      String source,
+      Map tags,
+      Map annotations)
+      throws IOException {
+    throw new UnsupportedOperationException("Not applicable");
+  }
+
+  @Override
+  public void sendLog(
+      String name, double value, Long timestamp, String source, Map tags)
+      throws IOException {
+    throw new UnsupportedOperationException("Not applicable");
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/LineDelimitedSenderTask.java b/proxy/src/main/java/com/wavefront/agent/handlers/LineDelimitedSenderTask.java
index 71d730ed9..5a0534495 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/LineDelimitedSenderTask.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/LineDelimitedSenderTask.java
@@ -1,26 +1,16 @@
 package com.wavefront.agent.handlers;
 
-import com.google.common.util.concurrent.RateLimiter;
-import com.google.common.util.concurrent.RecyclableRateLimiter;
-
-import com.wavefront.agent.api.ForceQueueEnabledAgentAPI;
-import com.wavefront.api.agent.Constants;
-import com.wavefront.ingester.StringLineIngester;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
-import com.yammer.metrics.core.Timer;
-import com.yammer.metrics.core.TimerContext;
-
+import com.wavefront.agent.data.EntityProperties;
+import com.wavefront.agent.data.LineDelimitedDataSubmissionTask;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.agent.data.TaskResult;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.agent.queueing.TaskSizeEstimator;
+import com.wavefront.api.ProxyV2API;
 import java.util.List;
 import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
+import java.util.concurrent.ScheduledExecutorService;
 import javax.annotation.Nullable;
-import javax.ws.rs.core.Response;
 
 /**
  * SenderTask for newline-delimited data.
@@ -29,153 +19,72 @@
  */
 class LineDelimitedSenderTask extends AbstractSenderTask {
 
-  private static final Logger logger = Logger.getLogger(LineDelimitedSenderTask.class.getCanonicalName());
-
+  private final ProxyV2API proxyAPI;
+  private final UUID proxyId;
   private final String pushFormat;
+  private final TaskSizeEstimator taskSizeEstimator;
+  private final TaskQueue backlog;
 
   /**
-   * Warn about exceeding the rate limit no more than once per 10 seconds (per thread)
-   */
-  private final RateLimiter warningMessageRateLimiter = RateLimiter.create(0.1);
-
-  private final RecyclableRateLimiter pushRateLimiter;
-
-  private final Counter permitsGranted;
-  private final Counter permitsDenied;
-  private final Counter permitsRetried;
-  private final Counter batchesSuccessful;
-  private final Counter batchesFailed;
-  private final Timer batchSendTime;
-
-  private final AtomicInteger pushFlushInterval;
-
-  private ForceQueueEnabledAgentAPI proxyAPI;
-  private UUID proxyId;
-
-
-  /**
-   * Create new LineDelimitedSenderTask instance.
-   *
-   * @param entityType        entity type that dictates the data processing flow.
-   * @param pushFormat        format parameter passed to the API endpoint.
-   * @param proxyAPI          handles interaction with Wavefront servers as well as queueing.
-   * @param proxyId           proxy ID.
-   * @param handle            handle (usually port number), that serves as an identifier for the metrics pipeline.
-   * @param threadId          thread number.
-   * @param pushRateLimiter   rate limiter to control outbound point rate.
-   * @param pushFlushInterval interval between flushes.
-   * @param itemsPerBatch     max points per flush.
-   * @param memoryBufferLimit max points in task's memory buffer before queueing.
+   * @param handlerKey pipeline handler key
+   * @param pushFormat format parameter passed to the API endpoint.
+   * @param proxyAPI handles interaction with Wavefront servers as well as queueing.
+   * @param proxyId proxy ID.
+   * @param properties container for mutable proxy settings.
+   * @param scheduler executor service for running this task
+   * @param threadId thread number.
+   * @param taskSizeEstimator optional task size estimator used to calculate approximate buffer fill
+   *     rate.
+   * @param backlog backing queue.
    */
-  LineDelimitedSenderTask(String entityType, String pushFormat, ForceQueueEnabledAgentAPI proxyAPI,
-                          UUID proxyId, String handle, int threadId,
-                          final RecyclableRateLimiter pushRateLimiter,
-                          final AtomicInteger pushFlushInterval,
-                          @Nullable final AtomicInteger itemsPerBatch,
-                          @Nullable final AtomicInteger memoryBufferLimit) {
-    super(entityType, handle, threadId, itemsPerBatch, memoryBufferLimit);
+  LineDelimitedSenderTask(
+      HandlerKey handlerKey,
+      String pushFormat,
+      ProxyV2API proxyAPI,
+      UUID proxyId,
+      final EntityProperties properties,
+      ScheduledExecutorService scheduler,
+      int threadId,
+      @Nullable final TaskSizeEstimator taskSizeEstimator,
+      TaskQueue backlog) {
+    super(handlerKey, threadId, properties, scheduler);
     this.pushFormat = pushFormat;
     this.proxyId = proxyId;
-    this.pushFlushInterval = pushFlushInterval;
     this.proxyAPI = proxyAPI;
-    this.pushRateLimiter = pushRateLimiter;
-
-
-    this.permitsGranted = Metrics.newCounter(new MetricName("limiter", "", "permits-granted"));
-    this.permitsDenied = Metrics.newCounter(new MetricName("limiter", "", "permits-denied"));
-    this.permitsRetried = Metrics.newCounter(new MetricName("limiter", "", "permits-retried"));
-    this.batchesSuccessful = Metrics.newCounter(new MetricName("push." + handle, "", "batches"));
-    this.batchesFailed = Metrics.newCounter(new MetricName("push." + handle, "", "batches-errors"));
-    this.batchSendTime = Metrics.newTimer(new MetricName("push." + handle, "", "duration"),
-        TimeUnit.MILLISECONDS, TimeUnit.MINUTES);
-
-    this.scheduler.schedule(this, pushFlushInterval.get(), TimeUnit.MILLISECONDS);
+    this.taskSizeEstimator = taskSizeEstimator;
+    this.backlog = backlog;
   }
 
   @Override
-  public void run() {
-    long nextRunMillis = this.pushFlushInterval.get();
-    isSending = true;
-    try {
-      List current = createBatch();
-      if (current.size() == 0) {
-        return;
-      }
-      if (pushRateLimiter == null || pushRateLimiter.tryAcquire(current.size())) {
-        if (pushRateLimiter != null) this.permitsGranted.inc(current.size());
-
-        TimerContext timerContext = this.batchSendTime.time();
-        Response response = null;
-        try {
-          response = proxyAPI.postPushData(
-              proxyId,
-              Constants.GRAPHITE_BLOCK_WORK_UNIT,
-              System.currentTimeMillis(),
-              pushFormat,
-              StringLineIngester.joinPushData(current));
-          int itemsInList = current.size();
-          this.attemptedCounter.inc(itemsInList);
-          if (response.getStatus() == Response.Status.NOT_ACCEPTABLE.getStatusCode()) {
-            if (pushRateLimiter != null) {
-              this.pushRateLimiter.recyclePermits(itemsInList);
-              this.permitsRetried.inc(itemsInList);
-            }
-            this.queuedCounter.inc(itemsInList);
-          }
-        } finally {
-          timerContext.stop();
-          if (response != null) response.close();
-        }
-      } else {
-        this.permitsDenied.inc(current.size());
-        // if proxy rate limit exceeded, try again in 250..500ms (to introduce some degree of fairness)
-        nextRunMillis = 250 + (int) (Math.random() * 250);
-        if (warningMessageRateLimiter.tryAcquire()) {
-          logger.warning("[" + handle + " thread " + threadId + "]: WF-4 Proxy rate limiter active " +
-              "(pending " + entityType + ": " + datum.size() + "), will retry");
-        }
-        synchronized (mutex) { // return the batch to the beginning of the queue
-          datum.addAll(0, current);
-        }
-      }
-    } catch (Throwable t) {
-      logger.log(Level.SEVERE, "Unexpected error in flush loop", t);
-    } finally {
-      isSending = false;
-      scheduler.schedule(this, nextRunMillis, TimeUnit.MILLISECONDS);
-    }
+  TaskResult processSingleBatch(List batch) {
+    LineDelimitedDataSubmissionTask task =
+        new LineDelimitedDataSubmissionTask(
+            proxyAPI,
+            proxyId,
+            properties,
+            backlog,
+            pushFormat,
+            handlerKey.getEntityType(),
+            handlerKey.getHandle(),
+            batch,
+            null);
+    if (taskSizeEstimator != null) taskSizeEstimator.scheduleTaskForSizing(task);
+    return task.execute();
   }
 
   @Override
-  public void drainBuffersToQueue() {
-    int lastBatchSize = Integer.MIN_VALUE;
-    // roughly limit number of points to flush to the the current buffer size (+1 blockSize max)
-    // if too many points arrive at the proxy while it's draining, they will be taken care of in the next run
-    int toFlush = datum.size();
-    while (toFlush > 0) {
-      List pushData = createBatch();
-      int pushDataPointCount = pushData.size();
-      if (pushDataPointCount > 0) {
-        proxyAPI.postPushData(proxyId, Constants.GRAPHITE_BLOCK_WORK_UNIT,
-            System.currentTimeMillis(), pushFormat,
-            StringLineIngester.joinPushData(pushData), true);
-
-        // update the counters as if this was a failed call to the API
-        this.attemptedCounter.inc(pushDataPointCount);
-        this.queuedCounter.inc(pushDataPointCount);
-        if (pushRateLimiter != null) {
-          this.permitsDenied.inc(pushDataPointCount);
-        }
-        toFlush -= pushDataPointCount;
-
-        // stop draining buffers if the batch is smaller than the previous one
-        if (pushDataPointCount < lastBatchSize) {
-          break;
-        }
-        lastBatchSize = pushDataPointCount;
-      } else {
-        break;
-      }
-    }
+  void flushSingleBatch(List batch, @Nullable QueueingReason reason) {
+    LineDelimitedDataSubmissionTask task =
+        new LineDelimitedDataSubmissionTask(
+            proxyAPI,
+            proxyId,
+            properties,
+            backlog,
+            pushFormat,
+            handlerKey.getEntityType(),
+            handlerKey.getHandle(),
+            batch,
+            null);
+    task.enqueue(reason);
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/LineDelimitedUtils.java b/proxy/src/main/java/com/wavefront/agent/handlers/LineDelimitedUtils.java
new file mode 100644
index 000000000..d506aa8c5
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/LineDelimitedUtils.java
@@ -0,0 +1,36 @@
+package com.wavefront.agent.handlers;
+
+import java.util.Collection;
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * A collection of helper methods around plaintext newline-delimited payloads.
+ *
+ * @author vasily@wavefront.com
+ */
+public abstract class LineDelimitedUtils {
+  static final String PUSH_DATA_DELIMITER = "\n";
+
+  private LineDelimitedUtils() {}
+
+  /**
+   * Split a newline-delimited payload into a string array.
+   *
+   * @param pushData payload to split.
+   * @return string array
+   */
+  @Deprecated
+  public static String[] splitPushData(String pushData) {
+    return StringUtils.split(pushData, PUSH_DATA_DELIMITER);
+  }
+
+  /**
+   * Join a batch of strings into a payload string.
+   *
+   * @param pushData collection of strings.
+   * @return payload
+   */
+  public static String joinPushData(Collection pushData) {
+    return StringUtils.join(pushData, PUSH_DATA_DELIMITER);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/LogSenderTask.java b/proxy/src/main/java/com/wavefront/agent/handlers/LogSenderTask.java
new file mode 100644
index 000000000..313c6e758
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/LogSenderTask.java
@@ -0,0 +1,90 @@
+package com.wavefront.agent.handlers;
+
+import com.wavefront.agent.data.EntityProperties;
+import com.wavefront.agent.data.LogDataSubmissionTask;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.agent.data.TaskResult;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.api.LogAPI;
+import com.wavefront.dto.Log;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ScheduledExecutorService;
+import javax.annotation.Nullable;
+
+/**
+ * This class is responsible for accumulating logs and uploading them in batches.
+ *
+ * @author amitw@vmware.com
+ */
+public class LogSenderTask extends AbstractSenderTask {
+  private final LogAPI logAPI;
+  private final UUID proxyId;
+  private final TaskQueue backlog;
+
+  /**
+   * @param handlerKey handler key, that serves as an identifier of the log pipeline.
+   * @param logAPI handles interaction with log systems as well as queueing.
+   * @param proxyId id of the proxy.
+   * @param threadId thread number.
+   * @param properties container for mutable proxy settings.
+   * @param scheduler executor service for running this task
+   * @param backlog backing queue
+   */
+  LogSenderTask(
+      HandlerKey handlerKey,
+      LogAPI logAPI,
+      UUID proxyId,
+      int threadId,
+      EntityProperties properties,
+      ScheduledExecutorService scheduler,
+      TaskQueue backlog) {
+    super(handlerKey, threadId, properties, scheduler);
+    this.logAPI = logAPI;
+    this.proxyId = proxyId;
+    this.backlog = backlog;
+  }
+
+  @Override
+  TaskResult processSingleBatch(List batch) {
+    LogDataSubmissionTask task =
+        new LogDataSubmissionTask(
+            logAPI, proxyId, properties, backlog, handlerKey.getHandle(), batch, null);
+    return task.execute();
+  }
+
+  @Override
+  public void flushSingleBatch(List batch, @Nullable QueueingReason reason) {
+    LogDataSubmissionTask task =
+        new LogDataSubmissionTask(
+            logAPI, proxyId, properties, backlog, handlerKey.getHandle(), batch, null);
+    task.enqueue(reason);
+  }
+
+  @Override
+  protected int getDataSize(List batch) {
+    int size = 0;
+    for (Log l : batch) {
+      size += l.getDataSize();
+    }
+    return size;
+  }
+
+  @Override
+  protected int getBlockSize(List datum, int rateLimit, int batchSize) {
+    int maxDataSize = Math.min(rateLimit, batchSize);
+    int size = 0;
+    for (int i = 0; i < datum.size(); i++) {
+      size += datum.get(i).getDataSize();
+      if (size > maxDataSize) {
+        return i;
+      }
+    }
+    return datum.size();
+  }
+
+  @Override
+  protected int getObjectSize(Log object) {
+    return object.getDataSize();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/ReportLogHandlerImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/ReportLogHandlerImpl.java
new file mode 100644
index 000000000..f92d42d1c
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/ReportLogHandlerImpl.java
@@ -0,0 +1,135 @@
+package com.wavefront.agent.handlers;
+
+import static com.wavefront.agent.LogsUtil.getOrCreateLogsCounterFromRegistry;
+import static com.wavefront.agent.LogsUtil.getOrCreateLogsHistogramFromRegistry;
+import static com.wavefront.data.Validation.validateLog;
+
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.agent.formatter.DataFormat;
+import com.wavefront.api.agent.ValidationConfiguration;
+import com.wavefront.common.Clock;
+import com.wavefront.dto.Log;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.BurstRateTrackingCounter;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricsRegistry;
+import java.util.Collection;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import wavefront.report.Annotation;
+import wavefront.report.ReportLog;
+
+/**
+ * This class will validate parsed logs and distribute them among SenderTask threads.
+ *
+ * @author amitw@vmware.com
+ */
+public class ReportLogHandlerImpl extends AbstractReportableEntityHandler {
+  private static final Function LOG_SERIALIZER =
+      value -> new Log(value).toString();
+
+  private final Logger validItemsLogger;
+  final ValidationConfiguration validationConfig;
+  private final MetricsRegistry registry;
+  private DataFormat format;
+  /**
+   * @param senderTaskMap sender tasks.
+   * @param handlerKey pipeline key.
+   * @param blockedItemsPerBatch number of blocked items that are allowed to be written into the
+   *     main log.
+   * @param validationConfig validation configuration.
+   * @param setupMetrics Whether we should report counter metrics.
+   * @param receivedRateSink where to report received rate.
+   * @param blockedLogsLogger logger for blocked logs.
+   * @param validLogsLogger logger for valid logs.
+   */
+  public ReportLogHandlerImpl(
+      final HandlerKey handlerKey,
+      final int blockedItemsPerBatch,
+      @Nullable final Map>> senderTaskMap,
+      @Nonnull final ValidationConfiguration validationConfig,
+      final boolean setupMetrics,
+      @Nullable final BiConsumer receivedRateSink,
+      @Nullable final Logger blockedLogsLogger,
+      @Nullable final Logger validLogsLogger) {
+    super(
+        handlerKey,
+        blockedItemsPerBatch,
+        LOG_SERIALIZER,
+        senderTaskMap,
+        true,
+        receivedRateSink,
+        blockedLogsLogger);
+    this.validItemsLogger = validLogsLogger;
+    this.validationConfig = validationConfig;
+    registry = setupMetrics ? Metrics.defaultRegistry() : LOCAL_REGISTRY;
+  }
+
+  @Override
+  protected void initializeCounters() {
+    this.blockedCounter =
+        getOrCreateLogsCounterFromRegistry(registry, format, metricPrefix, "blocked");
+    this.rejectedCounter =
+        getOrCreateLogsCounterFromRegistry(registry, format, metricPrefix, "rejected");
+    if (format == DataFormat.LOGS_JSON_CLOUDWATCH) {
+      MetricName receivedMetricName =
+          new MetricName(metricPrefix + "." + format.name().toLowerCase(), "", "received");
+      registry.newCounter(receivedMetricName).inc();
+      BurstRateTrackingCounter receivedStats =
+          new BurstRateTrackingCounter(receivedMetricName, registry, 1000);
+      registry.newGauge(
+          new MetricName(
+              metricPrefix + "." + format.name().toLowerCase(), "", "received.max-burst-rate"),
+          new Gauge() {
+            @Override
+            public Double value() {
+              return receivedStats.getMaxBurstRateAndClear();
+            }
+          });
+    }
+  }
+
+  @Override
+  protected void reportInternal(ReportLog log) {
+    initializeCounters();
+    getOrCreateLogsHistogramFromRegistry(registry, format, metricPrefix + ".received", "tagCount")
+        .update(log.getAnnotations().size());
+
+    getOrCreateLogsHistogramFromRegistry(
+            registry, format, metricPrefix + ".received", "messageLength")
+        .update(log.getMessage().length());
+
+    Histogram receivedTagLength =
+        getOrCreateLogsHistogramFromRegistry(
+            registry, format, metricPrefix + ".received", "tagLength");
+    for (Annotation a : log.getAnnotations()) {
+      receivedTagLength.update(a.getValue().length());
+    }
+
+    validateLog(log, validationConfig);
+    getOrCreateLogsHistogramFromRegistry(registry, format, metricPrefix + ".received", "lag")
+        .update(Clock.now() - log.getTimestamp());
+
+    Log logObj = new Log(log);
+    getOrCreateLogsCounterFromRegistry(registry, format, metricPrefix + ".received", "bytes")
+        .inc(logObj.getDataSize());
+    getTask(APIContainer.CENTRAL_TENANT_NAME).add(logObj);
+    getReceivedCounter().inc();
+    attemptedCounter.inc();
+    if (validItemsLogger != null && validItemsLogger.isLoggable(Level.FINEST)) {
+      validItemsLogger.info(LOG_SERIALIZER.apply(log));
+    }
+  }
+
+  @Override
+  public void setLogFormat(DataFormat format) {
+    this.format = format;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/ReportPointHandlerImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/ReportPointHandlerImpl.java
index 30dd75ba4..e4dc4536c 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/ReportPointHandlerImpl.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/ReportPointHandlerImpl.java
@@ -1,118 +1,123 @@
 package com.wavefront.agent.handlers;
 
+import static com.wavefront.data.Validation.validatePoint;
+
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.api.agent.ValidationConfiguration;
 import com.wavefront.common.Clock;
-import com.wavefront.data.ReportableEntityType;
+import com.wavefront.common.Utils;
+import com.wavefront.data.DeltaCounterValueException;
 import com.wavefront.ingester.ReportPointSerializer;
-import com.wavefront.data.Validation;
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.Histogram;
 import com.yammer.metrics.core.MetricName;
-
-import org.apache.commons.lang.math.NumberUtils;
-
+import com.yammer.metrics.core.MetricsRegistry;
 import java.util.Collection;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.logging.Logger;
-
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import wavefront.report.Histogram;
 import wavefront.report.ReportPoint;
 
-import static com.wavefront.data.Validation.validatePoint;
-
 /**
  * Handler that processes incoming ReportPoint objects, validates them and hands them over to one of
  * the {@link SenderTask} threads.
  *
  * @author vasily@wavefront.com
  */
-class ReportPointHandlerImpl extends AbstractReportableEntityHandler {
-
-  private static final Logger logger = Logger.getLogger(AbstractReportableEntityHandler.class.getCanonicalName());
-  private static final Logger validPointsLogger = Logger.getLogger("RawValidPoints");
-  private static final Random RANDOM = new Random();
-
-  private final Counter attemptedCounter;
-  private final Counter queuedCounter;
-  private final Histogram receivedPointLag;
+class ReportPointHandlerImpl extends AbstractReportableEntityHandler {
 
-  private boolean logData = false;
-  private final double logSampleRate;
-  private volatile long logStateUpdatedMillis = 0L;
-
-  /**
-   * Value of system property wavefront.proxy.logpoints (for backwards compatibility)
-   */
-  private final boolean logPointsFlag;
+  final Logger validItemsLogger;
+  final ValidationConfiguration validationConfig;
+  final Function recompressor;
+  final com.yammer.metrics.core.Histogram receivedPointLag;
+  final com.yammer.metrics.core.Histogram receivedTagCount;
+  final Supplier discardedCounterSupplier;
 
   /**
-   * Create new instance.
+   * Creates a new instance that handles either histograms or points.
    *
-   * @param handle               handle/port number
-   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into the main log file.
-   * @param senderTasks          sender tasks
+   * @param handlerKey handler key for the metrics pipeline.
+   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into
+   *     the main log file.
+   * @param senderTaskMap map of tenant name and tasks actually handling data transfer to the
+   *     Wavefront endpoint corresponding to the tenant name
+   * @param validationConfig validation configuration.
+   * @param setupMetrics Whether we should report counter metrics.
+   * @param receivedRateSink Where to report received rate.
+   * @param blockedItemLogger logger for blocked items (optional).
+   * @param validItemsLogger sampling logger for valid items (optional).
+   * @param recompressor histogram recompressor (optional)
    */
-  ReportPointHandlerImpl(final String handle,
-                         final int blockedItemsPerBatch,
-                         final Collection senderTasks) {
-    super(ReportableEntityType.POINT, handle, blockedItemsPerBatch, new ReportPointSerializer(), senderTasks);
-    String logPointsProperty = System.getProperty("wavefront.proxy.logpoints");
-    this.logPointsFlag = logPointsProperty != null && logPointsProperty.equalsIgnoreCase("true");
-    String logPointsSampleRateProperty = System.getProperty("wavefront.proxy.logpoints.sample-rate");
-    this.logSampleRate = NumberUtils.isNumber(logPointsSampleRateProperty) ?
-        Double.parseDouble(logPointsSampleRateProperty) : 1.0d;
-
-    this.receivedPointLag = Metrics.newHistogram(new MetricName("points." + handle + ".received", "", "lag"));
-    this.attemptedCounter = Metrics.newCounter(new MetricName("points." + handle, "", "sent"));
-    this.queuedCounter = Metrics.newCounter(new MetricName("points." + handle, "", "queued"));
-
-    this.statisticOutputExecutor.scheduleAtFixedRate(this::printStats, 10, 10, TimeUnit.SECONDS);
-    this.statisticOutputExecutor.scheduleAtFixedRate(this::printTotal, 1, 1, TimeUnit.MINUTES);
+  ReportPointHandlerImpl(
+      final HandlerKey handlerKey,
+      final int blockedItemsPerBatch,
+      @Nullable final Map>> senderTaskMap,
+      @Nonnull final ValidationConfiguration validationConfig,
+      final boolean setupMetrics,
+      @Nullable final BiConsumer receivedRateSink,
+      @Nullable final Logger blockedItemLogger,
+      @Nullable final Logger validItemsLogger,
+      @Nullable final Function recompressor) {
+    super(
+        handlerKey,
+        blockedItemsPerBatch,
+        new ReportPointSerializer(),
+        senderTaskMap,
+        setupMetrics,
+        receivedRateSink,
+        blockedItemLogger);
+    super.initializeCounters();
+    this.validationConfig = validationConfig;
+    this.validItemsLogger = validItemsLogger;
+    this.recompressor = recompressor;
+    MetricsRegistry registry = setupMetrics ? Metrics.defaultRegistry() : LOCAL_REGISTRY;
+    this.receivedPointLag =
+        registry.newHistogram(
+            new MetricName(handlerKey.toString() + ".received", "", "lag"), false);
+    this.receivedTagCount =
+        registry.newHistogram(
+            new MetricName(handlerKey.toString() + ".received", "", "tagCount"), false);
+    this.discardedCounterSupplier =
+        Utils.lazySupplier(
+            () -> Metrics.newCounter(new MetricName(handlerKey.toString(), "", "discarded")));
   }
 
   @Override
-  @SuppressWarnings("unchecked")
   void reportInternal(ReportPoint point) {
-    validatePoint(point, handle, Validation.Level.NUMERIC_ONLY);
-
-    String strPoint = serializer.apply(point);
-
-    refreshValidPointsLoggerState();
-
-    if ((logData || logPointsFlag) &&
-        (logSampleRate >= 1.0d || (logSampleRate > 0.0d && RANDOM.nextDouble() < logSampleRate))) {
-      // we log valid points only if system property wavefront.proxy.logpoints is true or RawValidPoints log level is
-      // set to "ALL". this is done to prevent introducing overhead and accidentally logging points to the main log
-      // Additionally, honor sample rate limit, if set.
-      validPointsLogger.info(strPoint);
+    receivedTagCount.update(point.getAnnotations().size());
+    try {
+      validatePoint(point, validationConfig);
+    } catch (DeltaCounterValueException e) {
+      discardedCounterSupplier.get().inc();
+      return;
     }
-    getTask().add(strPoint);
-    receivedCounter.inc();
     receivedPointLag.update(Clock.now() - point.getTimestamp());
-  }
-
-  private void refreshValidPointsLoggerState() {
-    if (logStateUpdatedMillis + TimeUnit.SECONDS.toMillis(1) < System.currentTimeMillis()) {
-      // refresh validPointsLogger level once a second
-      if (logData != validPointsLogger.isLoggable(Level.FINEST)) {
-        logData = !logData;
-        logger.info("Valid points logging is now " + (logData ?
-            "enabled with " + (logSampleRate * 100) + "% sampling":
-            "disabled"));
+    if (point.getValue() instanceof Histogram && recompressor != null) {
+      Histogram histogram = (Histogram) point.getValue();
+      point.setValue(recompressor.apply(histogram));
+    }
+    final String strPoint = serializer.apply(point);
+    getTask(APIContainer.CENTRAL_TENANT_NAME).add(strPoint);
+    getReceivedCounter().inc();
+    // check if data points contains the tag key indicating this point should be multicasted
+    if (isMulticastingActive
+        && point.getAnnotations() != null
+        && point.getAnnotations().containsKey(MULTICASTING_TENANT_TAG_KEY)) {
+      String[] multicastingTenantNames =
+          point.getAnnotations().get(MULTICASTING_TENANT_TAG_KEY).trim().split(",");
+      point.getAnnotations().remove(MULTICASTING_TENANT_TAG_KEY);
+      for (String multicastingTenantName : multicastingTenantNames) {
+        // if the tenant name indicated in point tag is not configured, just ignore
+        if (getTask(multicastingTenantName) != null) {
+          getTask(multicastingTenantName).add(serializer.apply(point));
+        }
       }
-      logStateUpdatedMillis = System.currentTimeMillis();
     }
-  }
-
-  private void printStats() {
-    logger.info("[" + this.handle + "] Points received rate: " + this.getReceivedOneMinuteRate() +
-        " pps (1 min), " + getReceivedFiveMinuteRate() + " pps (5 min), " +
-        this.receivedBurstRateCurrent + " pps (current).");
-  }
-
-  private void printTotal() {
-    logger.info("[" + this.handle + "] Total points processed since start: " + this.attemptedCounter.count() +
-        "; blocked: " + this.blockedCounter.count());
+    if (validItemsLogger != null) validItemsLogger.info(strPoint);
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/ReportSourceTagHandlerImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/ReportSourceTagHandlerImpl.java
index 576e65208..2e891803b 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/ReportSourceTagHandlerImpl.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/ReportSourceTagHandlerImpl.java
@@ -1,19 +1,19 @@
 package com.wavefront.agent.handlers;
 
 import com.google.common.annotations.VisibleForTesting;
-
-import com.wavefront.data.ReportableEntityType;
-import com.wavefront.ingester.ReportSourceTagSerializer;
+import com.wavefront.agent.api.APIContainer;
 import com.wavefront.data.Validation;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
-
+import com.wavefront.dto.SourceTag;
+import java.util.ArrayList;
 import java.util.Collection;
-import java.util.concurrent.TimeUnit;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
 import java.util.logging.Logger;
-
+import javax.annotation.Nullable;
 import wavefront.report.ReportSourceTag;
+import wavefront.report.SourceOperationType;
 
 /**
  * This class will validate parsed source tags and distribute them among SenderTask threads.
@@ -21,57 +21,49 @@
  * @author Suranjan Pramanik (suranjan@wavefront.com).
  * @author vasily@wavefront.com
  */
-public class ReportSourceTagHandlerImpl extends AbstractReportableEntityHandler {
-
-  private static final Logger logger = Logger.getLogger(AbstractReportableEntityHandler.class.getCanonicalName());
-
-  private final Counter attemptedCounter;
-  private final Counter queuedCounter;
-
-  public ReportSourceTagHandlerImpl(final String handle, final int blockedItemsPerBatch,
-                                    final Collection senderTasks) {
-    super(ReportableEntityType.SOURCE_TAG, handle, blockedItemsPerBatch, new ReportSourceTagSerializer(), senderTasks);
-    this.attemptedCounter = Metrics.newCounter(new MetricName("sourceTags." + handle, "", "sent"));
-    this.queuedCounter = Metrics.newCounter(new MetricName("sourceTags." + handle, "", "queued"));
+class ReportSourceTagHandlerImpl
+    extends AbstractReportableEntityHandler {
+  private static final Function SOURCE_TAG_SERIALIZER =
+      value -> new SourceTag(value).toString();
 
-    statisticOutputExecutor.scheduleAtFixedRate(this::printStats, 10, 10, TimeUnit.SECONDS);
-    statisticOutputExecutor.scheduleAtFixedRate(this::printTotal, 1, 1, TimeUnit.MINUTES);
+  public ReportSourceTagHandlerImpl(
+      HandlerKey handlerKey,
+      final int blockedItemsPerBatch,
+      @Nullable final Map>> senderTaskMap,
+      @Nullable final BiConsumer receivedRateSink,
+      final Logger blockedItemLogger) {
+    super(
+        handlerKey,
+        blockedItemsPerBatch,
+        SOURCE_TAG_SERIALIZER,
+        senderTaskMap,
+        true,
+        receivedRateSink,
+        blockedItemLogger);
+    super.initializeCounters();
   }
 
   @Override
-  @SuppressWarnings("unchecked")
   protected void reportInternal(ReportSourceTag sourceTag) {
-    if (!annotationKeysAreValid(sourceTag)) {
-      throw new IllegalArgumentException("WF-401: SourceTag annotation key has illegal characters.");
+    if (!annotationsAreValid(sourceTag)) {
+      throw new IllegalArgumentException(
+          "WF-401: SourceTag annotation key has illegal characters.");
     }
-    getTask(sourceTag).add(sourceTag);
+    getTask(sourceTag).add(new SourceTag(sourceTag));
+    getReceivedCounter().inc();
+    // tagK=tagV based multicasting is not support
   }
 
   @VisibleForTesting
-  static boolean annotationKeysAreValid(ReportSourceTag sourceTag) {
-    if (sourceTag.getAnnotations() != null) {
-      for (String key : sourceTag.getAnnotations()) {
-        if (!Validation.charactersAreValid(key)) {
-          return false;
-        }
-      }
-    }
-    return true;
-  }
-
-  private void printStats() {
-    logger.info("[" + this.handle + "] sourceTags received rate: " + getReceivedOneMinuteRate() +
-        " pps (1 min), " + getReceivedFiveMinuteRate() + " pps (5 min), " +
-        this.receivedBurstRateCurrent + " pps (current).");
-  }
-
-  private void printTotal() {
-    logger.info("[" + this.handle + "] Total sourceTags processed since start: " + this.attemptedCounter.count() +
-        "; blocked: " + this.blockedCounter.count());
+  static boolean annotationsAreValid(ReportSourceTag sourceTag) {
+    if (sourceTag.getOperation() == SourceOperationType.SOURCE_DESCRIPTION) return true;
+    return sourceTag.getAnnotations().stream().allMatch(Validation::charactersAreValid);
   }
 
-  private SenderTask getTask(ReportSourceTag sourceTag) {
+  private SenderTask getTask(ReportSourceTag sourceTag) {
     // we need to make sure the we preserve the order of operations for each source
+    List> senderTasks =
+        new ArrayList<>(senderTaskMap.get(APIContainer.CENTRAL_TENANT_NAME));
     return senderTasks.get(Math.abs(sourceTag.getSource().hashCode()) % senderTasks.size());
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/ReportSourceTagSenderTask.java b/proxy/src/main/java/com/wavefront/agent/handlers/ReportSourceTagSenderTask.java
deleted file mode 100644
index f502d5ba1..000000000
--- a/proxy/src/main/java/com/wavefront/agent/handlers/ReportSourceTagSenderTask.java
+++ /dev/null
@@ -1,188 +0,0 @@
-package com.wavefront.agent.handlers;
-
-import com.google.common.util.concurrent.RateLimiter;
-import com.google.common.util.concurrent.RecyclableRateLimiter;
-
-import com.wavefront.agent.api.ForceQueueEnabledAgentAPI;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
-import com.yammer.metrics.core.Timer;
-import com.yammer.metrics.core.TimerContext;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.ws.rs.core.Response;
-
-import wavefront.report.ReportSourceTag;
-
-/**
- * This class is responsible for accumulating the source tag changes and post it in a batch. This
- * class is similar to PostPushDataTimedTask.
- *
- * @author Suranjan Pramanik (suranjan@wavefront.com)
- * @author vasily@wavefront.com
- */
-class ReportSourceTagSenderTask extends AbstractSenderTask {
-  private static final Logger logger = Logger.getLogger(ReportSourceTagSenderTask.class.getCanonicalName());
-
-  /**
-   * Warn about exceeding the rate limit no more than once per 10 seconds (per thread)
-   */
-  private final RateLimiter warningMessageRateLimiter = RateLimiter.create(0.1);
-
-  private final Timer batchSendTime;
-
-  private final ForceQueueEnabledAgentAPI proxyAPI;
-  private final AtomicInteger pushFlushInterval;
-  private final RecyclableRateLimiter rateLimiter;
-  private final Counter permitsGranted;
-  private final Counter permitsDenied;
-  private final Counter permitsRetried;
-
-  /**
-   * Create new instance
-   *
-   * @param proxyAPI          handles interaction with Wavefront servers as well as queueing.
-   * @param handle            handle (usually port number), that serves as an identifier for the metrics pipeline.
-   * @param threadId          thread number.
-   * @param rateLimiter       rate limiter to control outbound point rate.
-   * @param pushFlushInterval interval between flushes.
-   * @param itemsPerBatch     max points per flush.
-   * @param memoryBufferLimit max points in task's memory buffer before queueing.
-   *
-   */
-  ReportSourceTagSenderTask(ForceQueueEnabledAgentAPI proxyAPI, String handle, int threadId,
-                            AtomicInteger pushFlushInterval,
-                            @Nullable RecyclableRateLimiter rateLimiter,
-                            @Nullable AtomicInteger itemsPerBatch,
-                            @Nullable AtomicInteger memoryBufferLimit) {
-    super("sourceTags", handle, threadId, itemsPerBatch, memoryBufferLimit);
-    this.proxyAPI = proxyAPI;
-    this.batchSendTime = Metrics.newTimer(new MetricName("api.sourceTags." + handle, "", "duration"),
-        TimeUnit.MILLISECONDS, TimeUnit.MINUTES);
-    this.pushFlushInterval = pushFlushInterval;
-    this.rateLimiter = rateLimiter;
-
-    this.permitsGranted = Metrics.newCounter(new MetricName("limiter", "", "permits-granted"));
-    this.permitsDenied = Metrics.newCounter(new MetricName("limiter", "", "permits-denied"));
-    this.permitsRetried = Metrics.newCounter(new MetricName("limiter", "", "permits-retried"));
-
-
-    this.scheduler.schedule(this, this.pushFlushInterval.get(), TimeUnit.MILLISECONDS);
-  }
-
-  @Override
-  public void run() {
-    long nextRunMillis = this.pushFlushInterval.get();
-    isSending = true;
-    try {
-      List current = createBatch();
-      if (current.size() == 0) return;
-      Response response = null;
-      boolean forceToQueue = false;
-      Iterator iterator = current.iterator();
-      while (iterator.hasNext()) {
-        TimerContext timerContext = this.batchSendTime.time();
-        if (rateLimiter == null || rateLimiter.tryAcquire()) {
-          if (rateLimiter != null) this.permitsGranted.inc();
-          ReportSourceTag sourceTag = iterator.next();
-
-          try {
-            response = executeSourceTagAction(proxyAPI, sourceTag, forceToQueue);
-            this.attemptedCounter.inc();
-            if (response != null && response.getStatus() == Response.Status.NOT_ACCEPTABLE.getStatusCode()) {
-              if (rateLimiter != null) {
-                this.rateLimiter.recyclePermits(1);
-                this.permitsRetried.inc(1);
-              }
-              this.queuedCounter.inc();
-              forceToQueue = true;
-            }
-          } finally {
-            timerContext.stop();
-            if (response != null) response.close();
-          }
-        } else {
-          final List remainingItems = new ArrayList<>();
-          iterator.forEachRemaining(remainingItems::add);
-          permitsDenied.inc(remainingItems.size());
-          nextRunMillis = 250 + (int) (Math.random() * 250);
-          if (warningMessageRateLimiter.tryAcquire()) {
-            logger.warning("[" + handle + " thread " + threadId + "]: WF-4 Proxy rate limiter active " +
-                "(pending " + entityType + ": " + datum.size() + "), will retry");
-          }
-          synchronized (mutex) { // return the batch to the beginning of the queue
-            datum.addAll(0, remainingItems);
-          }
-        }
-      }
-    } catch (Throwable t) {
-      logger.log(Level.SEVERE, "Unexpected error in flush loop", t);
-    } finally {
-      isSending = false;
-      scheduler.schedule(this, nextRunMillis, TimeUnit.MILLISECONDS);
-    }
-  }
-
-  @Override
-  public void drainBuffersToQueue() {
-    int lastBatchSize = Integer.MIN_VALUE;
-    // roughly limit number of points to flush to the the current buffer size (+1 blockSize max)
-    // if too many points arrive at the proxy while it's draining, they will be taken care of in the next run
-    int toFlush = datum.size();
-    while (toFlush > 0) {
-      List items = createBatch();
-      int batchSize = items.size();
-      if (batchSize == 0) return;
-      for (ReportSourceTag sourceTag : items) {
-        executeSourceTagAction(proxyAPI, sourceTag, true);
-        this.attemptedCounter.inc();
-        this.queuedCounter.inc();
-      }
-      toFlush -= batchSize;
-
-      // stop draining buffers if the batch is smaller than the previous one
-      if (batchSize < lastBatchSize) {
-        break;
-      }
-      lastBatchSize = batchSize;
-    }
-  }
-
-  @Nullable
-  protected static Response executeSourceTagAction(ForceQueueEnabledAgentAPI wavefrontAPI, ReportSourceTag sourceTag,
-                                                boolean forceToQueue) {
-    switch (sourceTag.getSourceTagLiteral()) {
-      case "SourceDescription":
-        if (sourceTag.getAction().equals("delete")) {
-          return wavefrontAPI.removeDescription(sourceTag.getSource(), forceToQueue);
-        } else {
-          return wavefrontAPI.setDescription(sourceTag.getSource(), sourceTag.getDescription(), forceToQueue);
-        }
-      case "SourceTag":
-        if (sourceTag.getAction().equals("delete")) {
-          // call the api, if we receive a 406 message then we add them to the queue
-          // TODO: right now it only deletes the first tag (because that server-side api
-          // only handles one tag at a time. Once the server-side api is updated we
-          // should update this code to remove multiple tags at a time.
-          return wavefrontAPI.removeTag(sourceTag.getSource(), sourceTag.getAnnotations().get(0), forceToQueue);
-
-        } else { //
-          // call the api, if we receive a 406 message then we add them to the queue
-          return wavefrontAPI.setTags(sourceTag.getSource(), sourceTag.getAnnotations(), forceToQueue);
-        }
-      default:
-        logger.warning("None of the literals matched. Expected SourceTag or " +
-            "SourceDescription. Input = " + sourceTag);
-        return null;
-    }
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandler.java b/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandler.java
index 333063590..e02fb4782 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandler.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandler.java
@@ -1,19 +1,17 @@
 package com.wavefront.agent.handlers;
 
-import java.util.function.Function;
-
+import com.wavefront.agent.formatter.DataFormat;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
 
 /**
- * Handler that processes incoming objects of a single entity type, validates them and hands them over to one of
- * the {@link SenderTask} threads.
+ * Handler that processes incoming objects of a single entity type, validates them and hands them
+ * over to one of the {@link SenderTask} threads.
  *
  * @author vasily@wavefront.com
- *
  * @param  the type of input objects handled.
  */
-public interface ReportableEntityHandler {
+public interface ReportableEntityHandler {
 
   /**
    * Validate and accept the input object.
@@ -23,39 +21,22 @@ public interface ReportableEntityHandler {
   void report(T t);
 
   /**
-   * Validate and accept the input object. If validation fails, convert messageObject to string and write to log.
-   *
-   * @param t                 object to accept.
-   * @param messageObject     object to write to log if validation fails.
-   * @param messageSerializer function to convert messageObject to string.
-   */
-  void report(T t, @Nullable Object messageObject, @NotNull Function messageSerializer);
-
-
-  /**
-   * Handle the input object as blocked. Blocked objects are otherwise valid objects that are rejected based on
-   * user-defined criteria.
+   * Handle the input object as blocked. Blocked objects are otherwise valid objects that are
+   * rejected based on user-defined criteria.
    *
    * @param t object to block.
    */
   void block(T t);
 
   /**
-   * Handle the input object as blocked. Blocked objects are otherwise valid objects that are rejected based on
-   * user-defined criteria.
+   * Handle the input object as blocked. Blocked objects are otherwise valid objects that are
+   * rejected based on user-defined criteria.
    *
-   * @param t       object to block.
+   * @param t object to block.
    * @param message message to write to the main log.
    */
   void block(@Nullable T t, @Nullable String message);
 
-  /**
-   * Reject the input object as invalid, i.e. rejected based on criteria defined by Wavefront.
-   *
-   * @param t object to reject.
-   */
-  void reject(T t);
-
   /**
    * Reject the input object as invalid, i.e. rejected based on criteria defined by Wavefront.
    *
@@ -70,5 +51,10 @@ public interface ReportableEntityHandler {
    * @param t string to reject and to write to RawBlockedPointsLog.
    * @param message more user-friendly message to write to the main log.
    */
-  void reject(@NotNull String t, @Nullable String message);
+  void reject(@Nonnull String t, @Nullable String message);
+
+  void setLogFormat(DataFormat format);
+
+  /** Gracefully shutdown the pipeline. */
+  void shutdown();
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandlerFactory.java b/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandlerFactory.java
index 40a599e6a..02001d4c9 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandlerFactory.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandlerFactory.java
@@ -1,5 +1,8 @@
 package com.wavefront.agent.handlers;
 
+import com.wavefront.data.ReportableEntityType;
+import javax.annotation.Nonnull;
+
 /**
  * Factory for {@link ReportableEntityHandler} objects.
  *
@@ -13,13 +16,20 @@ public interface ReportableEntityHandlerFactory {
    * @param handlerKey unique identifier for the handler.
    * @return new or existing handler.
    */
-  ReportableEntityHandler getHandler(HandlerKey handlerKey);
+   ReportableEntityHandler getHandler(HandlerKey handlerKey);
 
   /**
-   * Perform finalizing tasks on handlers.
+   * Create, or return existing, {@link ReportableEntityHandler}.
+   *
+   * @param entityType ReportableEntityType for the handler.
+   * @param handle handle.
+   * @return new or existing handler.
    */
-  default void shutdown() {
-    // no-op
+  default  ReportableEntityHandler getHandler(
+      ReportableEntityType entityType, String handle) {
+    return getHandler(HandlerKey.of(entityType, handle));
   }
 
+  /** Shutdown pipeline for a specific handle. */
+  void shutdown(@Nonnull String handle);
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandlerFactoryImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandlerFactoryImpl.java
index b3c96b999..253bb1880 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandlerFactoryImpl.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/ReportableEntityHandlerFactoryImpl.java
@@ -1,63 +1,226 @@
 package com.wavefront.agent.handlers;
 
-import java.util.HashMap;
+import static com.wavefront.data.ReportableEntityType.TRACE_SPAN_LOGS;
+
+import com.wavefront.agent.data.EntityPropertiesFactory;
+import com.wavefront.api.agent.ValidationConfiguration;
+import com.wavefront.common.Utils;
+import com.wavefront.common.logger.SamplingLogger;
+import com.wavefront.data.ReportableEntityType;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
 import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.math.NumberUtils;
+import wavefront.report.Histogram;
 
 /**
  * Caching factory for {@link ReportableEntityHandler} objects. Makes sure there's only one handler
- * for each {@link HandlerKey}, which makes it possible to spin up handlers on demand at runtime,
- * as well as redirecting traffic to a different pipeline.
+ * for each {@link HandlerKey}, which makes it possible to spin up handlers on demand at runtime, as
+ * well as redirecting traffic to a different pipeline.
  *
  * @author vasily@wavefront.com
  */
 public class ReportableEntityHandlerFactoryImpl implements ReportableEntityHandlerFactory {
-  private static final Logger logger = Logger.getLogger(ReportableEntityHandlerFactoryImpl.class.getCanonicalName());
+  private static final Logger logger = Logger.getLogger("sampling");
 
-  private static final int SOURCE_TAGS_NUM_THREADS = 2;
+  public static final Logger VALID_POINTS_LOGGER =
+      new SamplingLogger(
+          ReportableEntityType.POINT,
+          Logger.getLogger("RawValidPoints"),
+          getSystemPropertyAsDouble("wavefront.proxy.logpoints.sample-rate"),
+          "true".equalsIgnoreCase(System.getProperty("wavefront.proxy.logpoints")),
+          logger::info);
+  public static final Logger VALID_HISTOGRAMS_LOGGER =
+      new SamplingLogger(
+          ReportableEntityType.HISTOGRAM,
+          Logger.getLogger("RawValidHistograms"),
+          getSystemPropertyAsDouble("wavefront.proxy.logpoints.sample-rate"),
+          "true".equalsIgnoreCase(System.getProperty("wavefront.proxy.logpoints")),
+          logger::info);
+  private static final Logger VALID_SPANS_LOGGER =
+      new SamplingLogger(
+          ReportableEntityType.TRACE,
+          Logger.getLogger("RawValidSpans"),
+          getSystemPropertyAsDouble("wavefront.proxy.logspans.sample-rate"),
+          false,
+          logger::info);
+  private static final Logger VALID_SPAN_LOGS_LOGGER =
+      new SamplingLogger(
+          ReportableEntityType.TRACE_SPAN_LOGS,
+          Logger.getLogger("RawValidSpanLogs"),
+          getSystemPropertyAsDouble("wavefront.proxy.logspans.sample-rate"),
+          false,
+          logger::info);
+  private static final Logger VALID_EVENTS_LOGGER =
+      new SamplingLogger(
+          ReportableEntityType.EVENT,
+          Logger.getLogger("RawValidEvents"),
+          getSystemPropertyAsDouble("wavefront.proxy.logevents.sample-rate"),
+          false,
+          logger::info);
+  private static final Logger VALID_LOGS_LOGGER =
+      new SamplingLogger(
+          ReportableEntityType.LOGS,
+          Logger.getLogger("RawValidLogs"),
+          getSystemPropertyAsDouble("wavefront.proxy.loglogs.sample-rate"),
+          false,
+          logger::info);
 
-  private Map handlers = new HashMap<>();
+  protected final Map>> handlers =
+      new ConcurrentHashMap<>();
 
   private final SenderTaskFactory senderTaskFactory;
   private final int blockedItemsPerBatch;
-  private final int defaultFlushThreads;
+  private final ValidationConfiguration validationConfig;
+  private final Logger blockedPointsLogger;
+  private final Logger blockedHistogramsLogger;
+  private final Logger blockedSpansLogger;
+  private final Logger blockedLogsLogger;
+  private final Function histogramRecompressor;
+  private final Map entityPropsFactoryMap;
 
   /**
    * Create new instance.
    *
-   * @param senderTaskFactory    SenderTaskFactory instance used to create SenderTasks for new handlers
-   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into the main log file.
-   * @param defaultFlushThreads  control fanout for SenderTasks.
+   * @param senderTaskFactory SenderTaskFactory instance used to create SenderTasks for new
+   *     handlers.
+   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into
+   *     the main log file.
+   * @param validationConfig validation configuration.
    */
-  public ReportableEntityHandlerFactoryImpl(final SenderTaskFactory senderTaskFactory,
-                                            final int blockedItemsPerBatch,
-                                            final int defaultFlushThreads) {
+  public ReportableEntityHandlerFactoryImpl(
+      final SenderTaskFactory senderTaskFactory,
+      final int blockedItemsPerBatch,
+      @Nonnull final ValidationConfiguration validationConfig,
+      final Logger blockedPointsLogger,
+      final Logger blockedHistogramsLogger,
+      final Logger blockedSpansLogger,
+      @Nullable Function histogramRecompressor,
+      final Map entityPropsFactoryMap,
+      final Logger blockedLogsLogger) {
     this.senderTaskFactory = senderTaskFactory;
     this.blockedItemsPerBatch = blockedItemsPerBatch;
-    this.defaultFlushThreads = defaultFlushThreads;
+    this.validationConfig = validationConfig;
+    this.blockedPointsLogger = blockedPointsLogger;
+    this.blockedHistogramsLogger = blockedHistogramsLogger;
+    this.blockedSpansLogger = blockedSpansLogger;
+    this.histogramRecompressor = histogramRecompressor;
+    this.blockedLogsLogger = blockedLogsLogger;
+    this.entityPropsFactoryMap = entityPropsFactoryMap;
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public  ReportableEntityHandler getHandler(HandlerKey handlerKey) {
+    BiConsumer receivedRateSink =
+        (tenantName, rate) ->
+            entityPropsFactoryMap
+                .get(tenantName)
+                .get(handlerKey.getEntityType())
+                .reportReceivedRate(handlerKey.getHandle(), rate);
+    return (ReportableEntityHandler)
+        handlers
+            .computeIfAbsent(handlerKey.getHandle(), h -> new ConcurrentHashMap<>())
+            .computeIfAbsent(
+                handlerKey.getEntityType(),
+                k -> {
+                  switch (handlerKey.getEntityType()) {
+                    case POINT:
+                      return new ReportPointHandlerImpl(
+                          handlerKey,
+                          blockedItemsPerBatch,
+                          senderTaskFactory.createSenderTasks(handlerKey),
+                          validationConfig,
+                          true,
+                          receivedRateSink,
+                          blockedPointsLogger,
+                          VALID_POINTS_LOGGER,
+                          null);
+                    case HISTOGRAM:
+                      return new ReportPointHandlerImpl(
+                          handlerKey,
+                          blockedItemsPerBatch,
+                          senderTaskFactory.createSenderTasks(handlerKey),
+                          validationConfig,
+                          true,
+                          receivedRateSink,
+                          blockedHistogramsLogger,
+                          VALID_HISTOGRAMS_LOGGER,
+                          histogramRecompressor);
+                    case SOURCE_TAG:
+                      return new ReportSourceTagHandlerImpl(
+                          handlerKey,
+                          blockedItemsPerBatch,
+                          senderTaskFactory.createSenderTasks(handlerKey),
+                          receivedRateSink,
+                          blockedPointsLogger);
+                    case TRACE:
+                      return new SpanHandlerImpl(
+                          handlerKey,
+                          blockedItemsPerBatch,
+                          senderTaskFactory.createSenderTasks(handlerKey),
+                          validationConfig,
+                          receivedRateSink,
+                          blockedSpansLogger,
+                          VALID_SPANS_LOGGER,
+                          (tenantName) ->
+                              entityPropsFactoryMap
+                                  .get(tenantName)
+                                  .getGlobalProperties()
+                                  .getDropSpansDelayedMinutes(),
+                          Utils.lazySupplier(
+                              () ->
+                                  getHandler(
+                                      HandlerKey.of(TRACE_SPAN_LOGS, handlerKey.getHandle()))));
+                    case TRACE_SPAN_LOGS:
+                      return new SpanLogsHandlerImpl(
+                          handlerKey,
+                          blockedItemsPerBatch,
+                          senderTaskFactory.createSenderTasks(handlerKey),
+                          receivedRateSink,
+                          blockedSpansLogger,
+                          VALID_SPAN_LOGS_LOGGER);
+                    case EVENT:
+                      return new EventHandlerImpl(
+                          handlerKey,
+                          blockedItemsPerBatch,
+                          senderTaskFactory.createSenderTasks(handlerKey),
+                          receivedRateSink,
+                          blockedPointsLogger,
+                          VALID_EVENTS_LOGGER);
+                    case LOGS:
+                      return new ReportLogHandlerImpl(
+                          handlerKey,
+                          blockedItemsPerBatch,
+                          senderTaskFactory.createSenderTasks(handlerKey),
+                          validationConfig,
+                          true,
+                          receivedRateSink,
+                          blockedLogsLogger,
+                          VALID_LOGS_LOGGER);
+                    default:
+                      throw new IllegalArgumentException(
+                          "Unexpected entity type "
+                              + handlerKey.getEntityType().name()
+                              + " for "
+                              + handlerKey.getHandle());
+                  }
+                });
   }
 
-  public ReportableEntityHandler getHandler(HandlerKey handlerKey) {
-    return handlers.computeIfAbsent(handlerKey, k -> {
-      switch (handlerKey.getEntityType()) {
-        case POINT:
-        case HISTOGRAM:
-          return new ReportPointHandlerImpl(handlerKey.getHandle(), blockedItemsPerBatch,
-              senderTaskFactory.createSenderTasks(handlerKey, defaultFlushThreads));
-        case SOURCE_TAG:
-          return new ReportSourceTagHandlerImpl(handlerKey.getHandle(), blockedItemsPerBatch,
-              senderTaskFactory.createSenderTasks(handlerKey, SOURCE_TAGS_NUM_THREADS));
-        case TRACE:
-          return new SpanHandlerImpl(handlerKey.getHandle(), blockedItemsPerBatch,
-              senderTaskFactory.createSenderTasks(handlerKey, defaultFlushThreads));
-        default:
-          throw new IllegalArgumentException("Unexpected entity type " + handlerKey.getEntityType().name() +
-              " for " + handlerKey.getHandle());
-      }
-    });
+  @Override
+  public void shutdown(@Nonnull String handle) {
+    if (handlers.containsKey(handle)) {
+      handlers.get(handle).values().forEach(ReportableEntityHandler::shutdown);
+    }
   }
 
-  public void shutdown() {
-    //
+  private static double getSystemPropertyAsDouble(String propertyName) {
+    String sampleRateProperty = propertyName == null ? null : System.getProperty(propertyName);
+    return NumberUtils.isNumber(sampleRateProperty) ? Double.parseDouble(sampleRateProperty) : 1.0d;
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/SenderTask.java b/proxy/src/main/java/com/wavefront/agent/handlers/SenderTask.java
index 71c4d8840..f5afba7b7 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/SenderTask.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/SenderTask.java
@@ -1,13 +1,16 @@
 package com.wavefront.agent.handlers;
 
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.common.Managed;
+import javax.annotation.Nullable;
+
 /**
  * Batch and ship valid items to Wavefront servers
  *
  * @author vasily@wavefront.com
- *
  * @param  the type of input objects handled.
  */
-public interface SenderTask {
+public interface SenderTask extends Managed {
 
   /**
    * Add valid item to the send queue (memory buffers).
@@ -17,19 +20,8 @@ public interface SenderTask {
   void add(T item);
 
   /**
-   * Add multiple valid items to the send queue (memory buffers).
-   *
-   * @param items items to add to the send queue.
-   */
-  default void add(Iterable items) {
-    for (T item : items) {
-      add(item);
-    }
-  }
-
-  /**
-   * Calculate a numeric score (the lower the better) that is intended to help the {@link ReportableEntityHandler}
-   * to choose the best SenderTask to handle over data to.
+   * Calculate a numeric score (the lower the better) that is intended to help the {@link
+   * ReportableEntityHandler} choose the best SenderTask to handle over data to.
    *
    * @return task score
    */
@@ -37,11 +29,8 @@ default void add(Iterable items) {
 
   /**
    * Force memory buffer flush.
+   *
+   * @param reason reason for queueing.
    */
-  void drainBuffersToQueue();
-
-  /**
-   * Shut down the scheduler for this task (prevent future scheduled runs).
-   */
-  void shutdown();
+  void drainBuffersToQueue(@Nullable QueueingReason reason);
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/SenderTaskFactory.java b/proxy/src/main/java/com/wavefront/agent/handlers/SenderTaskFactory.java
index 7440ca37d..b62f44a1c 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/SenderTaskFactory.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/SenderTaskFactory.java
@@ -1,8 +1,10 @@
 package com.wavefront.agent.handlers;
 
+import com.wavefront.agent.data.QueueingReason;
 import java.util.Collection;
-
-import javax.validation.constraints.NotNull;
+import java.util.Map;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 /**
  * Factory for {@link SenderTask} objects.
@@ -15,14 +17,27 @@ public interface SenderTaskFactory {
    * Create a collection of {@link SenderTask objects} for a specified handler key.
    *
    * @param handlerKey unique identifier for the handler.
-   * @param numThreads create a specified number of threads.
-   * @return created tasks.
+   * @return created tasks corresponding to different Wavefront endpoints {@link
+   *     com.wavefront.api.ProxyV2API}.
    */
-  Collection createSenderTasks(@NotNull HandlerKey handlerKey,
-                                           final int numThreads);
+   Map>> createSenderTasks(@Nonnull HandlerKey handlerKey);
+
+  /** Shut down all tasks. */
+  void shutdown();
 
   /**
-   * Shut down all tasks.
+   * Shut down specific pipeline
+   *
+   * @param handle pipeline's handle
    */
-  void shutdown();
+  void shutdown(@Nonnull String handle);
+
+  /**
+   * Drain memory buffers to queue for all tasks.
+   *
+   * @param reason reason for queueing
+   */
+  void drainBuffersToQueue(@Nullable QueueingReason reason);
+
+  void truncateBuffers();
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/SenderTaskFactoryImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/SenderTaskFactoryImpl.java
index 7aa189eb5..cf82e371a 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/SenderTaskFactoryImpl.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/SenderTaskFactoryImpl.java
@@ -1,22 +1,41 @@
 package com.wavefront.agent.handlers;
 
-import com.google.common.util.concurrent.RecyclableRateLimiter;
+import static com.wavefront.api.agent.Constants.PUSH_FORMAT_HISTOGRAM;
+import static com.wavefront.api.agent.Constants.PUSH_FORMAT_TRACING;
+import static com.wavefront.api.agent.Constants.PUSH_FORMAT_TRACING_SPAN_LOGS;
+import static com.wavefront.api.agent.Constants.PUSH_FORMAT_WAVEFRONT;
 
-import com.wavefront.agent.api.ForceQueueEnabledAgentAPI;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Maps;
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.agent.data.EntityProperties;
+import com.wavefront.agent.data.EntityPropertiesFactory;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.agent.queueing.QueueController;
+import com.wavefront.agent.queueing.QueueingFactory;
+import com.wavefront.agent.queueing.TaskQueueFactory;
+import com.wavefront.agent.queueing.TaskSizeEstimator;
+import com.wavefront.api.ProxyV2API;
+import com.wavefront.common.Managed;
+import com.wavefront.common.NamedThreadFactory;
+import com.wavefront.common.TaggedMetricName;
 import com.wavefront.data.ReportableEntityType;
-
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Gauge;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import static com.wavefront.api.agent.Constants.PUSH_FORMAT_HISTOGRAM;
-import static com.wavefront.api.agent.Constants.PUSH_FORMAT_TRACING;
-import static com.wavefront.api.agent.Constants.PUSH_FORMAT_WAVEFRONT;
 
 /**
  * Factory for {@link SenderTask} objects.
@@ -24,82 +43,305 @@
  * @author vasily@wavefront.com
  */
 public class SenderTaskFactoryImpl implements SenderTaskFactory {
+  private final Logger log = Logger.getLogger(SenderTaskFactoryImpl.class.getCanonicalName());
 
-  private List managedTasks = new ArrayList<>();
+  private final Map> entityTypes = new ConcurrentHashMap<>();
+  private final Map executors = new ConcurrentHashMap<>();
+  private final Map>> managedTasks = new ConcurrentHashMap<>();
+  private final Map managedServices = new ConcurrentHashMap<>();
 
-  private final ForceQueueEnabledAgentAPI proxyAPI;
-  private final UUID proxyId;
-  private final RecyclableRateLimiter globalRateLimiter;
-  private final AtomicInteger pushFlushInterval;
-  private final AtomicInteger pointsPerBatch;
-  private final AtomicInteger memoryBufferLimit;
+  /** Keep track of all {@link TaskSizeEstimator} instances to calculate global buffer fill rate. */
+  private final Map taskSizeEstimators = new ConcurrentHashMap<>();
 
-  private static final RecyclableRateLimiter sourceTagRateLimiter = RecyclableRateLimiter.create(5, 10);
+  private final APIContainer apiContainer;
+  private final UUID proxyId;
+  private final TaskQueueFactory taskQueueFactory;
+  private final QueueingFactory queueingFactory;
+  private final Map entityPropsFactoryMap;
 
   /**
    * Create new instance.
    *
-   * @param proxyAPI          handles interaction with Wavefront servers as well as queueing.
-   * @param proxyId           proxy ID.
-   * @param globalRateLimiter rate limiter to control outbound point rate.
-   * @param pushFlushInterval interval between flushes.
-   * @param itemsPerBatch     max points per flush.
-   * @param memoryBufferLimit max points in task's memory buffer before queueing.
+   * @param apiContainer handles interaction with Wavefront servers as well as queueing.
+   * @param proxyId proxy ID.
+   * @param taskQueueFactory factory for backing queues.
+   * @param queueingFactory factory for queueing.
+   * @param entityPropsFactoryMap map of factory for entity-specific wrappers for multiple
+   *     multicasting mutable proxy settings.
    */
-  public SenderTaskFactoryImpl(final ForceQueueEnabledAgentAPI proxyAPI,
-                               final UUID proxyId,
-                               final RecyclableRateLimiter globalRateLimiter,
-                               final AtomicInteger pushFlushInterval,
-                               @Nullable final AtomicInteger itemsPerBatch,
-                               @Nullable final AtomicInteger memoryBufferLimit) {
-    this.proxyAPI = proxyAPI;
+  public SenderTaskFactoryImpl(
+      final APIContainer apiContainer,
+      final UUID proxyId,
+      final TaskQueueFactory taskQueueFactory,
+      @Nullable final QueueingFactory queueingFactory,
+      final Map entityPropsFactoryMap) {
+    this.apiContainer = apiContainer;
     this.proxyId = proxyId;
-    this.globalRateLimiter = globalRateLimiter;
-    this.pushFlushInterval = pushFlushInterval;
-    this.pointsPerBatch = itemsPerBatch;
-    this.memoryBufferLimit = memoryBufferLimit;
+    this.taskQueueFactory = taskQueueFactory;
+    this.queueingFactory = queueingFactory;
+    this.entityPropsFactoryMap = entityPropsFactoryMap;
+    // global `~proxy.buffer.fill-rate` metric aggregated from all task size estimators
+    Metrics.newGauge(
+        new TaggedMetricName("buffer", "fill-rate"),
+        new Gauge() {
+          @Override
+          public Long value() {
+            List sizes =
+                taskSizeEstimators.values().stream()
+                    .map(TaskSizeEstimator::getBytesPerMinute)
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toList());
+            return sizes.size() == 0 ? null : sizes.stream().mapToLong(x -> x).sum();
+          }
+        });
+  }
+
+  @SuppressWarnings("unchecked")
+  public Map>> createSenderTasks(@Nonnull HandlerKey handlerKey) {
+    ReportableEntityType entityType = handlerKey.getEntityType();
+    String handle = handlerKey.getHandle();
+
+    ScheduledExecutorService scheduler;
+    Map>> toReturn = Maps.newHashMap();
+    // MONIT-25479: HandlerKey(EntityType, Port) --> HandlerKey(EntityType, Port, TenantName)
+    // Every SenderTask is tenant specific from this point
+    for (String tenantName : apiContainer.getTenantNameList()) {
+      int numThreads = entityPropsFactoryMap.get(tenantName).get(entityType).getFlushThreads();
+      HandlerKey tenantHandlerKey = HandlerKey.of(entityType, handle, tenantName);
+
+      scheduler =
+          executors.computeIfAbsent(
+              tenantHandlerKey,
+              x ->
+                  Executors.newScheduledThreadPool(
+                      numThreads,
+                      new NamedThreadFactory(
+                          "submitter-"
+                              + tenantHandlerKey.getEntityType()
+                              + "-"
+                              + tenantHandlerKey.getHandle())));
+
+      toReturn.put(tenantName, generateSenderTaskList(tenantHandlerKey, numThreads, scheduler));
+    }
+    return toReturn;
   }
 
-  public Collection createSenderTasks(@NotNull HandlerKey handlerKey,
-                                                  final int numThreads) {
-    List toReturn = new ArrayList<>(numThreads);
+  private Collection> generateSenderTaskList(
+      HandlerKey handlerKey, int numThreads, ScheduledExecutorService scheduler) {
+    String tenantName = handlerKey.getTenantName();
+    if (tenantName == null) {
+      throw new IllegalArgumentException(
+          "Tenant name in handlerKey should not be null when " + "generating sender task list.");
+    }
+    TaskSizeEstimator taskSizeEstimator = new TaskSizeEstimator(handlerKey.getHandle());
+    taskSizeEstimators.put(handlerKey, taskSizeEstimator);
+    ReportableEntityType entityType = handlerKey.getEntityType();
+    List> senderTaskList = new ArrayList<>(numThreads);
+    ProxyV2API proxyV2API = apiContainer.getProxyV2APIForTenant(tenantName);
+    EntityProperties properties = entityPropsFactoryMap.get(tenantName).get(entityType);
     for (int threadNo = 0; threadNo < numThreads; threadNo++) {
-      SenderTask senderTask;
-      switch (handlerKey.getEntityType()) {
+      SenderTask senderTask;
+      switch (entityType) {
         case POINT:
-          senderTask = new LineDelimitedSenderTask(ReportableEntityType.POINT.toString(), PUSH_FORMAT_WAVEFRONT,
-              proxyAPI, proxyId, handlerKey.getHandle(), threadNo, globalRateLimiter, pushFlushInterval,
-              pointsPerBatch, memoryBufferLimit);
+        case DELTA_COUNTER:
+          senderTask =
+              new LineDelimitedSenderTask(
+                  handlerKey,
+                  PUSH_FORMAT_WAVEFRONT,
+                  proxyV2API,
+                  proxyId,
+                  properties,
+                  scheduler,
+                  threadNo,
+                  taskSizeEstimator,
+                  taskQueueFactory.getTaskQueue(handlerKey, threadNo));
           break;
         case HISTOGRAM:
-          senderTask = new LineDelimitedSenderTask(ReportableEntityType.HISTOGRAM.toString(), PUSH_FORMAT_HISTOGRAM,
-              proxyAPI, proxyId, handlerKey.getHandle(), threadNo, globalRateLimiter, pushFlushInterval,
-              pointsPerBatch, memoryBufferLimit);
+          senderTask =
+              new LineDelimitedSenderTask(
+                  handlerKey,
+                  PUSH_FORMAT_HISTOGRAM,
+                  proxyV2API,
+                  proxyId,
+                  properties,
+                  scheduler,
+                  threadNo,
+                  taskSizeEstimator,
+                  taskQueueFactory.getTaskQueue(handlerKey, threadNo));
           break;
         case SOURCE_TAG:
-          senderTask = new ReportSourceTagSenderTask(proxyAPI, handlerKey.getHandle(), threadNo, pushFlushInterval,
-              sourceTagRateLimiter, pointsPerBatch, memoryBufferLimit);
+          // In MONIT-25479, SOURCE_TAG does not support tag based multicasting. But still
+          // generated tasks for each tenant in case we have other multicasting mechanism
+          senderTask =
+              new SourceTagSenderTask(
+                  handlerKey,
+                  apiContainer.getSourceTagAPIForTenant(tenantName),
+                  threadNo,
+                  properties,
+                  scheduler,
+                  taskQueueFactory.getTaskQueue(handlerKey, threadNo));
           break;
         case TRACE:
-          senderTask = new LineDelimitedSenderTask(ReportableEntityType.TRACE.toString(), PUSH_FORMAT_TRACING,
-              proxyAPI, proxyId, handlerKey.getHandle(), threadNo, globalRateLimiter, pushFlushInterval,
-              pointsPerBatch, memoryBufferLimit);
+          senderTask =
+              new LineDelimitedSenderTask(
+                  handlerKey,
+                  PUSH_FORMAT_TRACING,
+                  proxyV2API,
+                  proxyId,
+                  properties,
+                  scheduler,
+                  threadNo,
+                  taskSizeEstimator,
+                  taskQueueFactory.getTaskQueue(handlerKey, threadNo));
+          break;
+        case TRACE_SPAN_LOGS:
+          // In MONIT-25479, TRACE_SPAN_LOGS does not support tag based multicasting. But
+          // still
+          // generated tasks for each tenant in case we have other multicasting mechanism
+          senderTask =
+              new LineDelimitedSenderTask(
+                  handlerKey,
+                  PUSH_FORMAT_TRACING_SPAN_LOGS,
+                  proxyV2API,
+                  proxyId,
+                  properties,
+                  scheduler,
+                  threadNo,
+                  taskSizeEstimator,
+                  taskQueueFactory.getTaskQueue(handlerKey, threadNo));
+          break;
+        case EVENT:
+          senderTask =
+              new EventSenderTask(
+                  handlerKey,
+                  apiContainer.getEventAPIForTenant(tenantName),
+                  proxyId,
+                  threadNo,
+                  properties,
+                  scheduler,
+                  taskQueueFactory.getTaskQueue(handlerKey, threadNo));
+          break;
+        case LOGS:
+          senderTask =
+              new LogSenderTask(
+                  handlerKey,
+                  apiContainer.getLogAPI(),
+                  proxyId,
+                  threadNo,
+                  entityPropsFactoryMap.get(tenantName).get(entityType),
+                  scheduler,
+                  taskQueueFactory.getTaskQueue(handlerKey, threadNo));
           break;
         default:
-          throw new IllegalArgumentException("Unexpected entity type " + handlerKey.getEntityType().name() +
-              " for " + handlerKey.getHandle());
+          throw new IllegalArgumentException(
+              "Unexpected entity type "
+                  + handlerKey.getEntityType().name()
+                  + " for "
+                  + handlerKey.getHandle());
       }
-      toReturn.add(senderTask);
-      managedTasks.add(senderTask);
+      senderTaskList.add(senderTask);
+      senderTask.start();
     }
-    return toReturn;
+    if (queueingFactory != null) {
+      QueueController controller = queueingFactory.getQueueController(handlerKey, numThreads);
+      managedServices.put(handlerKey, controller);
+      controller.start();
+    }
+    managedTasks.put(handlerKey, senderTaskList);
+    entityTypes
+        .computeIfAbsent(handlerKey.getHandle(), x -> new ArrayList<>())
+        .add(handlerKey.getEntityType());
+    return senderTaskList;
   }
 
   @Override
   public void shutdown() {
-    for (SenderTask task : managedTasks) {
-      task.shutdown();
+    managedTasks.values().stream().flatMap(Collection::stream).forEach(Managed::stop);
+    taskSizeEstimators.values().forEach(TaskSizeEstimator::shutdown);
+    managedServices.values().forEach(Managed::stop);
+    executors
+        .values()
+        .forEach(
+            x -> {
+              try {
+                x.shutdown();
+                x.awaitTermination(1000, TimeUnit.MILLISECONDS);
+              } catch (InterruptedException e) {
+                // ignore
+              }
+            });
+  }
+
+  /**
+   * shutdown() is called from outside layer where handle is not tenant specific in order to
+   * properly shut down all tenant specific tasks, iterate through the tenant list and shut down
+   * correspondingly.
+   *
+   * @param handle pipeline's handle
+   */
+  @Override
+  public void shutdown(@Nonnull String handle) {
+    for (String tenantName : apiContainer.getTenantNameList()) {
+      String tenantHandlerKey = HandlerKey.generateTenantSpecificHandle(handle, tenantName);
+      List types = entityTypes.get(tenantHandlerKey);
+      if (types == null) return;
+      try {
+        types.forEach(
+            x -> taskSizeEstimators.remove(HandlerKey.of(x, handle, tenantName)).shutdown());
+        types.forEach(x -> managedServices.remove(HandlerKey.of(x, handle, tenantName)).stop());
+        types.forEach(
+            x ->
+                managedTasks
+                    .remove(HandlerKey.of(x, handle, tenantName))
+                    .forEach(
+                        t -> {
+                          t.stop();
+                          t.drainBuffersToQueue(null);
+                        }));
+        types.forEach(x -> executors.remove(HandlerKey.of(x, handle, tenantName)).shutdown());
+      } finally {
+        entityTypes.remove(tenantHandlerKey);
+      }
     }
   }
 
+  @Override
+  public void drainBuffersToQueue(QueueingReason reason) {
+    managedTasks.values().stream()
+        .flatMap(Collection::stream)
+        .forEach(x -> x.drainBuffersToQueue(reason));
+  }
+
+  @Override
+  public void truncateBuffers() {
+    managedServices
+        .entrySet()
+        .forEach(
+            handlerKeyManagedEntry -> {
+              System.out.println(
+                  "Truncating buffers: Queue with handlerKey " + handlerKeyManagedEntry.getKey());
+              log.info(
+                  "Truncating buffers: Queue with handlerKey " + handlerKeyManagedEntry.getKey());
+              QueueController pp = handlerKeyManagedEntry.getValue();
+              pp.truncateBuffers();
+            });
+  }
+
+  @VisibleForTesting
+  public void flushNow(@Nonnull HandlerKey handlerKey) {
+    HandlerKey tenantHandlerKey;
+    ReportableEntityType entityType = handlerKey.getEntityType();
+    String handle = handlerKey.getHandle();
+    for (String tenantName : apiContainer.getTenantNameList()) {
+      tenantHandlerKey = HandlerKey.of(entityType, handle, tenantName);
+      managedTasks
+          .get(tenantHandlerKey)
+          .forEach(
+              task -> {
+                if (task instanceof AbstractSenderTask) {
+                  ((AbstractSenderTask) task).run();
+                }
+              });
+    }
+  }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/SourceTagSenderTask.java b/proxy/src/main/java/com/wavefront/agent/handlers/SourceTagSenderTask.java
new file mode 100644
index 000000000..ccc01ed13
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/SourceTagSenderTask.java
@@ -0,0 +1,135 @@
+package com.wavefront.agent.handlers;
+
+import com.wavefront.agent.data.EntityProperties;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.agent.data.SourceTagSubmissionTask;
+import com.wavefront.agent.data.TaskResult;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.api.SourceTagAPI;
+import com.wavefront.dto.SourceTag;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+
+/**
+ * This class is responsible for accumulating the source tag changes and post it in a batch. This
+ * class is similar to PostPushDataTimedTask.
+ *
+ * @author Suranjan Pramanik (suranjan@wavefront.com)
+ * @author vasily@wavefront.com
+ */
+class SourceTagSenderTask extends AbstractSenderTask {
+  private static final Logger logger =
+      Logger.getLogger(SourceTagSenderTask.class.getCanonicalName());
+
+  private final SourceTagAPI proxyAPI;
+  private final TaskQueue backlog;
+
+  /**
+   * Create new instance
+   *
+   * @param proxyAPI handles interaction with Wavefront servers as well as queueing.
+   * @param handlerKey metrics pipeline handler key.
+   * @param threadId thread number.
+   * @param properties container for mutable proxy settings.
+   * @param scheduler executor service for this task
+   * @param backlog backing queue
+   */
+  SourceTagSenderTask(
+      HandlerKey handlerKey,
+      SourceTagAPI proxyAPI,
+      int threadId,
+      EntityProperties properties,
+      ScheduledExecutorService scheduler,
+      TaskQueue backlog) {
+    super(handlerKey, threadId, properties, scheduler);
+    this.proxyAPI = proxyAPI;
+    this.backlog = backlog;
+  }
+
+  @Override
+  TaskResult processSingleBatch(List batch) {
+    throw new UnsupportedOperationException("Not implemented");
+  }
+
+  @Override
+  public void run() {
+    long nextRunMillis = properties.getPushFlushInterval();
+    isSending = true;
+    try {
+      List current = createBatch();
+      if (current.size() == 0) return;
+      Iterator iterator = current.iterator();
+      while (iterator.hasNext()) {
+        if (rateLimiter == null || rateLimiter.tryAcquire()) {
+          SourceTag tag = iterator.next();
+          SourceTagSubmissionTask task =
+              new SourceTagSubmissionTask(
+                  proxyAPI, properties, backlog, handlerKey.getHandle(), tag, null);
+          TaskResult result = task.execute();
+          this.attemptedCounter.inc();
+          switch (result) {
+            case DELIVERED:
+              continue;
+            case PERSISTED:
+            case PERSISTED_RETRY:
+              if (rateLimiter != null) rateLimiter.recyclePermits(1);
+              continue;
+            case RETRY_LATER:
+              final List remainingItems = new ArrayList<>();
+              remainingItems.add(tag);
+              iterator.forEachRemaining(remainingItems::add);
+              undoBatch(remainingItems);
+              if (rateLimiter != null) rateLimiter.recyclePermits(1);
+              return;
+            default:
+          }
+        } else {
+          final List remainingItems = new ArrayList<>();
+          iterator.forEachRemaining(remainingItems::add);
+          undoBatch(remainingItems);
+          // if proxy rate limit exceeded, try again in 1/4..1/2 of flush interval
+          // to introduce some degree of fairness.
+          nextRunMillis = (int) (1 + Math.random()) * nextRunMillis / 4;
+          final long willRetryIn = nextRunMillis;
+          throttledLogger.log(
+              Level.INFO,
+              () ->
+                  "["
+                      + handlerKey.getHandle()
+                      + " thread "
+                      + threadId
+                      + "]: WF-4 Proxy rate limiter "
+                      + "active (pending "
+                      + handlerKey.getEntityType()
+                      + ": "
+                      + datum.size()
+                      + "), will retry in "
+                      + willRetryIn
+                      + "ms");
+          return;
+        }
+      }
+    } catch (Throwable t) {
+      logger.log(Level.SEVERE, "Unexpected error in flush loop", t);
+    } finally {
+      isSending = false;
+      scheduler.schedule(this, nextRunMillis, TimeUnit.MILLISECONDS);
+    }
+  }
+
+  @Override
+  void flushSingleBatch(List batch, @Nullable QueueingReason reason) {
+    for (SourceTag tag : batch) {
+      SourceTagSubmissionTask task =
+          new SourceTagSubmissionTask(
+              proxyAPI, properties, backlog, handlerKey.getHandle(), tag, null);
+      task.enqueue(reason);
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/SpanHandlerImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/SpanHandlerImpl.java
index 8e3daf5a7..a88b0fd20 100644
--- a/proxy/src/main/java/com/wavefront/agent/handlers/SpanHandlerImpl.java
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/SpanHandlerImpl.java
@@ -1,103 +1,151 @@
 package com.wavefront.agent.handlers;
 
-import com.wavefront.agent.SharedMetricsRegistry;
-import com.wavefront.data.ReportableEntityType;
+import static com.wavefront.agent.sampler.SpanSampler.SPAN_SAMPLING_POLICY_TAG;
+import static com.wavefront.data.Validation.validateSpan;
+
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.api.agent.ValidationConfiguration;
+import com.wavefront.common.Clock;
+import com.wavefront.data.AnnotationUtils;
 import com.wavefront.ingester.SpanSerializer;
 import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
 import com.yammer.metrics.core.MetricName;
-
-import org.apache.commons.lang3.math.NumberUtils;
-
 import java.util.Collection;
-import java.util.Random;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.logging.Logger;
-
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import wavefront.report.Annotation;
 import wavefront.report.Span;
+import wavefront.report.SpanLogs;
 
 /**
- * Handler that processes incoming Span objects, validates them and hands them over to one of
- * the {@link SenderTask} threads.
+ * Handler that processes incoming Span objects, validates them and hands them over to one of the
+ * {@link SenderTask} threads.
  *
  * @author vasily@wavefront.com
  */
-public class SpanHandlerImpl extends AbstractReportableEntityHandler {
-
-  private static final Logger logger = Logger.getLogger(AbstractReportableEntityHandler.class.getCanonicalName());
-  private static final Logger validTracesLogger = Logger.getLogger("RawValidSpans");
-  private static final Random RANDOM = new Random();
-  private static SharedMetricsRegistry metricsRegistry = SharedMetricsRegistry.getInstance();
-
-  private final Counter attemptedCounter;
-  private final Counter queuedCounter;
+public class SpanHandlerImpl extends AbstractReportableEntityHandler {
 
-  private boolean logData = false;
-  private final double logSampleRate;
-  private volatile long logStateUpdatedMillis = 0L;
+  private final ValidationConfiguration validationConfig;
+  private final Logger validItemsLogger;
+  private final Function dropSpansDelayedMinutes;
+  private final com.yammer.metrics.core.Histogram receivedTagCount;
+  private final com.yammer.metrics.core.Counter policySampledSpanCounter;
+  private final Supplier> spanLogsHandler;
 
   /**
-   * Create new instance.
-   *
-   * @param handle               handle / port number.
-   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into the main log file.
-   * @param sendDataTasks        sender tasks.
+   * @param handlerKey pipeline hanler key.
+   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into
+   *     the main log file.
+   * @param senderTaskMap map of tenant name and tasks actually handling data transfer to the
+   *     Wavefront endpoint corresponding to the tenant name
+   * @param validationConfig parameters for data validation.
+   * @param receivedRateSink where to report received rate.
+   * @param blockedItemLogger logger for blocked items.
+   * @param validItemsLogger logger for valid items.
+   * @param dropSpansDelayedMinutes latency threshold for dropping delayed spans.
+   * @param spanLogsHandler spanLogs handler.
    */
-  SpanHandlerImpl(final String handle,
-                  final int blockedItemsPerBatch,
-                  final Collection sendDataTasks) {
-    super(ReportableEntityType.TRACE, handle, blockedItemsPerBatch, new SpanSerializer(), sendDataTasks);
-
-    String logTracesSampleRateProperty = System.getProperty("wavefront.proxy.logspans.sample-rate");
-    this.logSampleRate = NumberUtils.isNumber(logTracesSampleRateProperty) ?
-        Double.parseDouble(logTracesSampleRateProperty) : 1.0d;
-
-    this.attemptedCounter = Metrics.newCounter(new MetricName("spans." + handle, "", "sent"));
-    this.queuedCounter = Metrics.newCounter(new MetricName("spans." + handle, "", "queued"));
-
-    this.statisticOutputExecutor.scheduleAtFixedRate(this::printStats, 10, 10, TimeUnit.SECONDS);
-    this.statisticOutputExecutor.scheduleAtFixedRate(this::printTotal, 1, 1, TimeUnit.MINUTES);
+  SpanHandlerImpl(
+      final HandlerKey handlerKey,
+      final int blockedItemsPerBatch,
+      final Map>> senderTaskMap,
+      @Nonnull final ValidationConfiguration validationConfig,
+      @Nullable final BiConsumer receivedRateSink,
+      @Nullable final Logger blockedItemLogger,
+      @Nullable final Logger validItemsLogger,
+      @Nonnull final Function dropSpansDelayedMinutes,
+      @Nonnull final Supplier> spanLogsHandler) {
+    super(
+        handlerKey,
+        blockedItemsPerBatch,
+        new SpanSerializer(),
+        senderTaskMap,
+        true,
+        receivedRateSink,
+        blockedItemLogger);
+    super.initializeCounters();
+    this.validationConfig = validationConfig;
+    this.validItemsLogger = validItemsLogger;
+    this.dropSpansDelayedMinutes = dropSpansDelayedMinutes;
+    this.receivedTagCount =
+        Metrics.newHistogram(
+            new MetricName(handlerKey.toString() + ".received", "", "tagCount"), false);
+    this.spanLogsHandler = spanLogsHandler;
+    this.policySampledSpanCounter =
+        Metrics.newCounter(new MetricName(handlerKey.toString(), "", "sampler.policy.saved"));
   }
 
   @Override
-  @SuppressWarnings("unchecked")
   protected void reportInternal(Span span) {
-    String strSpan = serializer.apply(span);
-
-    refreshValidDataLoggerState();
-
-    if (logData && (logSampleRate >= 1.0d || (logSampleRate > 0.0d && RANDOM.nextDouble() < logSampleRate))) {
-      // we log valid trace data only if RawValidSpans log level is set to "ALL". This is done to prevent
-      // introducing overhead and accidentally logging raw data to the main log. Honor sample rate limit, if set.
-      validTracesLogger.info(strSpan);
+    receivedTagCount.update(span.getAnnotations().size());
+    Integer maxSpanDelay = dropSpansDelayedMinutes.apply(APIContainer.CENTRAL_TENANT_NAME);
+    if (maxSpanDelay != null
+        && span.getStartMillis() + span.getDuration()
+            < Clock.now() - TimeUnit.MINUTES.toMillis(maxSpanDelay)) {
+      this.reject(span, "span is older than acceptable delay of " + maxSpanDelay + " minutes");
+      return;
     }
-    getTask().add(strSpan);
-    receivedCounter.inc();
-  }
-
-  private void refreshValidDataLoggerState() {
-    if (logStateUpdatedMillis + TimeUnit.SECONDS.toMillis(1) < System.currentTimeMillis()) {
-      // refresh validTracesLogger level once a second
-      if (logData != validTracesLogger.isLoggable(Level.FINEST)) {
-        logData = !logData;
-        logger.info("Valid spans logging is now " + (logData ?
-            "enabled with " + (logSampleRate * 100) + "% sampling":
-            "disabled"));
+    // Spans cannot exceed 24 hours future fill
+    if (span.getStartMillis() > Clock.now() + TimeUnit.HOURS.toMillis(24)) {
+      this.reject(span, "Span outside of reasonable timeframe");
+      return;
+    }
+    // PUB-323 Allow "*" in span name by converting "*" to "-"
+    if (span.getName().contains("*")) {
+      span.setName(span.getName().replace('*', '-'));
+    }
+    validateSpan(span, validationConfig, spanLogsHandler.get()::report);
+    if (span.getAnnotations() != null
+        && AnnotationUtils.getValue(span.getAnnotations(), SPAN_SAMPLING_POLICY_TAG) != null) {
+      this.policySampledSpanCounter.inc();
+    }
+    final String strSpan = serializer.apply(span);
+    getTask(APIContainer.CENTRAL_TENANT_NAME).add(strSpan);
+    getReceivedCounter().inc();
+    // check if span annotations contains the tag key indicating this span should be multicasted
+    if (isMulticastingActive
+        && span.getAnnotations() != null
+        && AnnotationUtils.getValue(span.getAnnotations(), MULTICASTING_TENANT_TAG_KEY) != null) {
+      String[] multicastingTenantNames =
+          AnnotationUtils.getValue(span.getAnnotations(), MULTICASTING_TENANT_TAG_KEY)
+              .trim()
+              .split(",");
+      removeSpanAnnotation(span.getAnnotations(), MULTICASTING_TENANT_TAG_KEY);
+      for (String multicastingTenantName : multicastingTenantNames) {
+        // if the tenant name indicated in span tag is not configured, just ignore
+        if (getTask(multicastingTenantName) != null) {
+          maxSpanDelay = dropSpansDelayedMinutes.apply(multicastingTenantName);
+          if (maxSpanDelay != null
+              && span.getStartMillis() + span.getDuration()
+                  < Clock.now() - TimeUnit.MINUTES.toMillis(maxSpanDelay)) {
+            // just ignore, reduce unnecessary cost on multicasting cluster
+            continue;
+          }
+          getTask(multicastingTenantName).add(serializer.apply(span));
+        }
       }
-      logStateUpdatedMillis = System.currentTimeMillis();
     }
+    if (validItemsLogger != null) validItemsLogger.info(strSpan);
   }
 
-  private void printStats() {
-    logger.info("[" + this.handle + "] Tracing spans received rate: " + getReceivedOneMinuteRate() +
-        " sps (1 min), " + getReceivedFiveMinuteRate() + " sps (5 min), " +
-        this.receivedBurstRateCurrent + " sps (current).");
-  }
-
-  private void printTotal() {
-    logger.info("[" + this.handle + "] Total trace spans processed since start: " + this.attemptedCounter.count() +
-        "; blocked: " + this.blockedCounter.count());
-
+  // MONIT-26010: this is a temp helper function to remove MULTICASTING_TENANT_TAG
+  // TODO: refactor this into AnnotationUtils or figure out a better removing implementation
+  private static void removeSpanAnnotation(List annotations, String key) {
+    Annotation toRemove = null;
+    for (Annotation annotation : annotations) {
+      if (annotation.getKey().equals(key)) {
+        toRemove = annotation;
+        // we should have only one matching
+        break;
+      }
+    }
+    annotations.remove(toRemove);
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/SpanLogsHandlerImpl.java b/proxy/src/main/java/com/wavefront/agent/handlers/SpanLogsHandlerImpl.java
new file mode 100644
index 000000000..08ad0f4a7
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/SpanLogsHandlerImpl.java
@@ -0,0 +1,62 @@
+package com.wavefront.agent.handlers;
+
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.ingester.SpanLogsSerializer;
+import java.util.Collection;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import wavefront.report.SpanLogs;
+
+/**
+ * Handler that processes incoming SpanLogs objects, validates them and hands them over to one of
+ * the {@link SenderTask} threads.
+ *
+ * @author vasily@wavefront.com
+ */
+public class SpanLogsHandlerImpl extends AbstractReportableEntityHandler {
+  private final Logger validItemsLogger;
+
+  /**
+   * Create new instance.
+   *
+   * @param handlerKey pipeline handler key.
+   * @param blockedItemsPerBatch controls sample rate of how many blocked points are written into
+   *     the main log file.
+   * @param senderTaskMap map of tenant name and tasks actually handling data transfer to the
+   *     Wavefront endpoint corresponding to the tenant name
+   * @param receivedRateSink where to report received rate.
+   * @param blockedItemLogger logger for blocked items.
+   * @param validItemsLogger logger for valid items.
+   */
+  SpanLogsHandlerImpl(
+      final HandlerKey handlerKey,
+      final int blockedItemsPerBatch,
+      @Nullable final Map>> senderTaskMap,
+      @Nullable final BiConsumer receivedRateSink,
+      @Nullable final Logger blockedItemLogger,
+      @Nullable final Logger validItemsLogger) {
+    super(
+        handlerKey,
+        blockedItemsPerBatch,
+        new SpanLogsSerializer(),
+        senderTaskMap,
+        true,
+        receivedRateSink,
+        blockedItemLogger);
+    super.initializeCounters();
+    this.validItemsLogger = validItemsLogger;
+  }
+
+  @Override
+  protected void reportInternal(SpanLogs spanLogs) {
+    String strSpanLogs = serializer.apply(spanLogs);
+    if (strSpanLogs != null) {
+      getTask(APIContainer.CENTRAL_TENANT_NAME).add(strSpanLogs);
+      getReceivedCounter().inc();
+      if (validItemsLogger != null) validItemsLogger.info(strSpanLogs);
+      // tagK=tagV based multicasting is not supported
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/handlers/TrafficShapingRateLimitAdjuster.java b/proxy/src/main/java/com/wavefront/agent/handlers/TrafficShapingRateLimitAdjuster.java
new file mode 100644
index 000000000..7df91006c
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/handlers/TrafficShapingRateLimitAdjuster.java
@@ -0,0 +1,101 @@
+package com.wavefront.agent.handlers;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.RecyclableRateLimiter;
+import com.wavefront.agent.data.EntityProperties;
+import com.wavefront.agent.data.EntityPropertiesFactory;
+import com.wavefront.common.EvictingRingBuffer;
+import com.wavefront.common.Managed;
+import com.wavefront.common.SynchronizedEvictingRingBuffer;
+import com.wavefront.data.ReportableEntityType;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.logging.Logger;
+
+/**
+ * Experimental: use automatic traffic shaping (set rate limiter based on recently received per
+ * second rates, heavily biased towards last 5 minutes)
+ *
+ * @author vasily@wavefront.com.
+ */
+public class TrafficShapingRateLimitAdjuster extends TimerTask implements Managed {
+  private static final Logger log =
+      Logger.getLogger(TrafficShapingRateLimitAdjuster.class.getCanonicalName());
+  private static final int MIN_RATE_LIMIT = 10; // 10 pps
+  private static final double TOLERANCE_PERCENT = 5.0;
+
+  private final Map entityPropsFactoryMap;
+  private final double headroom;
+  private final Map> perEntityStats =
+      new EnumMap<>(ReportableEntityType.class);
+  private final Timer timer;
+  private final int windowSeconds;
+
+  /**
+   * @param entityPropsFactoryMap map of factory for entity properties factory (to control rate
+   *     limiters)
+   * @param windowSeconds size of the moving time window to average point rate
+   * @param headroom headroom multiplier
+   */
+  public TrafficShapingRateLimitAdjuster(
+      Map entityPropsFactoryMap,
+      int windowSeconds,
+      double headroom) {
+    this.windowSeconds = windowSeconds;
+    Preconditions.checkArgument(headroom >= 1.0, "headroom can't be less than 1!");
+    Preconditions.checkArgument(windowSeconds > 0, "windowSeconds needs to be > 0!");
+    this.entityPropsFactoryMap = entityPropsFactoryMap;
+    this.headroom = headroom;
+    this.timer = new Timer("traffic-shaping-adjuster-timer");
+  }
+
+  @Override
+  public void run() {
+    for (ReportableEntityType type : ReportableEntityType.values()) {
+      for (EntityPropertiesFactory propsFactory : entityPropsFactoryMap.values()) {
+        EntityProperties props = propsFactory.get(type);
+        long rate = props.getTotalReceivedRate();
+        EvictingRingBuffer stats =
+            perEntityStats.computeIfAbsent(
+                type, x -> new SynchronizedEvictingRingBuffer<>(windowSeconds));
+        if (rate > 0 || stats.size() > 0) {
+          stats.add(rate);
+          if (stats.size() >= 60) { // need at least 1 minute worth of stats to enable the limiter
+            RecyclableRateLimiter rateLimiter = props.getRateLimiter();
+            adjustRateLimiter(type, stats, rateLimiter);
+          }
+        }
+      }
+    }
+  }
+
+  @Override
+  public void start() {
+    timer.scheduleAtFixedRate(this, 1000, 1000);
+  }
+
+  @Override
+  public void stop() {
+    timer.cancel();
+  }
+
+  @VisibleForTesting
+  void adjustRateLimiter(
+      ReportableEntityType type,
+      EvictingRingBuffer sample,
+      RecyclableRateLimiter rateLimiter) {
+    List samples = sample.toList();
+    double suggestedLimit =
+        MIN_RATE_LIMIT
+            + (samples.stream().mapToLong(i -> i).sum() / (double) samples.size()) * headroom;
+    double currentRate = rateLimiter.getRate();
+    if (Math.abs(currentRate - suggestedLimit) > currentRate * TOLERANCE_PERCENT / 100) {
+      log.fine("Setting rate limit for " + type.toString() + " to " + suggestedLimit);
+      rateLimiter.setRate(suggestedLimit);
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/DroppingSender.java b/proxy/src/main/java/com/wavefront/agent/histogram/DroppingSender.java
deleted file mode 100644
index 6353cf84e..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/DroppingSender.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package com.wavefront.agent.histogram;
-
-import com.squareup.tape.ObjectQueue;
-import com.tdunning.math.stats.TDigest;
-
-import java.util.Random;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Dummy sender. Consumes TDigests from an ObjectQueue and sleeps for an amount of time to simulate sending
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class DroppingSender implements Runnable {
-  private static final Logger logger = Logger.getLogger(DroppingSender.class.getCanonicalName());
-
-  private final ObjectQueue input;
-  private final Random r;
-
-  public DroppingSender(ObjectQueue input) {
-    this.input = input;
-    r = new Random();
-  }
-
-  @Override
-  public void run() {
-    ReportPoint current;
-
-    while ((current = input.peek()) != null) {
-      input.remove();
-      logger.log(Level.INFO, "Sent " + current);
-    }
-
-    try {
-      Thread.sleep(100L + (long) (r.nextDouble() * 400D));
-    } catch (InterruptedException e) {
-      // eat
-    }
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/Granularity.java b/proxy/src/main/java/com/wavefront/agent/histogram/Granularity.java
new file mode 100644
index 000000000..44e00e19b
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/histogram/Granularity.java
@@ -0,0 +1,63 @@
+package com.wavefront.agent.histogram;
+
+import org.apache.commons.lang.time.DateUtils;
+
+/**
+ * Standard supported aggregation Granularities. Refactored from HistogramUtils.
+ *
+ * @author Tim Schmidt (tim@wavefront.com)
+ * @author vasily@wavefront.com
+ */
+public enum Granularity {
+  MINUTE((int) DateUtils.MILLIS_PER_MINUTE),
+  HOUR((int) DateUtils.MILLIS_PER_HOUR),
+  DAY((int) DateUtils.MILLIS_PER_DAY);
+
+  private final int inMillis;
+
+  Granularity(int inMillis) {
+    this.inMillis = inMillis;
+  }
+
+  /**
+   * Duration of a corresponding bin in milliseconds.
+   *
+   * @return bin length in milliseconds
+   */
+  public int getInMillis() {
+    return inMillis;
+  }
+
+  /**
+   * Bin id for an epoch time is the epoch time in the corresponding granularity.
+   *
+   * @param timeMillis epoch time in milliseconds
+   * @return the bin id
+   */
+  public int getBinId(long timeMillis) {
+    return (int) (timeMillis / inMillis);
+  }
+
+  @Override
+  public String toString() {
+    switch (this) {
+      case DAY:
+        return "day";
+      case HOUR:
+        return "hour";
+      case MINUTE:
+        return "minute";
+    }
+    return "unknown";
+  }
+
+  public static Granularity fromMillis(long millis) {
+    if (millis <= 60 * 1000) {
+      return MINUTE;
+    } else if (millis <= 60 * 60 * 1000) {
+      return HOUR;
+    } else {
+      return DAY;
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/HistogramKey.java b/proxy/src/main/java/com/wavefront/agent/histogram/HistogramKey.java
new file mode 100644
index 000000000..01986b88b
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/histogram/HistogramKey.java
@@ -0,0 +1,143 @@
+package com.wavefront.agent.histogram;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * Uniquely identifies a time-series - time-interval pair. These are the base sample aggregation
+ * scopes on the agent. Refactored from HistogramUtils.
+ *
+ * @author Tim Schmidt (tim@wavefront.com)
+ * @author vasily@wavefront.com
+ */
+public class HistogramKey {
+  // NOTE: fields are not final to allow object reuse
+  private byte granularityOrdinal;
+  private int binId;
+  private String metric;
+  @Nullable private String source;
+  @Nullable private String[] tags;
+
+  HistogramKey(
+      byte granularityOrdinal,
+      int binId,
+      @Nonnull String metric,
+      @Nullable String source,
+      @Nullable String[] tags) {
+    this.granularityOrdinal = granularityOrdinal;
+    this.binId = binId;
+    this.metric = metric;
+    this.source = source;
+    this.tags = ((tags == null || tags.length == 0) ? null : tags);
+  }
+
+  HistogramKey() {}
+
+  public byte getGranularityOrdinal() {
+    return granularityOrdinal;
+  }
+
+  public int getBinId() {
+    return binId;
+  }
+
+  public String getMetric() {
+    return metric;
+  }
+
+  @Nullable
+  public String getSource() {
+    return source;
+  }
+
+  @Nullable
+  public String[] getTags() {
+    return tags;
+  }
+
+  @Override
+  public String toString() {
+    return "HistogramKey{"
+        + "granularityOrdinal="
+        + granularityOrdinal
+        + ", binId="
+        + binId
+        + ", metric='"
+        + metric
+        + '\''
+        + ", source='"
+        + source
+        + '\''
+        + ", tags="
+        + Arrays.toString(tags)
+        + '}';
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    HistogramKey histogramKey = (HistogramKey) o;
+    if (granularityOrdinal != histogramKey.granularityOrdinal) return false;
+    if (binId != histogramKey.binId) return false;
+    if (!metric.equals(histogramKey.metric)) return false;
+    if (!Objects.equals(source, histogramKey.source)) return false;
+    return Arrays.equals(tags, histogramKey.tags);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = 1;
+    result = 31 * result + (int) granularityOrdinal;
+    result = 31 * result + binId;
+    result = 31 * result + metric.hashCode();
+    result = 31 * result + (source != null ? source.hashCode() : 0);
+    result = 31 * result + Arrays.hashCode(tags);
+    return result;
+  }
+
+  /** Unpacks tags into a map. */
+  public Map getTagsAsMap() {
+    if (tags == null || tags.length == 0) {
+      return ImmutableMap.of();
+    }
+    Map annotations = new HashMap<>(tags.length / 2);
+    for (int i = 0; i < tags.length - 1; i += 2) {
+      annotations.put(tags[i], tags[i + 1]);
+    }
+    return annotations;
+  }
+
+  public long getBinTimeMillis() {
+    return getBinDurationInMillis() * binId;
+  }
+
+  public long getBinDurationInMillis() {
+    return Granularity.values()[granularityOrdinal].getInMillis();
+  }
+
+  void setGranularityOrdinal(byte granularityOrdinal) {
+    this.granularityOrdinal = granularityOrdinal;
+  }
+
+  void setBinId(int binId) {
+    this.binId = binId;
+  }
+
+  void setMetric(String metric) {
+    this.metric = metric;
+  }
+
+  void setSource(@Nullable String source) {
+    this.source = source;
+  }
+
+  void setTags(@Nullable String[] tags) {
+    this.tags = tags;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/HistogramLineIngester.java b/proxy/src/main/java/com/wavefront/agent/histogram/HistogramLineIngester.java
deleted file mode 100644
index 1609ef989..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/HistogramLineIngester.java
+++ /dev/null
@@ -1,163 +0,0 @@
-package com.wavefront.agent.histogram;
-
-import com.google.common.base.Charsets;
-
-import com.wavefront.common.TaggedMetricName;
-import com.wavefront.metrics.ExpectedAgentMetric;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-
-import java.net.BindException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelDuplexHandler;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelOption;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.ServerChannel;
-import io.netty.channel.epoll.Epoll;
-import io.netty.channel.epoll.EpollEventLoopGroup;
-import io.netty.channel.epoll.EpollServerSocketChannel;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.codec.LineBasedFrameDecoder;
-import io.netty.handler.codec.string.StringDecoder;
-import io.netty.handler.timeout.IdleState;
-import io.netty.handler.timeout.IdleStateEvent;
-import io.netty.handler.timeout.IdleStateHandler;
-
-/**
- * A {@link ChannelInitializer} for Histogram samples via TCP.
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class HistogramLineIngester extends ChannelInitializer implements Runnable {
-  /**
-   * Default number of seconds before the channel idle timeout handler closes the connection.
-   */
-  private static final int CHANNEL_IDLE_TIMEOUT_IN_SECS_DEFAULT = (int) TimeUnit.DAYS.toSeconds(1);
-  private static final int MAXIMUM_OUTSTANDING_CONNECTIONS = 1024;
-
-  private static final AtomicLong connectionId = new AtomicLong(0);
-
-  private static final Logger logger = Logger.getLogger(HistogramLineIngester.class.getCanonicalName());
-  private final Counter activeListeners = Metrics.newCounter(ExpectedAgentMetric.ACTIVE_LISTENERS.metricName);
-  private final Counter bindErrors = Metrics.newCounter(ExpectedAgentMetric.LISTENERS_BIND_ERRORS.metricName);
-  private final Counter connectionsAccepted;
-  private final Counter connectionsIdleClosed;
-
-  // The final handlers to be installed.
-  private final ArrayList handlers;
-  private final int port;
-  private int maxLength = 64 * 1024;
-  private int channelIdleTimeout = CHANNEL_IDLE_TIMEOUT_IN_SECS_DEFAULT;
-
-
-  public HistogramLineIngester(Collection handlers, int port) {
-    this.handlers = new ArrayList<>(handlers);
-    this.port = port;
-    this.connectionsAccepted = Metrics.newCounter(new TaggedMetricName("listeners", "connections.accepted",
-        "port", String.valueOf(port)));
-    this.connectionsIdleClosed = Metrics.newCounter(new TaggedMetricName("listeners", "connections.idle.closed",
-        "port", String.valueOf(port)));
-  }
-
-  public HistogramLineIngester withMaxLength(int maxLength) {
-    this.maxLength = maxLength;
-    return this;
-  }
-
-  public HistogramLineIngester withChannelIdleTimeout(int channelIdleTimeout) {
-    this.channelIdleTimeout = channelIdleTimeout;
-    return this;
-  }
-
-  @Override
-  public void run() {
-    activeListeners.inc();
-    ServerBootstrap bootstrap = new ServerBootstrap();
-
-    EventLoopGroup parent;
-    EventLoopGroup children;
-    Class socketChannelClass;
-    if (Epoll.isAvailable()) {
-      logger.fine("Using native socket transport for port " + port);
-      parent = new EpollEventLoopGroup(1);
-      children = new EpollEventLoopGroup(handlers.size());
-      socketChannelClass = EpollServerSocketChannel.class;
-    } else {
-      logger.fine("Using NIO socket transport for port " + port);
-      parent = new NioEventLoopGroup(1);
-      children = new NioEventLoopGroup(handlers.size());
-      socketChannelClass = NioServerSocketChannel.class;
-    }
-
-    try {
-      bootstrap
-          .group(parent, children)
-          .channel(socketChannelClass)
-          .option(ChannelOption.SO_BACKLOG, MAXIMUM_OUTSTANDING_CONNECTIONS)
-          .localAddress(port)
-          .childHandler(this);
-
-      ChannelFuture f = bootstrap.bind().sync();
-      f.channel().closeFuture().sync();
-    } catch (final InterruptedException e) {
-      logger.log(Level.WARNING, "Interrupted");
-      parent.shutdownGracefully();
-      children.shutdownGracefully();
-      logger.info("Listener on port " + String.valueOf(port) + " shut down");
-    } catch (Exception e) {
-      // ChannelFuture throws undeclared checked exceptions, so we need to handle it
-      if (e instanceof BindException) {
-        bindErrors.inc();
-        logger.severe("Unable to start listener - port " + String.valueOf(port) + " is already in use!");
-      } else {
-        logger.log(Level.SEVERE, "HistogramLineIngester exception: ", e);
-      }
-    } finally {
-      activeListeners.dec();
-    }
-  }
-
-  @Override
-  protected void initChannel(Channel ch) throws Exception {
-    // Round robin channel to handler assignment.
-    int idx = (int) (Math.abs(connectionId.getAndIncrement()) % handlers.size());
-    ChannelHandler handler = handlers.get(idx);
-    connectionsAccepted.inc();
-
-    // Add decoders and timeout, add handler()
-    ChannelPipeline pipeline = ch.pipeline();
-    pipeline.addLast(
-        new LineBasedFrameDecoder(maxLength, true, false),
-        new StringDecoder(Charsets.UTF_8),
-        new IdleStateHandler(channelIdleTimeout, 0, 0),
-        new ChannelDuplexHandler() {
-          @Override
-          public void userEventTriggered(ChannelHandlerContext ctx,
-                                         Object evt) throws Exception {
-            if (evt instanceof IdleStateEvent) {
-              if (((IdleStateEvent) evt).state() == IdleState.READER_IDLE) {
-                connectionsIdleClosed.inc();
-                logger.info("Closing idle connection to histogram client, inactivity timeout " +
-                    channelIdleTimeout + "s expired: " + ctx.channel());
-                ctx.close();
-              }
-            }
-          }
-        },
-        handler);
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/HistogramRecompressor.java b/proxy/src/main/java/com/wavefront/agent/histogram/HistogramRecompressor.java
new file mode 100644
index 000000000..6372dbdd6
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/histogram/HistogramRecompressor.java
@@ -0,0 +1,102 @@
+package com.wavefront.agent.histogram;
+
+import static com.wavefront.agent.histogram.HistogramUtils.mergeHistogram;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.tdunning.math.stats.AgentDigest;
+import com.wavefront.common.TaggedMetricName;
+import com.wavefront.common.Utils;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import wavefront.report.Histogram;
+import wavefront.report.HistogramType;
+
+/**
+ * Recompresses histograms to reduce their size.
+ *
+ * @author vasily@wavefront.com
+ */
+public class HistogramRecompressor implements Function {
+  private final Supplier storageAccuracySupplier;
+  private final Supplier histogramsCompacted =
+      Utils.lazySupplier(
+          () -> Metrics.newCounter(new TaggedMetricName("histogram", "histograms_compacted")));
+  private final Supplier histogramsRecompressed =
+      Utils.lazySupplier(
+          () -> Metrics.newCounter(new TaggedMetricName("histogram", "histograms_recompressed")));
+
+  /** @param storageAccuracySupplier Supplier for histogram storage accuracy */
+  public HistogramRecompressor(Supplier storageAccuracySupplier) {
+    this.storageAccuracySupplier = storageAccuracySupplier;
+  }
+
+  @Override
+  public Histogram apply(Histogram input) {
+    Histogram result = input;
+    if (hasDuplicateCentroids(input)) {
+      // merge centroids with identical values first, and if we get the number of centroids
+      // low enough, we might not need to incur recompression overhead after all.
+      result = compactCentroids(input);
+      histogramsCompacted.get().inc();
+    }
+    if (result.getBins().size() > 2 * storageAccuracySupplier.get()) {
+      AgentDigest digest = new AgentDigest(storageAccuracySupplier.get(), 0);
+      mergeHistogram(digest, result);
+      digest.compress();
+      result = digest.toHistogram(input.getDuration());
+      histogramsRecompressed.get().inc();
+    }
+    return result;
+  }
+
+  @VisibleForTesting
+  static boolean hasDuplicateCentroids(wavefront.report.Histogram histogram) {
+    Set uniqueBins = new HashSet<>();
+    for (Double bin : histogram.getBins()) {
+      if (!uniqueBins.add(bin)) return true;
+    }
+    return false;
+  }
+
+  @VisibleForTesting
+  static wavefront.report.Histogram compactCentroids(wavefront.report.Histogram histogram) {
+    List bins = histogram.getBins();
+    List counts = histogram.getCounts();
+    int numCentroids = Math.min(bins.size(), counts.size());
+
+    List newBins = new ArrayList<>();
+    List newCounts = new ArrayList<>();
+
+    Double accumulatedValue = null;
+    int accumulatedCount = 0;
+    for (int i = 0; i < numCentroids; ++i) {
+      double value = bins.get(i);
+      int count = counts.get(i);
+      if (accumulatedValue == null) {
+        accumulatedValue = value;
+      } else if (value != accumulatedValue) {
+        newBins.add(accumulatedValue);
+        newCounts.add(accumulatedCount);
+        accumulatedValue = value;
+        accumulatedCount = 0;
+      }
+      accumulatedCount += count;
+    }
+    if (accumulatedValue != null) {
+      newCounts.add(accumulatedCount);
+      newBins.add(accumulatedValue);
+    }
+    return wavefront.report.Histogram.newBuilder()
+        .setDuration(histogram.getDuration())
+        .setBins(newBins)
+        .setCounts(newCounts)
+        .setType(HistogramType.TDIGEST)
+        .build();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/HistogramUtils.java b/proxy/src/main/java/com/wavefront/agent/histogram/HistogramUtils.java
new file mode 100644
index 000000000..ba94fafa0
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/histogram/HistogramUtils.java
@@ -0,0 +1,210 @@
+package com.wavefront.agent.histogram;
+
+import com.google.common.base.Preconditions;
+import com.tdunning.math.stats.AgentDigest;
+import com.tdunning.math.stats.TDigest;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.MetricName;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import net.openhft.chronicle.bytes.Bytes;
+import net.openhft.chronicle.core.io.IORuntimeException;
+import net.openhft.chronicle.core.util.ReadResolvable;
+import net.openhft.chronicle.hash.serialization.BytesReader;
+import net.openhft.chronicle.hash.serialization.BytesWriter;
+import net.openhft.chronicle.wire.WireIn;
+import net.openhft.chronicle.wire.WireOut;
+import wavefront.report.ReportPoint;
+
+/**
+ * Helpers around histograms
+ *
+ * @author Tim Schmidt (tim@wavefront.com).
+ */
+public final class HistogramUtils {
+  private HistogramUtils() {
+    // Not instantiable
+  }
+
+  /**
+   * Generates a {@link HistogramKey} according a prototype {@link ReportPoint} and {@link
+   * Granularity}.
+   */
+  public static HistogramKey makeKey(ReportPoint point, Granularity granularity) {
+    Preconditions.checkNotNull(point);
+    Preconditions.checkNotNull(granularity);
+
+    String[] annotations = null;
+    if (point.getAnnotations() != null) {
+      List> keyOrderedTags =
+          point.getAnnotations().entrySet().stream()
+              .sorted(Map.Entry.comparingByKey())
+              .collect(Collectors.toList());
+      annotations = new String[keyOrderedTags.size() * 2];
+      for (int i = 0; i < keyOrderedTags.size(); ++i) {
+        annotations[2 * i] = keyOrderedTags.get(i).getKey();
+        annotations[(2 * i) + 1] = keyOrderedTags.get(i).getValue();
+      }
+    }
+
+    return new HistogramKey(
+        (byte) granularity.ordinal(),
+        granularity.getBinId(point.getTimestamp()),
+        point.getMetric(),
+        point.getHost(),
+        annotations);
+  }
+
+  /**
+   * Creates a {@link ReportPoint} from a {@link HistogramKey} - {@link AgentDigest} pair
+   *
+   * @param histogramKey the key, defining metric, source, annotations, duration and start-time
+   * @param agentDigest the digest defining the centroids
+   * @return the corresponding point
+   */
+  public static ReportPoint pointFromKeyAndDigest(
+      HistogramKey histogramKey, AgentDigest agentDigest) {
+    return ReportPoint.newBuilder()
+        .setTimestamp(histogramKey.getBinTimeMillis())
+        .setMetric(histogramKey.getMetric())
+        .setHost(histogramKey.getSource())
+        .setAnnotations(histogramKey.getTagsAsMap())
+        .setTable("dummy")
+        .setValue(agentDigest.toHistogram((int) histogramKey.getBinDurationInMillis()))
+        .build();
+  }
+
+  /**
+   * Convert granularity to string. If null, we assume we are dealing with "distribution" port.
+   *
+   * @param granularity granularity
+   * @return string representation
+   */
+  public static String granularityToString(@Nullable Granularity granularity) {
+    return granularity == null ? "distribution" : granularity.toString();
+  }
+
+  /**
+   * Merges a histogram into a TDigest
+   *
+   * @param target target TDigest
+   * @param source histogram to merge
+   */
+  public static void mergeHistogram(final TDigest target, final wavefront.report.Histogram source) {
+    List means = source.getBins();
+    List counts = source.getCounts();
+
+    if (means != null && counts != null) {
+      int len = Math.min(means.size(), counts.size());
+
+      for (int i = 0; i < len; ++i) {
+        Integer count = counts.get(i);
+        Double mean = means.get(i);
+
+        if (count != null && count > 0 && mean != null && Double.isFinite(mean)) {
+          target.add(mean, count);
+        }
+      }
+    }
+  }
+
+  /**
+   * (For now, a rather trivial) encoding of {@link HistogramKey} the form short length and bytes
+   *
+   * 
Consider using chronicle-values or making this stateful with a local byte[] / Stringbuffers
+   * to be a little more efficient about encodings.
+   */
+  public static class HistogramKeyMarshaller
+      implements BytesReader,
+          BytesWriter,
+          ReadResolvable {
+    private static final HistogramKeyMarshaller INSTANCE = new HistogramKeyMarshaller();
+
+    private static final Histogram accumulatorKeySizes =
+        Metrics.newHistogram(new MetricName("histogram", "", "accumulatorKeySize"));
+
+    private HistogramKeyMarshaller() {
+      // Private Singleton
+    }
+
+    public static HistogramKeyMarshaller get() {
+      return INSTANCE;
+    }
+
+    @Nonnull
+    @Override
+    public HistogramKeyMarshaller readResolve() {
+      return INSTANCE;
+    }
+
+    private static void writeString(Bytes out, String s) {
+      Preconditions.checkArgument(
+          s == null || s.length() <= Short.MAX_VALUE, "String too long (more than 32K)");
+      byte[] bytes = s == null ? new byte[0] : s.getBytes(StandardCharsets.UTF_8);
+      out.writeShort((short) bytes.length);
+      out.write(bytes);
+    }
+
+    private static String readString(Bytes in) {
+      byte[] bytes = new byte[in.readShort()];
+      in.read(bytes);
+      return new String(bytes);
+    }
+
+    @Override
+    public void readMarshallable(@Nonnull WireIn wire) throws IORuntimeException {
+      // ignore, stateless
+    }
+
+    @Override
+    public void writeMarshallable(@Nonnull WireOut wire) {
+      // ignore, stateless
+    }
+
+    @Nonnull
+    @Override
+    public HistogramKey read(Bytes in, @Nullable HistogramKey using) {
+      if (using == null) {
+        using = new HistogramKey();
+      }
+      using.setGranularityOrdinal(in.readByte());
+      using.setBinId(in.readInt());
+      using.setMetric(readString(in));
+      using.setSource(readString(in));
+      int numTags = in.readShort();
+      if (numTags > 0) {
+        final String[] tags = new String[numTags];
+        for (int i = 0; i < numTags; ++i) {
+          tags[i] = readString(in);
+        }
+        using.setTags(tags);
+      }
+      return using;
+    }
+
+    @Override
+    public void write(Bytes out, @Nonnull HistogramKey toWrite) {
+      int accumulatorKeySize = 5;
+      out.writeByte(toWrite.getGranularityOrdinal());
+      out.writeInt(toWrite.getBinId());
+      accumulatorKeySize += 2 + toWrite.getMetric().length();
+      writeString(out, toWrite.getMetric());
+      accumulatorKeySize += 2 + (toWrite.getSource() == null ? 0 : toWrite.getSource().length());
+      writeString(out, toWrite.getSource());
+      short numTags = toWrite.getTags() == null ? 0 : (short) toWrite.getTags().length;
+      accumulatorKeySize += 2;
+      out.writeShort(numTags);
+      for (short i = 0; i < numTags; ++i) {
+        final String tag = toWrite.getTags()[i];
+        accumulatorKeySize += 2 + (tag == null ? 0 : tag.length());
+        writeString(out, tag);
+      }
+      accumulatorKeySizes.update(accumulatorKeySize);
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/MapLoader.java b/proxy/src/main/java/com/wavefront/agent/histogram/MapLoader.java
index fa01a8c16..f15ff1368 100644
--- a/proxy/src/main/java/com/wavefront/agent/histogram/MapLoader.java
+++ b/proxy/src/main/java/com/wavefront/agent/histogram/MapLoader.java
@@ -1,52 +1,46 @@
 package com.wavefront.agent.histogram;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Preconditions;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-import com.wavefront.agent.ResubmissionTask;
-import com.wavefront.agent.ResubmissionTaskDeserializer;
-
-import net.openhft.chronicle.hash.serialization.BytesReader;
-import net.openhft.chronicle.hash.serialization.BytesWriter;
-import net.openhft.chronicle.hash.serialization.SizedReader;
-import net.openhft.chronicle.hash.serialization.SizedWriter;
-import net.openhft.chronicle.map.ChronicleMap;
-import net.openhft.chronicle.map.VanillaChronicleMap;
-
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.io.Reader;
 import java.io.Writer;
-import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-
-import javax.validation.constraints.NotNull;
+import javax.annotation.Nonnull;
+import net.openhft.chronicle.hash.serialization.BytesReader;
+import net.openhft.chronicle.hash.serialization.BytesWriter;
+import net.openhft.chronicle.hash.serialization.SizedReader;
+import net.openhft.chronicle.hash.serialization.SizedWriter;
+import net.openhft.chronicle.map.ChronicleMap;
+import net.openhft.chronicle.map.VanillaChronicleMap;
 
 /**
- * Loader for {@link ChronicleMap}. If a file already exists at the given location, will make an attempt to load the map
- * from the existing file. Will fall-back to an in memory representation if the file cannot be loaded (see logs).
+ * Loader for {@link ChronicleMap}. If a file already exists at the given location, will make an
+ * attempt to load the map from the existing file. Will fall-back to an in memory representation if
+ * the file cannot be loaded (see logs).
  *
  * @author Tim Schmidt (tim@wavefront.com).
  */
-public class MapLoader & BytesWriter, VM extends SizedReader & SizedWriter> {
+public class MapLoader<
+    K, V, KM extends BytesReader & BytesWriter, VM extends SizedReader & SizedWriter> {
   private static final Logger logger = Logger.getLogger(MapLoader.class.getCanonicalName());
 
   /**
-   * Allow ChronicleMap to grow beyond initially allocated size instead of crashing. Since it makes the map a lot less
-   * efficient, we should log a warning if the actual number of elements exceeds the allocated.
-   * A bloat factor of 1000 is the highest possible value which we are going to use here, as we need to prevent
-   * crashes at all costs.
+   * Allow ChronicleMap to grow beyond initially allocated size instead of crashing. Since it makes
+   * the map a lot less efficient, we should log a warning if the actual number of elements exceeds
+   * the allocated. A bloat factor of 1000 is the highest possible value which we are going to use
+   * here, as we need to prevent crashes at all costs.
    */
   private static final double MAX_BLOAT_FACTOR = 1000;
 
+  private static final ObjectMapper JSON_PARSER = new ObjectMapper();
+
   private final Class keyClass;
   private final Class valueClass;
   private final long entries;
@@ -56,144 +50,162 @@ public class MapLoader & BytesWriter, VM exte
   private final VM valueMarshaller;
   private final boolean doPersist;
   private final LoadingCache> maps =
-      CacheBuilder.newBuilder().build(new CacheLoader>() {
-
-        private ChronicleMap newPersistedMap(File file) throws IOException {
-          return ChronicleMap.of(keyClass, valueClass)
-              .keyMarshaller(keyMarshaller)
-              .valueMarshaller(valueMarshaller)
-              .entries(entries)
-              .averageKeySize(avgKeySize)
-              .averageValueSize(avgValueSize)
-              .maxBloatFactor(MAX_BLOAT_FACTOR)
-              .createPersistedTo(file);
-        }
-
-        private ChronicleMap newInMemoryMap() {
-          return ChronicleMap.of(keyClass, valueClass)
-              .keyMarshaller(keyMarshaller)
-              .valueMarshaller(valueMarshaller)
-              .entries(entries)
-              .averageKeySize(avgKeySize)
-              .averageValueSize(avgValueSize)
-              .maxBloatFactor(MAX_BLOAT_FACTOR)
-              .create();
-        }
-
-        private MapSettings loadSettings(File file) throws IOException {
-          Gson gson = new GsonBuilder().
-              registerTypeHierarchyAdapter(Class.class, new MapSettings.ClassNameSerializer()).create();
-          Reader br = new BufferedReader(new FileReader(file));
-          return gson.fromJson(br, MapSettings.class);
-        }
-
-        private void saveSettings(MapSettings settings, File file) throws IOException {
-          Gson gson = new GsonBuilder().
-              registerTypeHierarchyAdapter(Class.class, new MapSettings.ClassNameSerializer()).create();
-          Writer writer = new FileWriter(file);
-          gson.toJson(settings, writer);
-          writer.close();
-        }
-
-        @Override
-        public ChronicleMap load(@NotNull File file) throws Exception {
-          if (!doPersist) {
-            logger.log(
-                Level.WARNING,
-                "Accumulator persistence is disabled, unflushed histograms will be lost on proxy shutdown."
-            );
-            return newInMemoryMap();
-          }
-
-          MapSettings newSettings = new MapSettings(keyClass, valueClass,
-              keyMarshaller.getClass(), valueMarshaller.getClass(), entries, avgKeySize, avgValueSize);
-          File settingsFile = new File(file.getAbsolutePath().concat(".settings"));
-          try {
-            if (file.exists()) {
-              if (settingsFile.exists()) {
-                MapSettings settings = loadSettings(settingsFile);
-                if (!settings.equals(newSettings)) {
-                  logger.info(file.getName() + " settings changed, reconfiguring (this may take a few moments)...");
-                  File originalFile = new File(file.getAbsolutePath());
-                  File oldFile = new File(file.getAbsolutePath().concat(".temp"));
-                  if (oldFile.exists()) {
-                    oldFile.delete();
-                  }
-                  file.renameTo(oldFile);
+      CacheBuilder.newBuilder()
+          .build(
+              new CacheLoader>() {
 
-                  ChronicleMap toMigrate = ChronicleMap
-                      .of(keyClass, valueClass)
-                      .entries(settings.getEntries())
-                      .averageKeySize(settings.getAvgKeySize())
-                      .averageValueSize(settings.getAvgValueSize())
-                      .recoverPersistedTo(oldFile, false);
+                private ChronicleMap newPersistedMap(File file) throws IOException {
+                  return ChronicleMap.of(keyClass, valueClass)
+                      .keyMarshaller(keyMarshaller)
+                      .valueMarshaller(valueMarshaller)
+                      .entries(entries)
+                      .averageKeySize(avgKeySize)
+                      .averageValueSize(avgValueSize)
+                      .maxBloatFactor(MAX_BLOAT_FACTOR)
+                      .createPersistedTo(file);
+                }
 
-                  ChronicleMap result = newPersistedMap(originalFile);
+                private ChronicleMap newInMemoryMap() {
+                  return ChronicleMap.of(keyClass, valueClass)
+                      .keyMarshaller(keyMarshaller)
+                      .valueMarshaller(valueMarshaller)
+                      .entries(entries)
+                      .averageKeySize(avgKeySize)
+                      .averageValueSize(avgValueSize)
+                      .maxBloatFactor(MAX_BLOAT_FACTOR)
+                      .create();
+                }
 
-                  if (toMigrate.size() > 0) {
-                    logger.info(originalFile.getName() + " starting data migration (" + toMigrate.size() + " records)");
-                    for (K key : toMigrate.keySet()) {
-                      result.put(key, toMigrate.get(key));
-                    }
-                    toMigrate.close();
-                    logger.info(originalFile.getName() + " data migration finished");
+                private MapSettings loadSettings(File file) throws IOException {
+                  return JSON_PARSER.readValue(new FileReader(file), MapSettings.class);
+                }
+
+                private void saveSettings(MapSettings settings, File file) throws IOException {
+                  Writer writer = new FileWriter(file);
+                  JSON_PARSER.writeValue(writer, settings);
+                  writer.close();
+                }
+
+                @Override
+                public ChronicleMap load(@Nonnull File file) throws Exception {
+                  if (!doPersist) {
+                    logger.log(
+                        Level.WARNING,
+                        "Accumulator persistence is disabled, unflushed histograms "
+                            + "will be lost on proxy shutdown.");
+                    return newInMemoryMap();
                   }
 
-                  saveSettings(newSettings, settingsFile);
-                  oldFile.delete();
-                  logger.info(originalFile.getName() + " reconfiguration finished");
+                  MapSettings newSettings = new MapSettings(entries, avgKeySize, avgValueSize);
+                  File settingsFile = new File(file.getAbsolutePath().concat(".settings"));
+                  try {
+                    if (file.exists()) {
+                      if (settingsFile.exists()) {
+                        MapSettings settings = loadSettings(settingsFile);
+                        if (!settings.equals(newSettings)) {
+                          logger.info(
+                              file.getName()
+                                  + " settings changed, reconfiguring (this may take a few moments)...");
+                          File originalFile = new File(file.getAbsolutePath());
+                          File oldFile = new File(file.getAbsolutePath().concat(".temp"));
+                          if (oldFile.exists()) {
+                            //noinspection ResultOfMethodCallIgnored
+                            oldFile.delete();
+                          }
+                          //noinspection ResultOfMethodCallIgnored
+                          file.renameTo(oldFile);
 
-                  return result;
-                }
-              }
-
-              logger.fine("Restoring accumulator state from " + file.getAbsolutePath());
-              // Note: this relies on an uncorrupted header, which according to the docs would be due to a hardware error or fs bug.
-              ChronicleMap result = ChronicleMap
-                  .of(keyClass, valueClass)
-                  .entries(entries)
-                  .averageKeySize(avgKeySize)
-                  .averageValueSize(avgValueSize)
-                  .recoverPersistedTo(file, false);
-
-              if (result.isEmpty()) {
-                // Create a new map with the supplied settings to be safe.
-                result.close();
-                file.delete();
-                logger.fine("Empty accumulator - reinitializing: " + file.getName());
-                result = newPersistedMap(file);
-              } else {
-                // Note: as of 3.10 all instances are.
-                if (result instanceof VanillaChronicleMap) {
-                  logger.fine("Accumulator map restored from " + file.getAbsolutePath());
-                  VanillaChronicleMap vcm = (VanillaChronicleMap) result;
-                  if (!vcm.keyClass().equals(keyClass) ||
-                      !vcm.valueClass().equals(valueClass)) {
-                    throw new IllegalStateException("Persisted map params are not matching expected map params "
-                        + " key " + "exp: " + keyClass.getSimpleName() + " act: " + vcm.keyClass().getSimpleName()
-                        + " val " + "exp: " + valueClass.getSimpleName() + " act: " + vcm.valueClass().getSimpleName());
+                          ChronicleMap toMigrate =
+                              ChronicleMap.of(keyClass, valueClass)
+                                  .entries(settings.getEntries())
+                                  .averageKeySize(settings.getAvgKeySize())
+                                  .averageValueSize(settings.getAvgValueSize())
+                                  .recoverPersistedTo(oldFile, false);
+
+                          ChronicleMap result = newPersistedMap(originalFile);
+
+                          if (toMigrate.size() > 0) {
+                            logger.info(
+                                originalFile.getName()
+                                    + " starting data migration ("
+                                    + toMigrate.size()
+                                    + " records)");
+                            for (K key : toMigrate.keySet()) {
+                              result.put(key, toMigrate.get(key));
+                            }
+                            toMigrate.close();
+                            logger.info(originalFile.getName() + " data migration finished");
+                          }
+
+                          saveSettings(newSettings, settingsFile);
+                          //noinspection ResultOfMethodCallIgnored
+                          oldFile.delete();
+                          logger.info(originalFile.getName() + " reconfiguration finished");
+
+                          return result;
+                        }
+                      }
+
+                      logger.fine("Restoring accumulator state from " + file.getAbsolutePath());
+                      // Note: this relies on an uncorrupted header, which
+                      // according to the docs
+                      // would be due to a hardware error or fs bug.
+                      ChronicleMap result =
+                          ChronicleMap.of(keyClass, valueClass)
+                              .entries(entries)
+                              .averageKeySize(avgKeySize)
+                              .averageValueSize(avgValueSize)
+                              .recoverPersistedTo(file, false);
+
+                      if (result.isEmpty()) {
+                        // Create a new map with the supplied settings to be
+                        // safe.
+                        result.close();
+                        //noinspection ResultOfMethodCallIgnored
+                        file.delete();
+                        logger.fine("Empty accumulator - reinitializing: " + file.getName());
+                        result = newPersistedMap(file);
+                      } else {
+                        // Note: as of 3.10 all instances are.
+                        if (result instanceof VanillaChronicleMap) {
+                          logger.fine("Accumulator map restored from " + file.getAbsolutePath());
+                          VanillaChronicleMap vcm = (VanillaChronicleMap) result;
+                          if (!vcm.keyClass().equals(keyClass)
+                              || !vcm.valueClass().equals(valueClass)) {
+                            throw new IllegalStateException(
+                                "Persisted map params are not matching expected map params "
+                                    + " key "
+                                    + "exp: "
+                                    + keyClass.getSimpleName()
+                                    + " act: "
+                                    + vcm.keyClass().getSimpleName()
+                                    + " val "
+                                    + "exp: "
+                                    + valueClass.getSimpleName()
+                                    + " act: "
+                                    + vcm.valueClass().getSimpleName());
+                          }
+                        }
+                      }
+                      saveSettings(newSettings, settingsFile);
+                      return result;
+
+                    } else {
+                      logger.fine("Accumulator map initialized as " + file.getName());
+                      saveSettings(newSettings, settingsFile);
+                      return newPersistedMap(file);
+                    }
+                  } catch (Exception e) {
+                    logger.log(
+                        Level.SEVERE,
+                        "Failed to load/create map from '"
+                            + file.getAbsolutePath()
+                            + "'. Please move or delete the file and restart the proxy! Reason: ",
+                        e);
+                    throw new RuntimeException(e);
                   }
                 }
-              }
-              saveSettings(newSettings, settingsFile);
-              return result;
-
-            } else {
-              logger.fine("Accumulator map initialized as " + file.getName());
-              saveSettings(newSettings, settingsFile);
-              return newPersistedMap(file);
-            }
-          } catch (Exception e) {
-            logger.log(
-                Level.SEVERE,
-                "Failed to load/create map from '" + file.getAbsolutePath() +
-                    "'. Please move or delete the file and restart the proxy! Reason: ",
-                e);
-            System.exit(-1);
-            return null;
-          }
-        }
-      });
+              });
 
   /**
    * Creates a new {@link MapLoader}
@@ -207,14 +219,15 @@ public ChronicleMap load(@NotNull File file) throws Exception {
    * @param valueMarshaller the value codec
    * @param doPersist whether to persist the map
    */
-  public MapLoader(Class keyClass,
-                   Class valueClass,
-                   long entries,
-                   double avgKeySize,
-                   double avgValueSize,
-                   KM keyMarshaller,
-                   VM valueMarshaller,
-                   boolean doPersist) {
+  public MapLoader(
+      Class keyClass,
+      Class valueClass,
+      long entries,
+      double avgKeySize,
+      double avgValueSize,
+      KM keyMarshaller,
+      VM valueMarshaller,
+      boolean doPersist) {
     this.keyClass = keyClass;
     this.valueClass = valueClass;
     this.entries = entries;
@@ -225,28 +238,32 @@ public MapLoader(Class keyClass,
     this.doPersist = doPersist;
   }
 
-  public ChronicleMap get(File f) {
+  public ChronicleMap get(File f) throws Exception {
     Preconditions.checkNotNull(f);
-    try {
-      return maps.get(f);
-    } catch (Exception e) {
-      logger.log(Level.SEVERE, "Failed loading map for " + f, e);
-      return null;
-    }
+    return maps.get(f);
   }
 
   @Override
   public String toString() {
-    return "MapLoader{" +
-        "keyClass=" + keyClass +
-        ", valueClass=" + valueClass +
-        ", entries=" + entries +
-        ", avgKeySize=" + avgKeySize +
-        ", avgValueSize=" + avgValueSize +
-        ", keyMarshaller=" + keyMarshaller +
-        ", valueMarshaller=" + valueMarshaller +
-        ", doPersist=" + doPersist +
-        ", maps=" + maps +
-        '}';
+    return "MapLoader{"
+        + "keyClass="
+        + keyClass
+        + ", valueClass="
+        + valueClass
+        + ", entries="
+        + entries
+        + ", avgKeySize="
+        + avgKeySize
+        + ", avgValueSize="
+        + avgValueSize
+        + ", keyMarshaller="
+        + keyMarshaller
+        + ", valueMarshaller="
+        + valueMarshaller
+        + ", doPersist="
+        + doPersist
+        + ", maps="
+        + maps
+        + '}';
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/MapSettings.java b/proxy/src/main/java/com/wavefront/agent/histogram/MapSettings.java
index 616910c23..f7a9a0d1a 100644
--- a/proxy/src/main/java/com/wavefront/agent/histogram/MapSettings.java
+++ b/proxy/src/main/java/com/wavefront/agent/histogram/MapSettings.java
@@ -1,63 +1,38 @@
 package com.wavefront.agent.histogram;
 
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-
-import java.lang.reflect.Type;
+import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
- * Stores settings ChronicleMap has been initialized with to trigger map re-creation when settings change
- * (since ChronicleMap doesn't persist init values for entries/avgKeySize/avgValueSize)
+ * Stores settings ChronicleMap has been initialized with to trigger map re-creation when settings
+ * change (since ChronicleMap doesn't persist init values for entries/avgKeySize/avgValueSize)
  *
  * @author vasily@wavefront.com
  */
 public class MapSettings {
-  private final Class keyClass;
-  private final Class valueClass;
-  private final Class keyMarshaller;
-  private final Class valueMarshaller;
-  private final long entries;
-  private final double avgKeySize;
-  private final double avgValueSize;
+  private long entries;
+  private double avgKeySize;
+  private double avgValueSize;
+
+  @SuppressWarnings("unused")
+  private MapSettings() {}
 
-  public MapSettings(Class keyClass, Class valueClass, Class keyMarshaller, Class valueMarshaller,
-                     long entries, double avgKeySize, double avgValueSize) {
-    this.keyClass = keyClass;
-    this.valueClass = valueClass;
-    this.keyMarshaller = keyMarshaller;
-    this.valueMarshaller = valueMarshaller;
+  public MapSettings(long entries, double avgKeySize, double avgValueSize) {
     this.entries = entries;
     this.avgKeySize = avgKeySize;
     this.avgValueSize = avgValueSize;
   }
 
-  public Class getKeyClass() {
-    return keyClass;
-  }
-
-  public Class getValueClass() {
-    return valueClass;
-  }
-
-  public Class getKeyMarshaller() {
-    return keyMarshaller;
-  }
-
-  public Class getValueMarshaller() {
-    return valueMarshaller;
-  }
-
+  @JsonProperty
   public long getEntries() {
     return entries;
   }
 
+  @JsonProperty
   public double getAvgKeySize() {
     return avgKeySize;
   }
 
+  @JsonProperty
   public double getAvgValueSize() {
     return avgValueSize;
   }
@@ -66,45 +41,10 @@ public double getAvgValueSize() {
   public boolean equals(Object o) {
     if (this == o) return true;
     if (o == null || getClass() != o.getClass()) return false;
-    MapSettings that = (MapSettings)o;
+    MapSettings that = (MapSettings) o;
 
-    return (this.keyClass == that.keyClass
-        && this.valueClass == that.valueClass
-        && this.keyMarshaller == that.keyMarshaller
-        && this.valueMarshaller == that.valueMarshaller
-        && this.entries == that.entries
+    return (this.entries == that.entries
         && this.avgKeySize == that.avgKeySize
         && this.avgValueSize == that.avgValueSize);
   }
-
-  @Override
-  public String toString() {
-    return "MapSettings{" +
-        "keyClass=" + (keyClass == null ? "(null)" : keyClass.getName()) +
-        ", valueClass=" + (valueClass == null ? "(null)" : valueClass.getName()) +
-        ", keyMarshaller=" + (keyMarshaller == null ? "(null)" : keyMarshaller.getName()) +
-        ", valueMarshaller=" + (valueMarshaller == null ? "(null)" : valueMarshaller.getName()) +
-        ", entries=" + entries +
-        ", avgKeySize=" + avgKeySize +
-        ", avgValueSize=" + avgValueSize +
-        '}';
-  }
-
-  public static class ClassNameSerializer implements JsonSerializer, JsonDeserializer {
-    @Override
-    public JsonElement serialize(Class src, Type type, JsonSerializationContext context) {
-      return context.serialize(src.getName());
-    }
-    @Override
-    public Class deserialize(JsonElement src, Type type, JsonDeserializationContext context) {
-      try {
-        return Class.forName(src.getAsString());
-      } catch (ClassNotFoundException e) {
-        return null;
-      }
-    }
-
-  }
-
-
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/PointHandlerDispatcher.java b/proxy/src/main/java/com/wavefront/agent/histogram/PointHandlerDispatcher.java
index 61b2dae01..f6bb4e240 100644
--- a/proxy/src/main/java/com/wavefront/agent/histogram/PointHandlerDispatcher.java
+++ b/proxy/src/main/java/com/wavefront/agent/histogram/PointHandlerDispatcher.java
@@ -1,96 +1,107 @@
 package com.wavefront.agent.histogram;
 
-import com.google.common.annotations.VisibleForTesting;
-
-import com.wavefront.agent.PointHandler;
-import com.wavefront.agent.histogram.accumulator.AccumulationCache;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.histogram.accumulator.Accumulator;
+import com.wavefront.common.TimeProvider;
+import com.wavefront.common.logger.MessageDedupingLogger;
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.Gauge;
 import com.yammer.metrics.core.MetricName;
-
 import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-
 import javax.annotation.Nullable;
-
 import wavefront.report.ReportPoint;
 
-import static java.lang.System.nanoTime;
-
 /**
  * Dispatch task for marshalling "ripe" digests for shipment to the agent to a point handler.
  *
  * @author Tim Schmidt (tim@wavefront.com).
  */
 public class PointHandlerDispatcher implements Runnable {
-  private final static Logger logger = Logger.getLogger(PointHandlerDispatcher.class.getCanonicalName());
+  private static final Logger logger =
+      Logger.getLogger(PointHandlerDispatcher.class.getCanonicalName());
+  private static final Logger featureDisabledLogger = new MessageDedupingLogger(logger, 2, 0.2);
 
   private final Counter dispatchCounter;
   private final Counter dispatchErrorCounter;
-  private final Histogram accumulatorSize;
-  private final Histogram dispatchProcessTime;
-  private final Histogram dispatchLagMillis;
+  private final Counter dispatchProcessTime;
 
-  private final AccumulationCache digests;
-  private final PointHandler output;
+  private final Accumulator digests;
+  private final AtomicLong digestsSize = new AtomicLong(0);
+  private final ReportableEntityHandler output;
   private final TimeProvider clock;
+  private final Supplier histogramDisabled;
   private final Integer dispatchLimit;
 
-  public PointHandlerDispatcher(AccumulationCache digests, PointHandler output,
-                                @Nullable Integer dispatchLimit, @Nullable Utils.Granularity granularity) {
-    this(digests, output, System::currentTimeMillis, dispatchLimit, granularity);
-  }
-
-  @VisibleForTesting
-  PointHandlerDispatcher(AccumulationCache digests, PointHandler output, TimeProvider clock,
-                         @Nullable Integer dispatchLimit, @Nullable Utils.Granularity granularity) {
+  public PointHandlerDispatcher(
+      Accumulator digests,
+      ReportableEntityHandler output,
+      TimeProvider clock,
+      Supplier histogramDisabled,
+      @Nullable Integer dispatchLimit,
+      @Nullable Granularity granularity) {
     this.digests = digests;
     this.output = output;
     this.clock = clock;
+    this.histogramDisabled = histogramDisabled;
     this.dispatchLimit = dispatchLimit;
 
-    String metricNamespace = "histogram.accumulator." + Utils.Granularity.granularityToString(granularity);
-    this.dispatchCounter = Metrics.newCounter(new MetricName(metricNamespace, "", "dispatched"));
-    this.dispatchErrorCounter = Metrics.newCounter(new MetricName(metricNamespace, "", "dispatch_errors"));
-    this.accumulatorSize = Metrics.newHistogram(new MetricName(metricNamespace, "", "size"));
-    this.dispatchProcessTime = Metrics.newHistogram(new MetricName(metricNamespace, "", "dispatch_process_nanos"));
-    this.dispatchLagMillis = Metrics.newHistogram(new MetricName(metricNamespace, "", "dispatch_lag_millis"));
+    String prefix = "histogram.accumulator." + HistogramUtils.granularityToString(granularity);
+    this.dispatchCounter = Metrics.newCounter(new MetricName(prefix, "", "dispatched"));
+    this.dispatchErrorCounter = Metrics.newCounter(new MetricName(prefix, "", "dispatch_errors"));
+    Metrics.newGauge(
+        new MetricName(prefix, "", "size"),
+        new Gauge() {
+          @Override
+          public Long value() {
+            return digestsSize.get();
+          }
+        });
+    this.dispatchProcessTime =
+        Metrics.newCounter(new MetricName(prefix, "", "dispatch_process_millis"));
   }
 
   @Override
   public void run() {
     try {
-      accumulatorSize.update(digests.size());
       AtomicInteger dispatchedCount = new AtomicInteger(0);
 
-      long startNanos = nanoTime();
-      Iterator index = digests.getRipeDigestsIterator(this.clock);
+      long startMillis = System.currentTimeMillis();
+      digestsSize.set(digests.size()); // update size before flushing, so we show a higher value
+      Iterator index = digests.getRipeDigestsIterator(this.clock);
       while (index.hasNext()) {
-        digests.compute(index.next(), (k, v) -> {
-          if (v == null) {
-            index.remove();
-            return null;
-          }
-          try {
-            ReportPoint out = Utils.pointFromKeyAndDigest(k, v);
-            output.reportPoint(out, null);
-            dispatchCounter.inc();
-          } catch (Exception e) {
-            dispatchErrorCounter.inc();
-            logger.log(Level.SEVERE, "Failed dispatching entry " + k, e);
-          }
-          dispatchLagMillis.update(System.currentTimeMillis() - v.getDispatchTimeMillis());
-          index.remove();
-          dispatchedCount.incrementAndGet();
-          return null;
-        });
-        if (dispatchLimit != null && dispatchedCount.get() >= dispatchLimit)
-          break;
+        digests.compute(
+            index.next(),
+            (k, v) -> {
+              if (v == null) {
+                index.remove();
+                return null;
+              }
+              if (histogramDisabled.get()) {
+                featureDisabledLogger.info("Histogram feature is not enabled on the server!");
+                dispatchErrorCounter.inc();
+              } else {
+                try {
+                  ReportPoint out = HistogramUtils.pointFromKeyAndDigest(k, v);
+                  output.report(out);
+                  dispatchCounter.inc();
+                } catch (Exception e) {
+                  dispatchErrorCounter.inc();
+                  logger.log(Level.SEVERE, "Failed dispatching entry " + k, e);
+                }
+              }
+              index.remove();
+              dispatchedCount.incrementAndGet();
+              return null;
+            });
+        if (dispatchLimit != null && dispatchedCount.get() >= dispatchLimit) break;
       }
-      dispatchProcessTime.update(nanoTime() - startNanos);
+      dispatchProcessTime.inc(System.currentTimeMillis() - startMillis);
     } catch (Exception e) {
       logger.log(Level.SEVERE, "PointHandlerDispatcher error", e);
     }
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/QueuingChannelHandler.java b/proxy/src/main/java/com/wavefront/agent/histogram/QueuingChannelHandler.java
deleted file mode 100644
index a02389947..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/QueuingChannelHandler.java
+++ /dev/null
@@ -1,90 +0,0 @@
-package com.wavefront.agent.histogram;
-
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.RateLimiter;
-
-import com.squareup.tape.ObjectQueue;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.logging.Logger;
-
-import javax.validation.constraints.NotNull;
-
-import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-
-/**
- * Inbound handler streaming a netty channel out to a square tape.
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-@ChannelHandler.Sharable
-public class QueuingChannelHandler extends SimpleChannelInboundHandler {
-  protected static final Logger logger = Logger.getLogger(QueuingChannelHandler.class.getCanonicalName());
-  private final ObjectQueue> tape;
-  private List buffer;
-  private final int maxCapacity;
-  private final AtomicBoolean histogramDisabled;
-
-  // log every 5 seconds
-  private final RateLimiter warningLoggerRateLimiter = RateLimiter.create(0.2);
-
-  private final Counter discardedHistogramPointsCounter = Metrics.newCounter(new MetricName(
-      "histogram.ingester.disabled", "", "discarded_points"));
-
-  public QueuingChannelHandler(@NotNull ObjectQueue> tape, int maxCapacity, AtomicBoolean histogramDisabled) {
-    Preconditions.checkNotNull(tape);
-    Preconditions.checkArgument(maxCapacity > 0);
-    this.tape = tape;
-    this.buffer = new ArrayList<>();
-    this.maxCapacity = maxCapacity;
-    this.histogramDisabled = histogramDisabled;
-  }
-
-  private void ship() {
-    List bufferCopy = null;
-    synchronized (this) {
-      int blockSize;
-      if (!buffer.isEmpty()) {
-        blockSize = Math.min(buffer.size(), maxCapacity);
-        bufferCopy = buffer.subList(0, blockSize);
-        buffer = new ArrayList<>(buffer.subList(blockSize, buffer.size()));
-      }
-    }
-    if (bufferCopy != null && bufferCopy.size() > 0) {
-      tape.add(bufferCopy);
-    }
-  }
-
-  private void innerAdd(T t) {
-    if (histogramDisabled.get()) {
-      // if histogram feature is disabled on the server increment counter and log it every 25 times ...
-      discardedHistogramPointsCounter.inc();
-      if (warningLoggerRateLimiter.tryAcquire()) {
-        logger.info("Ingested point discarded because histogram feature is disabled on the server");
-      }
-    } else {
-      // histograms are not disabled on the server, so add the input to the buffer
-      synchronized (this) {
-        buffer.add(t);
-      }
-    }
-  }
-
-  @Override
-  protected void channelRead0(ChannelHandlerContext channelHandlerContext, Object t) throws Exception {
-    if (t != null) {
-      innerAdd((T) t);
-    }
-  }
-
-  public Runnable getBufferFlushTask() {
-    return QueuingChannelHandler.this::ship;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/TapeDispatcher.java b/proxy/src/main/java/com/wavefront/agent/histogram/TapeDispatcher.java
deleted file mode 100644
index 30224d2e4..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/TapeDispatcher.java
+++ /dev/null
@@ -1,67 +0,0 @@
-package com.wavefront.agent.histogram;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import com.squareup.tape.ObjectQueue;
-import com.tdunning.math.stats.AgentDigest;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
-
-import java.util.concurrent.ConcurrentMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Dispatch task for marshalling "ripe" digests for shipment to the agent into a (Tape) ObjectQueue
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class TapeDispatcher implements Runnable {
-  private final static Logger logger = Logger.getLogger(TapeDispatcher.class.getCanonicalName());
-
-  private final Counter dispatchCounter = Metrics.newCounter(new MetricName("histogram", "", "dispatched"));
-
-
-  private final ConcurrentMap digests;
-  private final ObjectQueue output;
-  private final TimeProvider clock;
-
-  public TapeDispatcher(ConcurrentMap digests, ObjectQueue output) {
-    this(digests, output, System::currentTimeMillis);
-  }
-
-  @VisibleForTesting
-  TapeDispatcher(ConcurrentMap digests, ObjectQueue output, TimeProvider clock) {
-    this.digests = digests;
-    this.output = output;
-    this.clock = clock;
-  }
-
-  @Override
-  public void run() {
-
-    for (Utils.HistogramKey key : digests.keySet()) {
-      digests.compute(key, (k, v) -> {
-        if (v == null) {
-          return null;
-        }
-        // Remove and add to shipping queue
-        if (v.getDispatchTimeMillis() < clock.millisSinceEpoch()) {
-          try {
-            ReportPoint out = Utils.pointFromKeyAndDigest(k, v);
-            output.add(out);
-            dispatchCounter.inc();
-
-          } catch (Exception e) {
-            logger.log(Level.SEVERE, "Failed dispatching entry " + k, e);
-          }
-          return null;
-        }
-        return v;
-      });
-    }
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/TimeProvider.java b/proxy/src/main/java/com/wavefront/agent/histogram/TimeProvider.java
deleted file mode 100644
index cc3afbcf4..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/TimeProvider.java
+++ /dev/null
@@ -1,8 +0,0 @@
-package com.wavefront.agent.histogram;
-
-/**
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public interface TimeProvider {
-  long millisSinceEpoch();
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/Utils.java b/proxy/src/main/java/com/wavefront/agent/histogram/Utils.java
deleted file mode 100644
index 21586dcaa..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/Utils.java
+++ /dev/null
@@ -1,367 +0,0 @@
-package com.wavefront.agent.histogram;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-
-import com.tdunning.math.stats.AgentDigest;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Histogram;
-import com.yammer.metrics.core.MetricName;
-
-import net.openhft.chronicle.bytes.Bytes;
-import net.openhft.chronicle.core.io.IORuntimeException;
-import net.openhft.chronicle.core.util.ReadResolvable;
-import net.openhft.chronicle.hash.serialization.BytesReader;
-import net.openhft.chronicle.hash.serialization.BytesWriter;
-import net.openhft.chronicle.wire.WireIn;
-import net.openhft.chronicle.wire.WireOut;
-
-import org.apache.commons.lang.time.DateUtils;
-
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Helpers around histograms
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public final class Utils {
-  private static final Logger logger = Logger.getLogger(Utils.class.getCanonicalName());
-
-  private Utils() {
-    // Not instantiable
-  }
-
-  /**
-   * Standard supported aggregation Granularities.
-   */
-  public enum Granularity {
-    MINUTE((int) DateUtils.MILLIS_PER_MINUTE),
-    HOUR((int) DateUtils.MILLIS_PER_HOUR),
-    DAY((int) DateUtils.MILLIS_PER_DAY);
-
-    private final int inMillis;
-
-    Granularity(int inMillis) {
-      this.inMillis = inMillis;
-    }
-
-    /**
-     * Duration of a corresponding bin in milliseconds.
-     *
-     * @return bin length in milliseconds
-     */
-    public int getInMillis() {
-      return inMillis;
-    }
-
-    /**
-     * Bin id for an epoch time is the epoch time in the corresponding granularity.
-     *
-     * @param timeMillis epoch time in milliseconds
-     * @return the bin id
-     */
-    public int getBinId(long timeMillis) {
-      return (int) (timeMillis / inMillis);
-    }
-
-    public static Granularity fromMillis(long millis) {
-      if (millis <= 60 * 1000) {
-        return MINUTE;
-      } else if (millis <= 60 * 60 * 1000) {
-        return HOUR;
-      } else {
-        return DAY;
-      }
-    }
-
-    public static Granularity fromString(String granularityName) {
-      if (granularityName.equals("minute")) {
-        return MINUTE;
-      }
-      if (granularityName.equals("hour")) {
-        return HOUR;
-      }
-      if (granularityName.equals("day")) {
-        return DAY;
-      }
-      return null;
-    }
-
-    public static String granularityToString(@Nullable Granularity granularity) {
-      if (granularity == null) {
-        return "distribution";
-      }
-      switch (granularity) {
-        case DAY:
-          return "day";
-        case HOUR:
-          return "hour";
-        case MINUTE:
-          return "minute";
-      }
-      return "unknown";
-    }
-  }
-
-  /**
-   * Generates a {@link HistogramKey} according a prototype {@link ReportPoint} and {@link Granularity}.
-   */
-  public static HistogramKey makeKey(ReportPoint point, Granularity granularity) {
-    Preconditions.checkNotNull(point);
-    Preconditions.checkNotNull(granularity);
-
-    String[] annotations = null;
-    if (point.getAnnotations() != null) {
-      List> keyOrderedTags = point.getAnnotations().entrySet()
-          .stream().sorted(Comparator.comparing(Map.Entry::getKey)).collect(Collectors.toList());
-      annotations = new String[keyOrderedTags.size() * 2];
-      for (int i = 0; i < keyOrderedTags.size(); ++i) {
-        annotations[2 * i] = keyOrderedTags.get(i).getKey();
-        annotations[(2 * i) + 1] = keyOrderedTags.get(i).getValue();
-      }
-    }
-
-    return new HistogramKey(
-        (byte) granularity.ordinal(),
-        granularity.getBinId(point.getTimestamp()),
-        point.getMetric(),
-        point.getHost(),
-        annotations
-    );
-  }
-
-  /**
-   * Creates a {@link ReportPoint} from a {@link HistogramKey} - {@link AgentDigest} pair
-   *
-   * @param histogramKey the key, defining metric, source, annotations, duration and start-time
-   * @param agentDigest  the digest defining the centroids
-   * @return the corresponding point
-   */
-  public static ReportPoint pointFromKeyAndDigest(HistogramKey histogramKey, AgentDigest agentDigest) {
-    return ReportPoint.newBuilder()
-        .setTimestamp(histogramKey.getBinTimeMillis())
-        .setMetric(histogramKey.getMetric())
-        .setHost(histogramKey.getSource())
-        .setAnnotations(histogramKey.getTagsAsMap())
-        .setTable("dummy")
-        .setValue(agentDigest.toHistogram((int) histogramKey.getBinDurationInMillis()))
-        .build();
-  }
-
-  /**
-   * Uniquely identifies a time-series - time-interval pair. These are the base sample aggregation scopes on the agent.
-   */
-  public static class HistogramKey {
-    // NOTE: fields are not final to allow object reuse
-    private byte granularityOrdinal;
-    private int binId;
-    private String metric;
-    @Nullable
-    private String source;
-    @Nullable
-    private String[] tags;
-
-
-    private HistogramKey(byte granularityOrdinal, int binId, @NotNull String metric, @Nullable String source, @Nullable String[] tags) {
-      this.granularityOrdinal = granularityOrdinal;
-      this.binId = binId;
-      this.metric = metric;
-      this.source = source;
-      this.tags = ((tags == null || tags.length == 0) ? null : tags);
-    }
-
-    private HistogramKey() {
-      // For decoding
-    }
-
-    public byte getGranularityOrdinal() {
-      return granularityOrdinal;
-    }
-
-    public int getBinId() {
-      return binId;
-    }
-
-    public String getMetric() {
-      return metric;
-    }
-
-    @Nullable
-    public String getSource() {
-      return source;
-    }
-
-    @Nullable
-    public String[] getTags() {
-      return tags;
-    }
-
-    @Override
-    public String toString() {
-      return "HistogramKey{" +
-          "granularityOrdinal=" + granularityOrdinal +
-          ", binId=" + binId +
-          ", metric='" + metric + '\'' +
-          ", source='" + source + '\'' +
-          ", tags=" + Arrays.toString(tags) +
-          '}';
-    }
-
-    @Override
-    public boolean equals(Object o) {
-      if (this == o) return true;
-      if (o == null || getClass() != o.getClass()) return false;
-
-      HistogramKey histogramKey = (HistogramKey) o;
-
-      if (granularityOrdinal != histogramKey.granularityOrdinal) return false;
-      if (binId != histogramKey.binId) return false;
-      if (!metric.equals(histogramKey.metric)) return false;
-      if (source != null ? !source.equals(histogramKey.source) : histogramKey.source != null) return false;
-      return Arrays.equals(tags, histogramKey.tags);
-
-    }
-
-    @Override
-    public int hashCode() {
-      int result = 1;
-      result = 31 * result + (int) granularityOrdinal;
-      result = 31 * result + binId;
-      result = 31 * result + metric.hashCode();
-      result = 31 * result + (source != null ? source.hashCode() : 0);
-      result = 31 * result + Arrays.hashCode(tags);
-      return result;
-    }
-
-    /**
-     * Unpacks tags into a map.
-     */
-    public Map getTagsAsMap() {
-      if (tags == null || tags.length == 0) {
-        return ImmutableMap.of();
-      }
-
-      Map annotations = new HashMap<>(tags.length / 2);
-      for (int i = 0; i < tags.length - 1; i += 2) {
-        annotations.put(tags[i], tags[i + 1]);
-      }
-
-      return annotations;
-    }
-
-    public long getBinTimeMillis() {
-      return getBinDurationInMillis() * binId;
-    }
-
-    public long getBinDurationInMillis() {
-      return Granularity.values()[granularityOrdinal].getInMillis();
-    }
-  }
-
-  /**
-   * (For now, a rather trivial) encoding of {@link HistogramKey} the form short length and bytes
-   *
-   * Consider using chronicle-values or making this stateful with a local byte[]  / Stringbuffers to be a little more
-   * efficient about encodings.
-   */
-  public static class HistogramKeyMarshaller implements BytesReader, BytesWriter, ReadResolvable {
-    private static final HistogramKeyMarshaller INSTANCE = new HistogramKeyMarshaller();
-
-    private static final Histogram accumulatorKeySizes =
-        Metrics.newHistogram(new MetricName("histogram", "", "accumulatorKeySize"));
-
-    private HistogramKeyMarshaller() {
-      // Private Singleton
-    }
-
-    public static HistogramKeyMarshaller get() {
-      return INSTANCE;
-    }
-
-    @Override
-    public HistogramKeyMarshaller readResolve() {
-      return INSTANCE;
-    }
-
-    private static void writeString(Bytes out, String s) {
-      try {
-        Preconditions.checkArgument(s == null || s.length() <= Short.MAX_VALUE, "String too long (more than 32K)");
-        byte[] bytes = s == null ? new byte[0] : s.getBytes("UTF-8");
-        out.writeShort((short) bytes.length);
-        out.write(bytes);
-      } catch (UnsupportedEncodingException e) {
-        logger.log(Level.SEVERE, "Likely programmer error, String to Byte encoding failed: ", e);
-        e.printStackTrace();
-      }
-    }
-
-    private static String readString(Bytes in) {
-      byte[] bytes = new byte[in.readShort()];
-      in.read(bytes);
-      return new String(bytes);
-    }
-
-    @Override
-    public void readMarshallable(@NotNull WireIn wire) throws IORuntimeException {
-      // ignore, stateless
-    }
-
-    @Override
-    public void writeMarshallable(@NotNull WireOut wire) {
-      // ignore, stateless
-    }
-
-    @NotNull
-    @Override
-    public HistogramKey read(Bytes in, @Nullable HistogramKey using) {
-      if (using == null) {
-        using = new HistogramKey();
-      }
-      using.granularityOrdinal = in.readByte();
-      using.binId = in.readInt();
-      using.metric = readString(in);
-      using.source = readString(in);
-      int numTags = in.readShort();
-      if (numTags > 0) {
-        using.tags = new String[numTags];
-        for (int i = 0; i < numTags; ++i) {
-          using.tags[i] = readString(in);
-        }
-      }
-      return using;
-    }
-
-    @Override
-    public void write(Bytes out, @NotNull HistogramKey toWrite) {
-      int accumulatorKeySize = 5;
-      out.writeByte(toWrite.granularityOrdinal);
-      out.writeInt(toWrite.binId);
-      accumulatorKeySize += 2 + toWrite.metric.length();
-      writeString(out, toWrite.metric);
-      accumulatorKeySize += 2 + (toWrite.source == null ? 0 : toWrite.source.length());
-      writeString(out, toWrite.source);
-      short numTags = toWrite.tags == null ? 0 : (short) toWrite.tags.length;
-      accumulatorKeySize += 2;
-      out.writeShort(numTags);
-      for (short i = 0; i < numTags; ++i) {
-        accumulatorKeySize += 2 + (toWrite.tags[i] == null ? 0 : toWrite.tags[i].length());
-        writeString(out, toWrite.tags[i]);
-      }
-      accumulatorKeySizes.update(accumulatorKeySize);
-    }
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/AccumulationCache.java b/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/AccumulationCache.java
index d1c9b5db7..12f107dce 100644
--- a/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/AccumulationCache.java
+++ b/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/AccumulationCache.java
@@ -1,82 +1,89 @@
 package com.wavefront.agent.histogram.accumulator;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.RateLimiter;
+import static com.wavefront.agent.histogram.HistogramUtils.mergeHistogram;
 
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.CacheWriter;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.RemovalCause;
 import com.github.benmanes.caffeine.cache.Ticker;
+import com.google.common.annotations.VisibleForTesting;
 import com.tdunning.math.stats.AgentDigest;
-import com.tdunning.math.stats.TDigest;
-import com.wavefront.agent.histogram.TimeProvider;
-import com.wavefront.agent.histogram.Utils;
+import com.wavefront.agent.SharedMetricsRegistry;
+import com.wavefront.agent.histogram.HistogramKey;
+import com.wavefront.common.TimeProvider;
+import com.wavefront.common.logger.SharedRateLimitingLogger;
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Counter;
 import com.yammer.metrics.core.MetricName;
-
+import com.yammer.metrics.core.MetricsRegistry;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.BiFunction;
 import java.util.logging.Logger;
-
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-
 import wavefront.report.Histogram;
 
-import static com.wavefront.agent.histogram.Utils.HistogramKey;
-
 /**
  * Expose a local cache of limited size along with a task to flush that cache to the backing store.
  *
  * @author Tim Schmidt (tim@wavefront.com).
  */
-public class AccumulationCache {
-  private final static Logger logger = Logger.getLogger(AccumulationCache.class.getCanonicalName());
-
-  private final Counter binCreatedCounter = Metrics.newCounter(
-      new MetricName("histogram.accumulator", "", "bin_created"));
-  private final Counter flushedCounter = Metrics.newCounter(
-      new MetricName("histogram.accumulator.cache", "", "flushed"));
-  private final Counter cacheOverflowCounter = Metrics.newCounter(
-      new MetricName("histogram.accumulator.cache", "", "size_exceeded"));
+public class AccumulationCache implements Accumulator {
+  private static final Logger logger = Logger.getLogger(AccumulationCache.class.getCanonicalName());
+  private static final MetricsRegistry sharedRegistry = SharedMetricsRegistry.getInstance();
+
+  private final Counter binCreatedCounter;
+  private final Counter binMergedCounter;
+  private final Counter cacheBinCreatedCounter;
+  private final Counter cacheBinMergedCounter;
+  private final Counter flushedCounter;
+  private final Counter cacheOverflowCounter =
+      Metrics.newCounter(new MetricName("histogram.accumulator.cache", "", "size_exceeded"));
+  private final boolean cacheEnabled;
   private final Cache cache;
   private final ConcurrentMap backingStore;
-
+  private final AgentDigestFactory agentDigestFactory;
 
   /**
-   * In-memory index for dispatch timestamps to avoid iterating the backing store map, which is an expensive
-   * operation, as it requires value objects to be de-serialized first
+   * In-memory index for dispatch timestamps to avoid iterating the backing store map, which is an
+   * expensive operation, as it requires value objects to be de-serialized first.
    */
   private final ConcurrentMap keyIndex;
 
   /**
-   * Constructs a new AccumulationCache instance around {@code backingStore} and builds an in-memory index maintaining
-   * dispatch times in milliseconds for all HistogramKeys in backingStore
-   * Setting cacheSize to 0 disables in-memory caching so the cache only maintains the dispatch time index.
+   * Constructs a new AccumulationCache instance around {@code backingStore} and builds an in-memory
+   * index maintaining dispatch times in milliseconds for all HistogramKeys in the backingStore.
+   * Setting cacheSize to 0 disables in-memory caching so the cache only maintains the dispatch time
+   * index.
    *
-   * @param backingStore a {@code ConcurrentMap} storing AgentDigests
+   * @param backingStore a {@code ConcurrentMap} storing {@code AgentDigests}
+   * @param agentDigestFactory a factory that generates {@code AgentDigests} with pre-defined
+   *     compression level and TTL time
    * @param cacheSize maximum size of the cache
    * @param ticker a nanosecond-precision time source
    */
   public AccumulationCache(
       final ConcurrentMap backingStore,
+      final AgentDigestFactory agentDigestFactory,
       final long cacheSize,
+      String metricPrefix,
       @Nullable Ticker ticker) {
-    this(backingStore, cacheSize, ticker, null);
+    this(backingStore, agentDigestFactory, cacheSize, metricPrefix, ticker, null);
   }
 
   /**
-   * Constructs a new AccumulationCache instance around {@code backingStore} and builds an in-memory index maintaining
-   * dispatch times in milliseconds for all HistogramKeys in backingStore
-   * Setting cacheSize to 0 disables in-memory caching so the cache only maintains the dispatch time index.
+   * Constructs a new AccumulationCache instance around {@code backingStore} and builds an in-memory
+   * index maintaining dispatch times in milliseconds for all HistogramKeys in the backingStore.
+   * Setting cacheSize to 0 disables in-memory caching, so the cache only maintains the dispatch
+   * time index.
    *
-   * @param backingStore a {@code ConcurrentMap} storing AgentDigests
+   * @param backingStore a {@code ConcurrentMap} storing {@code AgentDigests}
+   * @param agentDigestFactory a factory that generates {@code AgentDigests} with pre-defined
+   *     compression level and TTL time
    * @param cacheSize maximum size of the cache
    * @param ticker a nanosecond-precision time source
    * @param onFailure a {@code Runnable} that is invoked when backing store overflows
@@ -84,10 +91,23 @@ public AccumulationCache(
   @VisibleForTesting
   protected AccumulationCache(
       final ConcurrentMap backingStore,
+      final AgentDigestFactory agentDigestFactory,
       final long cacheSize,
+      String metricPrefix,
       @Nullable Ticker ticker,
       @Nullable Runnable onFailure) {
     this.backingStore = backingStore;
+    this.agentDigestFactory = agentDigestFactory;
+    this.cacheEnabled = cacheSize > 0;
+    this.binCreatedCounter = Metrics.newCounter(new MetricName(metricPrefix, "", "bin_created"));
+    this.binMergedCounter = Metrics.newCounter(new MetricName(metricPrefix, "", "bin_merged"));
+    MetricsRegistry metricsRegistry = cacheEnabled ? Metrics.defaultRegistry() : sharedRegistry;
+    this.cacheBinCreatedCounter =
+        metricsRegistry.newCounter(new MetricName(metricPrefix + ".cache", "", "bin_created"));
+    this.cacheBinMergedCounter =
+        metricsRegistry.newCounter(new MetricName(metricPrefix + ".cache", "", "bin_merged"));
+    this.flushedCounter =
+        Metrics.newCounter(new MetricName(metricPrefix + ".cache", "", "flushed"));
     this.keyIndex = new ConcurrentHashMap<>(backingStore.size());
     final Runnable failureHandler = onFailure == null ? new AccumulationCacheMonitor() : onFailure;
     if (backingStore.size() > 0) {
@@ -97,47 +117,58 @@ protected AccumulationCache(
       }
       logger.info("Finished: Indexing histogram accumulator");
     }
-    this.cache = Caffeine.newBuilder()
-        .maximumSize(cacheSize)
-        .ticker(ticker == null ? Ticker.systemTicker() : ticker)
-        .writer(new CacheWriter() {
-          @Override
-          public void write(@Nonnull HistogramKey key, @Nonnull AgentDigest value) {
-            // ignored
-          }
+    this.cache =
+        Caffeine.newBuilder()
+            .maximumSize(cacheSize)
+            .ticker(ticker == null ? Ticker.systemTicker() : ticker)
+            .writer(
+                new CacheWriter() {
+                  @Override
+                  public void write(@Nonnull HistogramKey key, @Nonnull AgentDigest value) {
+                    // ignored
+                  }
 
-          @Override
-          public void delete(@Nonnull HistogramKey key, @Nullable AgentDigest value, @Nonnull RemovalCause cause) {
-            if (value == null) {
-              return;
-            }
-            flushedCounter.inc();
-            if (cause == RemovalCause.SIZE) cacheOverflowCounter.inc();
-            try {
-              // flush out to backing store
-              backingStore.merge(key, value, (digestA, digestB) -> {
-                if (digestA != null && digestB != null) {
-                  // Merge both digests
-                  if (digestA.centroidCount() >= digestB.centroidCount()) {
-                    digestA.add(digestB);
-                    return digestA;
-                  } else {
-                    digestB.add(digestA);
-                    return digestB;
+                  @Override
+                  public void delete(
+                      @Nonnull HistogramKey key,
+                      @Nullable AgentDigest value,
+                      @Nonnull RemovalCause cause) {
+                    if (value == null) {
+                      return;
+                    }
+                    flushedCounter.inc();
+                    if (cause == RemovalCause.SIZE && cacheEnabled) cacheOverflowCounter.inc();
+                    try {
+                      // flush out to backing store
+                      AgentDigest merged =
+                          backingStore.merge(
+                              key,
+                              value,
+                              (digestA, digestB) -> {
+                                // Merge both digests
+                                if (digestA.centroidCount() >= digestB.centroidCount()) {
+                                  digestA.add(digestB);
+                                  return digestA;
+                                } else {
+                                  digestB.add(digestA);
+                                  return digestB;
+                                }
+                              });
+                      if (merged == value) {
+                        binCreatedCounter.inc();
+                      } else {
+                        binMergedCounter.inc();
+                      }
+                    } catch (IllegalStateException e) {
+                      if (e.getMessage().contains("Attempt to allocate")) {
+                        failureHandler.run();
+                      } else {
+                        throw e;
+                      }
+                    }
                   }
-                } else {
-                  return (digestB == null ? digestA : digestB);
-                }
-              });
-            } catch (IllegalStateException e) {
-              if (e.getMessage().contains("Attempt to allocate")) {
-                failureHandler.run();
-              } else {
-                throw e;
-              }
-            }
-          }
-        }).build();
+                })
+            .build();
   }
 
   @VisibleForTesting
@@ -151,92 +182,128 @@ Cache getCache() {
    * @param key histogram key
    * @param value {@code AgentDigest} to be merged
    */
+  @Override
   public void put(HistogramKey key, @Nonnull AgentDigest value) {
-    cache.asMap().compute(key, (k, v) -> {
-      if (v == null) {
-        keyIndex.put(key, value.getDispatchTimeMillis());
-        return value;
-      } else {
-        keyIndex.compute(key, (k1, v1) -> (
-            v1 != null && v1 < v.getDispatchTimeMillis() ? v1 : v.getDispatchTimeMillis()));
-        v.add(value);
-        return v;
-      }
-    });
+    cache
+        .asMap()
+        .compute(
+            key,
+            (k, v) -> {
+              if (v == null) {
+                if (cacheEnabled) cacheBinCreatedCounter.inc();
+                keyIndex.put(key, value.getDispatchTimeMillis());
+                return value;
+              } else {
+                if (cacheEnabled) cacheBinMergedCounter.inc();
+                keyIndex.compute(
+                    key,
+                    (k1, v1) ->
+                        (v1 != null && v1 < v.getDispatchTimeMillis()
+                            ? v1
+                            : v.getDispatchTimeMillis()));
+                v.add(value);
+                return v;
+              }
+            });
   }
 
   /**
-   * Update {@code AgentDigest} in the cache with a double value. If such {@code AgentDigest} does not exist for
-   * the specified key, it will be created with the specified compression and ttlMillis settings.
+   * Update {@link AgentDigest} in the cache with a double value. If such {@code AgentDigest} does
+   * not exist for the specified key, it will be created using {@link AgentDigestFactory}
    *
    * @param key histogram key
    * @param value value to be merged into the {@code AgentDigest}
-   * @param compression default compression level for new bins
-   * @param ttlMillis default time-to-dispatch for new bins
    */
-  public void put(HistogramKey key, double value, short compression, long ttlMillis) {
-    cache.asMap().compute(key, (k, v) -> {
-      if (v == null) {
-        binCreatedCounter.inc();
-        AgentDigest t = new AgentDigest(compression, System.currentTimeMillis() + ttlMillis);
-        keyIndex.compute(key, (k1, v1) -> (
-            v1 != null && v1 < t.getDispatchTimeMillis() ? v1 : t.getDispatchTimeMillis()
-        ));
-        t.add(value);
-        return t;
-      } else {
-        keyIndex.compute(key, (k1, v1) -> (
-            v1 != null && v1 < v.getDispatchTimeMillis() ? v1 : v.getDispatchTimeMillis()
-        ));
-        v.add(value);
-        return v;
-      }
-    });
+  @Override
+  public void put(HistogramKey key, double value) {
+    cache
+        .asMap()
+        .compute(
+            key,
+            (k, v) -> {
+              if (v == null) {
+                if (cacheEnabled) cacheBinCreatedCounter.inc();
+                AgentDigest t = agentDigestFactory.newDigest();
+                keyIndex.compute(
+                    key,
+                    (k1, v1) ->
+                        (v1 != null && v1 < t.getDispatchTimeMillis()
+                            ? v1
+                            : t.getDispatchTimeMillis()));
+                t.add(value);
+                return t;
+              } else {
+                if (cacheEnabled) cacheBinMergedCounter.inc();
+                keyIndex.compute(
+                    key,
+                    (k1, v1) ->
+                        (v1 != null && v1 < v.getDispatchTimeMillis()
+                            ? v1
+                            : v.getDispatchTimeMillis()));
+                v.add(value);
+                return v;
+              }
+            });
   }
 
   /**
-   * Update {@code AgentDigest} in the cache with a {@code Histogram} value. If such {@code AgentDigest} does not exist
-   * for the specified key, it will be created with the specified compression and ttlMillis settings.
+   * Update {@link AgentDigest} in the cache with a {@code Histogram} value. If such {@code
+   * AgentDigest} does not exist for the specified key, it will be created using {@link
+   * AgentDigestFactory}.
    *
    * @param key histogram key
    * @param value a {@code Histogram} to be merged into the {@code AgentDigest}
-   * @param compression default compression level for new bins
-   * @param ttlMillis default time-to-dispatch in milliseconds for new bins
    */
-  public void put(HistogramKey key, Histogram value, short compression, long ttlMillis) {
-    cache.asMap().compute(key, (k, v) -> {
-      if (v == null) {
-        binCreatedCounter.inc();
-        AgentDigest t = new AgentDigest(compression, System.currentTimeMillis() + ttlMillis);
-        keyIndex.compute(key, (k1, v1) -> (
-            v1 != null && v1 < t.getDispatchTimeMillis() ? v1 : t.getDispatchTimeMillis()));
-        mergeHistogram(t, value);
-        return t;
-      } else {
-        keyIndex.compute(key, (k1, v1) -> (
-            v1 != null && v1 < v.getDispatchTimeMillis() ? v1 : v.getDispatchTimeMillis()));
-        mergeHistogram(v, value);
-        return v;
-      }
-    });
+  @Override
+  public void put(HistogramKey key, Histogram value) {
+    cache
+        .asMap()
+        .compute(
+            key,
+            (k, v) -> {
+              if (v == null) {
+                if (cacheEnabled) cacheBinCreatedCounter.inc();
+                AgentDigest t = agentDigestFactory.newDigest();
+                keyIndex.compute(
+                    key,
+                    (k1, v1) ->
+                        (v1 != null && v1 < t.getDispatchTimeMillis()
+                            ? v1
+                            : t.getDispatchTimeMillis()));
+                mergeHistogram(t, value);
+                return t;
+              } else {
+                if (cacheEnabled) cacheBinMergedCounter.inc();
+                keyIndex.compute(
+                    key,
+                    (k1, v1) ->
+                        (v1 != null && v1 < v.getDispatchTimeMillis()
+                            ? v1
+                            : v.getDispatchTimeMillis()));
+                mergeHistogram(v, value);
+                return v;
+              }
+            });
   }
 
   /**
    * Returns an iterator over "ripe" digests ready to be shipped
-
+   *
    * @param clock a millisecond-precision epoch time source
    * @return an iterator over "ripe" digests ready to be shipped
    */
+  @Override
   public Iterator getRipeDigestsIterator(TimeProvider clock) {
     return new Iterator() {
-      private final Iterator> indexIterator = keyIndex.entrySet().iterator();
+      private final Iterator> indexIterator =
+          keyIndex.entrySet().iterator();
       private HistogramKey nextHistogramKey;
 
       @Override
       public boolean hasNext() {
         while (indexIterator.hasNext()) {
-          Map.Entry entry = indexIterator.next();
-          if (entry.getValue() < clock.millisSinceEpoch()) {
+          Map.Entry entry = indexIterator.next();
+          if (entry.getValue() < clock.currentTimeMillis()) {
             nextHistogramKey = entry.getKey();
             return true;
           }
@@ -257,15 +324,18 @@ public void remove() {
   }
 
   /**
-   * Attempts to compute a mapping for the specified key and its current mapped value
-   * (or null if there is no current mapping).
+   * Attempts to compute a mapping for the specified key and its current mapped value (or null if
+   * there is no current mapping).
    *
-   * @param key               key with which the specified value is to be associated
+   * @param key key with which the specified value is to be associated
    * @param remappingFunction the function to compute a value
-   * @return                  the new value associated with the specified key, or null if none
+   * @return the new value associated with the specified key, or null if none
    */
-  public AgentDigest compute(HistogramKey key, BiFunction remappingFunction) {
+  @Override
+  public AgentDigest compute(
+      HistogramKey key,
+      BiFunction
+          remappingFunction) {
     return backingStore.compute(key, remappingFunction);
   }
 
@@ -274,41 +344,21 @@ public AgentDigest compute(HistogramKey key, BiFunction means = source.getBins();
-    List counts = source.getCounts();
-
-    if (means != null && counts != null) {
-      int len = Math.min(means.size(), counts.size());
-
-      for (int i = 0; i < len; ++i) {
-        Integer count = counts.get(i);
-        Double mean = means.get(i);
-
-        if (count != null && count > 0 && mean != null && Double.isFinite(mean)) {
-          target.add(mean, count);
-        }
-      }
-    }
-  }
-
-  /**
-   * Task to merge the contents of this cache with the corresponding backing store.
-   *
-   * @return the task
-   */
-  public Runnable getResolveTask() {
-    return cache::invalidateAll;
+  /** Merge the contents of this cache with the corresponding backing store. */
+  @Override
+  public void flush() {
+    cache.invalidateAll();
   }
 
-  public class AccumulationCacheMonitor implements Runnable {
-
+  private static class AccumulationCacheMonitor implements Runnable {
+    private final Logger throttledLogger =
+        new SharedRateLimitingLogger(logger, "accumulator-failure", 1.0d);
     private Counter failureCounter;
-    private final RateLimiter failureMessageLimiter = RateLimiter.create(1);
 
     @Override
     public void run() {
@@ -316,11 +366,11 @@ public void run() {
         failureCounter = Metrics.newCounter(new MetricName("histogram.accumulator", "", "failure"));
       }
       failureCounter.inc();
-      if (failureMessageLimiter.tryAcquire()) {
-        logger.severe("CRITICAL: Histogram accumulator overflow - losing histogram data!!! " +
-            "Accumulator size configuration setting is not appropriate for workload, please increase " +
-            "the value as appropriate and restart the proxy!");
-      }
+      throttledLogger.severe(
+          "CRITICAL: Histogram accumulator overflow - "
+              + "losing histogram data!!! Accumulator size configuration setting is "
+              + "not appropriate for the current workload, please increase the value "
+              + "as appropriate and restart the proxy!");
     }
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/AccumulationTask.java b/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/AccumulationTask.java
deleted file mode 100644
index 2ed353b94..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/AccumulationTask.java
+++ /dev/null
@@ -1,177 +0,0 @@
-package com.wavefront.agent.histogram.accumulator;
-
-import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-
-import com.squareup.tape.ObjectQueue;
-import com.tdunning.math.stats.AgentDigest;
-import com.wavefront.agent.PointHandler;
-import com.wavefront.data.Validation;
-import com.wavefront.agent.histogram.Utils;
-import com.wavefront.ingester.Decoder;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-
-import wavefront.report.Histogram;
-import wavefront.report.ReportPoint;
-
-import static com.wavefront.agent.histogram.Utils.Granularity.fromMillis;
-import static java.lang.System.nanoTime;
-
-/**
- * Histogram accumulation task. Parses {@link ReportPoint} based on the passed in {@link Decoder} from its input queue
- * and accumulates them in {@link AgentDigest}.
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class AccumulationTask implements Runnable {
-  private static final Logger logger = Logger.getLogger(AccumulationTask.class.getCanonicalName());
-
-  private final ObjectQueue> input;
-  private final AccumulationCache digests;
-  private final Decoder decoder;
-  private final List points = Lists.newArrayListWithExpectedSize(1);
-  private final PointHandler blockedPointsHandler;
-  private final Validation.Level validationLevel;
-  private final long ttlMillis;
-  private final Utils.Granularity granularity;
-  private final short compression;
-
-  // Metrics
-  private final Counter eventCounter;
-  private final Counter histogramCounter;
-  private final Counter ignoredCounter;
-  private final com.yammer.metrics.core.Histogram batchProcessTime;
-  private final com.yammer.metrics.core.Histogram histogramBinCount;
-  private final com.yammer.metrics.core.Histogram histogramSampleCount;
-
-
-  public AccumulationTask(ObjectQueue> input,
-                          AccumulationCache digests,
-                          Decoder decoder,
-                          PointHandler blockedPointsHandler,
-                          Validation.Level validationLevel,
-                          long ttlMillis,
-                          @Nullable Utils.Granularity granularity,
-                          short compression) {
-    this.input = input;
-    this.digests = digests;
-    this.decoder = decoder;
-    this.blockedPointsHandler = blockedPointsHandler;
-    this.validationLevel = validationLevel;
-    this.ttlMillis = ttlMillis;
-    this.granularity = granularity == null ? Utils.Granularity.DAY : granularity;
-    this.compression = compression;
-
-    String metricNamespace = "histogram.accumulator." + Utils.Granularity.granularityToString(granularity);
-    eventCounter = Metrics.newCounter(new MetricName(metricNamespace, "", "sample_added"));
-    histogramCounter = Metrics.newCounter(new MetricName(metricNamespace, "", "histogram_added"));
-    ignoredCounter = Metrics.newCounter(new MetricName(metricNamespace, "", "ignored"));
-    batchProcessTime = Metrics.newHistogram(new MetricName(metricNamespace, "", "batch_process_nanos"));
-    histogramBinCount = Metrics.newHistogram(new MetricName(metricNamespace, "", "histogram_bins"));
-    histogramSampleCount = Metrics.newHistogram(new MetricName(metricNamespace, "", "histogram_samples"));
-  }
-
-  @Override
-  public void run() {
-    while (input.size() > 0 && !Thread.currentThread().isInterrupted()) {
-      List lines = input.peek();
-      if (lines == null) { // remove corrupt data
-        input.remove();
-        continue;
-      }
-
-      long startNanos = nanoTime();
-      for (String line : lines) {
-        try {
-          // Ignore empty lines
-          if ((line = line.trim()).isEmpty()) {
-            continue;
-          }
-
-          // Parse line
-          points.clear();
-          try {
-            decoder.decodeReportPoints(line, points, "c");
-          } catch (Exception e) {
-            final Throwable cause = Throwables.getRootCause(e);
-            String errMsg = "WF-300 Cannot parse: \"" + line + "\", reason: \"" + e.getMessage() + "\"";
-            if (cause != null && cause.getMessage() != null) {
-              errMsg = errMsg + ", root cause: \"" + cause.getMessage() + "\"";
-            }
-            throw new IllegalArgumentException(errMsg);
-          }
-
-          // now have the point, continue like in PointHandlerImpl
-          ReportPoint event = points.get(0);
-
-          Validation.validatePoint(
-              event,
-              line,
-              validationLevel);
-
-          if (event.getValue() instanceof Double) {
-            // Get key
-            Utils.HistogramKey histogramKey = Utils.makeKey(event, granularity);
-            double value = (Double) event.getValue();
-            eventCounter.inc();
-
-            // atomic update
-            digests.put(histogramKey, value, compression, ttlMillis);
-          } else if (event.getValue() instanceof Histogram) {
-            Histogram value = (Histogram) event.getValue();
-            Utils.Granularity granularity = fromMillis(value.getDuration());
-
-            histogramBinCount.update(value.getCounts().size());
-            histogramSampleCount.update(value.getCounts().stream().mapToLong(x->x).sum());
-
-            // Key
-            Utils.HistogramKey histogramKey = Utils.makeKey(event, granularity);
-            histogramCounter.inc();
-
-            // atomic update
-            digests.put(histogramKey, value, compression, ttlMillis);
-          }
-        } catch (Exception e) {
-          if (!(e instanceof IllegalArgumentException)) {
-            logger.log(Level.SEVERE, "Unexpected error while parsing/accumulating sample: " + e.getMessage(), e);
-          }
-          ignoredCounter.inc();
-          if (StringUtils.isNotEmpty(e.getMessage())) {
-            blockedPointsHandler.handleBlockedPoint(e.getMessage());
-          }
-        }
-      } // end point processing
-      input.remove();
-      batchProcessTime.update(nanoTime() - startNanos);
-    } // end batch processing
-  }
-
-  @Override
-  public String toString() {
-    return "AccumulationTask{" +
-        "input=" + input +
-        ", digests=" + digests +
-        ", decoder=" + decoder +
-        ", points=" + points +
-        ", blockedPointsHandler=" + blockedPointsHandler +
-        ", validationLevel=" + validationLevel +
-        ", ttlMillis=" + ttlMillis +
-        ", granularity=" + granularity +
-        ", compression=" + compression +
-        ", accumulationCounter=" + eventCounter +
-        ", ignoredCounter=" + ignoredCounter +
-        '}';
-  }
-}
-
-
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/Accumulator.java b/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/Accumulator.java
new file mode 100644
index 000000000..9c4394d35
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/Accumulator.java
@@ -0,0 +1,75 @@
+package com.wavefront.agent.histogram.accumulator;
+
+import com.tdunning.math.stats.AgentDigest;
+import com.wavefront.agent.histogram.HistogramKey;
+import com.wavefront.common.TimeProvider;
+import java.util.Iterator;
+import java.util.function.BiFunction;
+import javax.annotation.Nonnull;
+import wavefront.report.Histogram;
+
+/**
+ * Caching wrapper around the backing store.
+ *
+ * @author vasily@wavefront.com
+ */
+public interface Accumulator {
+
+  /**
+   * Update {@code AgentDigest} in the cache with another {@code AgentDigest}.
+   *
+   * @param key histogram key
+   * @param value {@code AgentDigest} to be merged
+   */
+  void put(HistogramKey key, @Nonnull AgentDigest value);
+
+  /**
+   * Update {@link AgentDigest} in the cache with a double value. If such {@code AgentDigest} does
+   * not exist for the specified key, it will be created using {@link AgentDigestFactory}
+   *
+   * @param key histogram key
+   * @param value value to be merged into the {@code AgentDigest}
+   */
+  void put(HistogramKey key, double value);
+
+  /**
+   * Update {@link AgentDigest} in the cache with a {@code Histogram} value. If such {@code
+   * AgentDigest} does not exist for the specified key, it will be created using {@link
+   * AgentDigestFactory}.
+   *
+   * @param key histogram key
+   * @param value a {@code Histogram} to be merged into the {@code AgentDigest}
+   */
+  void put(HistogramKey key, Histogram value);
+
+  /**
+   * Attempts to compute a mapping for the specified key and its current mapped value (or null if
+   * there is no current mapping).
+   *
+   * @param key key with which the specified value is to be associated
+   * @param remappingFunction the function to compute a value
+   * @return the new value associated with the specified key, or null if none
+   */
+  AgentDigest compute(
+      HistogramKey key,
+      BiFunction
+          remappingFunction);
+
+  /**
+   * Returns an iterator over "ripe" digests ready to be shipped
+   *
+   * @param clock a millisecond-precision epoch time source
+   * @return an iterator over "ripe" digests ready to be shipped
+   */
+  Iterator getRipeDigestsIterator(TimeProvider clock);
+
+  /**
+   * Returns the number of items in the storage behind the cache
+   *
+   * @return number of items
+   */
+  long size();
+
+  /** Merge the contents of this cache with the corresponding backing store. */
+  void flush();
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/AgentDigestFactory.java b/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/AgentDigestFactory.java
new file mode 100644
index 000000000..ad33115a9
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/AgentDigestFactory.java
@@ -0,0 +1,33 @@
+package com.wavefront.agent.histogram.accumulator;
+
+import com.tdunning.math.stats.AgentDigest;
+import com.wavefront.common.TimeProvider;
+import java.util.function.Supplier;
+
+/**
+ * A simple factory for creating {@link AgentDigest} objects with a specific compression level and
+ * expiration TTL.
+ *
+ * @author vasily@wavefront.com
+ */
+public class AgentDigestFactory {
+  private final Supplier compressionSupplier;
+  private final long ttlMillis;
+  private final TimeProvider timeProvider;
+
+  /**
+   * @param compressionSupplier supplier for compression level setting.
+   * @param ttlMillis default ttlMillis for new digests.
+   * @param timeProvider time provider (in millis).
+   */
+  public AgentDigestFactory(
+      Supplier compressionSupplier, long ttlMillis, TimeProvider timeProvider) {
+    this.compressionSupplier = compressionSupplier;
+    this.ttlMillis = ttlMillis;
+    this.timeProvider = timeProvider;
+  }
+
+  public AgentDigest newDigest() {
+    return new AgentDigest(compressionSupplier.get(), timeProvider.currentTimeMillis() + ttlMillis);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/Layering.java b/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/Layering.java
deleted file mode 100644
index 555f1d37e..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/accumulator/Layering.java
+++ /dev/null
@@ -1,102 +0,0 @@
-package com.wavefront.agent.histogram.accumulator;
-
-import com.github.benmanes.caffeine.cache.CacheLoader;
-import com.github.benmanes.caffeine.cache.CacheWriter;
-import com.github.benmanes.caffeine.cache.RemovalCause;
-import com.tdunning.math.stats.AgentDigest;
-
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-import static com.wavefront.agent.histogram.Utils.HistogramKey;
-
-/**
- * Basic layering between Caffeine and some backing store. Dirty/Deleted entries are cached locally. Write-backs can be
- * scheduled via the corresponding writeBackTask. It exposes a KeySetAccessor for traversing the backing store's
- * keyspace.
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class Layering implements CacheWriter, CacheLoader {
-  private static final AgentDigest DELETED = new AgentDigest((short)20, 0L);
-  private final ConcurrentMap backingStore;
-  private final ConcurrentMap dirtyEntries;
-
-
-  public Layering(ConcurrentMap backingStore) {
-    this.backingStore = backingStore;
-    this.dirtyEntries = new ConcurrentHashMap<>();
-  }
-
-  @Override
-  public void write(@Nonnull HistogramKey histogramKey,
-                    @Nonnull AgentDigest agentDigest) {
-    dirtyEntries.put(histogramKey, agentDigest);
-  }
-
-  @Override
-  public void delete(@Nonnull HistogramKey histogramKey,
-                     @Nullable AgentDigest agentDigest,
-                     @Nonnull RemovalCause removalCause) {
-    // Only write through on explicit deletes (do exchanges have the
-    switch (removalCause) {
-      case EXPLICIT:
-        dirtyEntries.put(histogramKey, DELETED);
-        break;
-      default:
-    }
-  }
-
-  @Override
-  public AgentDigest load(@Nonnull HistogramKey key) throws Exception {
-    AgentDigest value = dirtyEntries.get(key);
-    if (value == null) {
-      value = backingStore.get(key);
-    } else if (value == DELETED) {
-      value = null;
-    }
-    return value;
-  }
-
-  /**
-   * Returns a runnable for writing back dirty entries to the backing store.
-   */
-  public Runnable getWriteBackTask() {
-    return new WriteBackTask();
-  }
-
-  private class WriteBackTask implements Runnable {
-
-    @Override
-    public void run() {
-      for (HistogramKey dirtyKey : dirtyEntries.keySet()) {
-        AgentDigest dirtyValue = dirtyEntries.remove(dirtyKey);
-        if (dirtyValue != null) {
-          if (dirtyValue == DELETED) {
-            backingStore.remove(dirtyKey);
-          } else {
-            backingStore.put(dirtyKey, dirtyValue);
-
-          }
-        }
-      }
-    }
-  }
-
-  public interface KeySetAccessor {
-    /**
-     * Keys in the combined set.
-     *
-     * @return
-     */
-    Set keySet();
-  }
-
-  public KeySetAccessor getKeySetAccessor() {
-    return backingStore::keySet;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/tape/TapeDeck.java b/proxy/src/main/java/com/wavefront/agent/histogram/tape/TapeDeck.java
deleted file mode 100644
index c39526ee6..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/tape/TapeDeck.java
+++ /dev/null
@@ -1,174 +0,0 @@
-package com.wavefront.agent.histogram.tape;
-
-import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
-import com.squareup.tape.FileObjectQueue;
-import com.squareup.tape.InMemoryObjectQueue;
-import com.squareup.tape.ObjectQueue;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.Gauge;
-import com.yammer.metrics.core.MetricName;
-
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-import java.io.File;
-import java.io.RandomAccessFile;
-import java.nio.channels.FileChannel;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Factory for Square Tape {@link ObjectQueue} instances for this agent.
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class TapeDeck {
-  private static final Logger logger = Logger.getLogger(TapeDeck.class.getCanonicalName());
-
-  private final LoadingCache> queues;
-  private final boolean doPersist;
-
-  /**
-   * @param converter payload (de-)/serializer.
-   * @param doPersist whether to persist the queue
-   */
-  public TapeDeck(final FileObjectQueue.Converter converter, boolean doPersist) {
-    this.doPersist = doPersist;
-    queues = CacheBuilder.newBuilder().build(new CacheLoader>() {
-      @Override
-      public ObjectQueue load(@NotNull File file) throws Exception {
-
-        ObjectQueue queue;
-
-        if (doPersist) {
-
-          // We need exclusive ownership of the file for this deck.
-          // This is really no guarantee that we have exclusive access to the file (see e.g. goo.gl/i4S7ha)
-          try {
-            queue = new FileObjectQueue<>(file, converter);
-            FileChannel channel = new RandomAccessFile(file, "rw").getChannel();
-            Preconditions.checkNotNull(channel.tryLock());
-          } catch (Exception e) {
-            logger.log(Level.SEVERE, "Error while loading persisted Tape Queue for file " + file +
-                ". Please move or delete the file and restart the proxy.", e);
-            System.exit(-1);
-            queue = null;
-          }
-        } else {
-          queue = new InMemoryObjectQueue<>();
-        }
-
-        return new ReportingObjectQueueWrapper<>(queue, file.getName());
-      }
-    });
-  }
-
-  @Nullable
-  public ObjectQueue getTape(@NotNull File f) {
-    try {
-      return queues.get(f);
-    } catch (Exception e) {
-      logger.log(Level.SEVERE, "Error while loading " + f, e);
-      throw new RuntimeException("Unable to provide ObjectQueue", e);
-    }
-  }
-
-  @Override
-  public String toString() {
-    return "TapeDeck{" +
-        "queues=" + queues +
-        ", doPersist=" + doPersist +
-        '}';
-  }
-
-  /**
-   * Threadsafe ObjectQueue wrapper with add, remove and peek counters;
-   */
-  private static class ReportingObjectQueueWrapper implements ObjectQueue {
-    private final ObjectQueue backingQueue;
-    private final Counter addCounter;
-    private final Counter removeCounter;
-    private final Counter peekCounter;
-
-    // maintain a fair lock on the queue
-    private final ReentrantLock queueLock = new ReentrantLock(true);
-
-    ReportingObjectQueueWrapper(ObjectQueue backingQueue, String title) {
-      this.addCounter = Metrics.newCounter(new MetricName("tape." + title, "", "add"));
-      this.removeCounter = Metrics.newCounter(new MetricName("tape." + title, "", "remove"));
-      this.peekCounter = Metrics.newCounter(new MetricName("tape." + title, "", "peek"));
-      Metrics.newGauge(new MetricName("tape." + title, "", "size"),
-          new Gauge() {
-            @Override
-            public Integer value() {
-              return backingQueue.size();
-            }
-          });
-
-      this.backingQueue = backingQueue;
-    }
-
-    @Override
-    public int size() {
-      int backingQueueSize;
-      try {
-        queueLock.lock();
-        backingQueueSize = backingQueue.size();
-      } finally {
-        queueLock.unlock();
-      }
-      return backingQueueSize;
-    }
-
-    @Override
-    public void add(T t) {
-      addCounter.inc();
-      try {
-        queueLock.lock();
-        backingQueue.add(t);
-      } finally {
-        queueLock.unlock();
-      }
-    }
-
-    @Override
-    public T peek() {
-      peekCounter.inc();
-      T t;
-      try {
-        queueLock.lock();
-        t = backingQueue.peek();
-      } finally {
-        queueLock.unlock();
-      }
-      return t;
-    }
-
-    @Override
-    public void remove() {
-      removeCounter.inc();
-      try {
-        queueLock.lock();
-        backingQueue.remove();
-      } finally {
-        queueLock.unlock();
-      }
-    }
-
-    @Override
-    public void setListener(Listener listener) {
-      try {
-        queueLock.lock();
-        backingQueue.setListener(listener);
-      } finally {
-        queueLock.unlock();
-      }
-    }
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/tape/TapeReportPointConverter.java b/proxy/src/main/java/com/wavefront/agent/histogram/tape/TapeReportPointConverter.java
deleted file mode 100644
index c3c079390..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/tape/TapeReportPointConverter.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package com.wavefront.agent.histogram.tape;
-
-import com.squareup.tape.FileObjectQueue;
-
-import org.apache.avro.io.BinaryEncoder;
-import org.apache.avro.io.DatumWriter;
-import org.apache.avro.io.DecoderFactory;
-import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.specific.SpecificDatumReader;
-import org.apache.avro.specific.SpecificDatumWriter;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Adapter exposing the Avro's {@link org.apache.avro.specific.SpecificRecord} encoding/decoding to Square tape's {@link
- * com.squareup.tape.FileObjectQueue.Converter} interface.
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class TapeReportPointConverter implements FileObjectQueue.Converter {
-  private static final TapeReportPointConverter INSTANCE = new TapeReportPointConverter();
-
-  private TapeReportPointConverter() {
-    // Singleton
-  }
-
-  public static TapeReportPointConverter get() {
-    return INSTANCE;
-  }
-
-  @Override
-  public ReportPoint from(byte[] bytes) throws IOException {
-    SpecificDatumReader reader = new SpecificDatumReader<>(ReportPoint.SCHEMA$);
-    org.apache.avro.io.Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
-
-    return reader.read(null, decoder);
-  }
-
-  @Override
-  public void toStream(ReportPoint point, OutputStream outputStream) throws IOException {
-    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null);
-    DatumWriter writer = new SpecificDatumWriter<>(ReportPoint.SCHEMA$);
-
-    writer.write(point, encoder);
-    encoder.flush();
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/histogram/tape/TapeStringListConverter.java b/proxy/src/main/java/com/wavefront/agent/histogram/tape/TapeStringListConverter.java
deleted file mode 100644
index 058a378aa..000000000
--- a/proxy/src/main/java/com/wavefront/agent/histogram/tape/TapeStringListConverter.java
+++ /dev/null
@@ -1,123 +0,0 @@
-package com.wavefront.agent.histogram.tape;
-
-import com.google.common.base.Preconditions;
-
-import com.squareup.tape.FileObjectQueue;
-
-import net.jpountz.lz4.LZ4BlockInputStream;
-import net.jpountz.lz4.LZ4BlockOutputStream;
-
-import org.apache.commons.io.IOUtils;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Logger;
-import java.util.zip.GZIPInputStream;
-
-/**
- * [Square] Tape converter for Lists of strings with LZ4 compression support.
- *
- * Binary format is signature[4b] = 0x54415045, version[4b] = 0x00000001, length[4b], {utf8Len[4b], utfSeq}*
- *
- * @author Tim Schmidt (tim@wavefront.com).
- * @author Vasily Vorontsov (vasily@wavefront.com)
- *
- */
-public class TapeStringListConverter implements FileObjectQueue.Converter> {
-  private static final Logger logger = Logger.getLogger(TapeStringListConverter.class.getCanonicalName());
-
-  private static final TapeStringListConverter INSTANCE_DEFAULT = new TapeStringListConverter(false);
-  private static final TapeStringListConverter INSTANCE_COMPRESSION_ENABLED = new TapeStringListConverter(true);
-  private static final Charset UTF8 = Charset.forName("UTF-8");
-  private static final int CONTAINER_SIGNATURE = 0x54415045;
-  private static final int CONTAINER_VERSION = 0x00000001;
-
-  private final boolean isCompressionEnabled;
-
-  private TapeStringListConverter(boolean isCompressionEnabled) {
-    this.isCompressionEnabled = isCompressionEnabled;
-  }
-
-  /**
-   * Returns the TapeStringListConverter object instance with default settings (no compression)
-   *
-   * @return TapeStringListConverter object instance
-   */
-  public static TapeStringListConverter getDefaultInstance() {
-    return INSTANCE_DEFAULT;
-  }
-
-  /**
-   * Returns the TapeStringListConverter object instance with LZ4 compression enabled
-   *
-   * @return TapeStringListConverter object instance
-   */
-  public static TapeStringListConverter getCompressionEnabledInstance() {
-    return INSTANCE_COMPRESSION_ENABLED;
-  }
-
-  @Override
-  public List from(byte[] bytes) throws IOException {
-    try {
-      byte[] uncompressedData;
-      if (bytes.length > 2 && bytes[0] == (byte) 0x1f && bytes[1] == (byte) 0x8b) { // gzip signature
-        uncompressedData = IOUtils.toByteArray(new GZIPInputStream(new ByteArrayInputStream(bytes)));
-      } else if (bytes.length > 2 && bytes[0] == (byte) 0x4c && bytes[1] == (byte) 0x5a) { // LZ block signature
-        uncompressedData = IOUtils.toByteArray(new LZ4BlockInputStream(new ByteArrayInputStream(bytes)));
-      } else {
-        uncompressedData = bytes;
-      }
-      Preconditions.checkArgument(uncompressedData.length > 12, "Uncompressed container must be at least 12 bytes");
-      ByteBuffer in = ByteBuffer.wrap(uncompressedData);
-      int signature = in.getInt();
-      int version = in.getInt();
-      if (signature != CONTAINER_SIGNATURE || version != CONTAINER_VERSION) {
-        logger.severe("WF-502: Unexpected data format retrieved from tape (signature = " + signature +
-            ", version = " + version);
-        return null;
-      }
-      int count = in.getInt();
-      List result = new ArrayList<>(count);
-      for (int i = 0; i < count; ++i) {
-        int len = in.getInt();
-        result.add(new String(uncompressedData, in.position(), len, UTF8));
-        in.position(in.position() + len);
-      }
-      return result;
-    } catch (Throwable t) {
-      logger.severe("WF-501: Corrupt data retrieved from tape, ignoring: " + t);
-      return null;
-    }
-  }
-
-  @Override
-  public void toStream(List stringList, OutputStream outputStream) throws IOException {
-    OutputStream wrapperStream = null;
-    DataOutputStream dOut;
-    if (isCompressionEnabled) {
-      wrapperStream = new LZ4BlockOutputStream(outputStream);
-      dOut = new DataOutputStream(wrapperStream);
-    } else {
-      dOut = new DataOutputStream(outputStream);
-    }
-    dOut.writeInt(CONTAINER_SIGNATURE);
-    dOut.writeInt(CONTAINER_VERSION);
-    dOut.writeInt(stringList.size());
-    for (String s : stringList) {
-      byte[] b = s.getBytes(UTF8);
-      dOut.writeInt(b.length);
-      dOut.write(b);
-    }
-    // flush?
-    dOut.close();
-    if (wrapperStream != null) {
-      wrapperStream.close();
-    }
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/AbstractHttpOnlyHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/AbstractHttpOnlyHandler.java
new file mode 100644
index 000000000..327366b05
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/AbstractHttpOnlyHandler.java
@@ -0,0 +1,47 @@
+package com.wavefront.agent.listeners;
+
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import java.net.URISyntaxException;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * Base class for HTTP-only listeners.
+ *
+ * @author vasily@wavefront.com
+ */
+@ChannelHandler.Sharable
+public abstract class AbstractHttpOnlyHandler extends AbstractPortUnificationHandler {
+  private static final Logger logger =
+      Logger.getLogger(AbstractHttpOnlyHandler.class.getCanonicalName());
+
+  /**
+   * Create new instance.
+   *
+   * @param tokenAuthenticator {@link TokenAuthenticator} for incoming requests.
+   * @param healthCheckManager shared health check endpoint handler.
+   * @param handle handle/port number.
+   */
+  public AbstractHttpOnlyHandler(
+      @Nullable final TokenAuthenticator tokenAuthenticator,
+      @Nullable final HealthCheckManager healthCheckManager,
+      @Nullable final String handle) {
+    super(tokenAuthenticator, healthCheckManager, handle);
+  }
+
+  protected abstract void handleHttpMessage(
+      final ChannelHandlerContext ctx, final FullHttpRequest request) throws URISyntaxException;
+
+  /** Discards plaintext content. */
+  @Override
+  protected void handlePlainTextMessage(
+      final ChannelHandlerContext ctx, @Nonnull final String message) {
+    pointsDiscarded.get().inc();
+    logger.warning("Input discarded: plaintext protocol is not supported on port " + handle);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/AbstractLineDelimitedHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/AbstractLineDelimitedHandler.java
new file mode 100644
index 000000000..1fa35c2a0
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/AbstractLineDelimitedHandler.java
@@ -0,0 +1,186 @@
+package com.wavefront.agent.listeners;
+
+import static com.wavefront.agent.LogsUtil.LOGS_DATA_FORMATS;
+import static com.wavefront.agent.LogsUtil.getOrCreateLogsHistogramFromRegistry;
+import static com.wavefront.agent.channel.ChannelUtils.errorMessageWithRootCause;
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+import static com.wavefront.agent.formatter.DataFormat.LOGS_JSON_ARR;
+import static com.wavefront.agent.formatter.DataFormat.LOGS_JSON_CLOUDWATCH;
+import static com.wavefront.agent.formatter.DataFormat.LOGS_JSON_LINES;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.MappingIterator;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Splitter;
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.formatter.DataFormat;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Histogram;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.util.CharsetUtil;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Base class for all line-based protocols. Supports TCP line protocol as well as HTTP POST with
+ * newline-delimited payload.
+ *
+ * @author vasily@wavefront.com.
+ */
+@ChannelHandler.Sharable
+public abstract class AbstractLineDelimitedHandler extends AbstractPortUnificationHandler {
+
+  public static final ObjectMapper JSON_PARSER = new ObjectMapper();
+  public static final String LOG_EVENTS_KEY = "logEvents";
+
+  /**
+   * @param tokenAuthenticator {@link TokenAuthenticator} for incoming requests.
+   * @param healthCheckManager shared health check endpoint handler.
+   * @param handle handle/port number.
+   */
+  public AbstractLineDelimitedHandler(
+      @Nullable final TokenAuthenticator tokenAuthenticator,
+      @Nullable final HealthCheckManager healthCheckManager,
+      @Nullable final String handle) {
+    super(tokenAuthenticator, healthCheckManager, handle);
+  }
+
+  /** Handles an incoming HTTP message. Accepts HTTP POST on all paths */
+  @Override
+  protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request) {
+    StringBuilder output = new StringBuilder();
+    HttpResponseStatus status;
+    try {
+      DataFormat format = getFormat(request);
+      processBatchMetrics(ctx, request, format);
+      // Log batches may contain new lines as part of the message payload so we special case
+      // handling breaking up the batches
+      Iterable lines;
+
+      if (format == LOGS_JSON_ARR) {
+        lines = extractLogsWithJsonArrayFormat(request);
+      } else if (format == LOGS_JSON_LINES) {
+        lines = extractLogsWithJsonLinesFormat(request);
+      } else if (format == LOGS_JSON_CLOUDWATCH) {
+        lines = extractLogsWithJsonCloudwatchFormat(request);
+      } else {
+        lines = extractLogsWithDefaultFormat(request);
+      }
+
+      lines.forEach(line -> processLine(ctx, line, format));
+      status = HttpResponseStatus.ACCEPTED;
+    } catch (Exception e) {
+      status = HttpResponseStatus.BAD_REQUEST;
+      output.append(errorMessageWithRootCause(e));
+      logWarning("WF-300: Failed to handle HTTP POST", e, ctx);
+    }
+    writeHttpResponse(ctx, status, output, request);
+  }
+
+  private Iterable extractLogsWithDefaultFormat(FullHttpRequest request) {
+    return Splitter.on('\n')
+        .trimResults()
+        .omitEmptyStrings()
+        .split(request.content().toString(CharsetUtil.UTF_8));
+  }
+
+  private Iterable extractLogsWithJsonCloudwatchFormat(FullHttpRequest request)
+      throws IOException {
+    JsonNode node =
+        JSON_PARSER
+            .readerFor(JsonNode.class)
+            .readValue(request.content().toString(CharsetUtil.UTF_8));
+
+    return extractLogsFromArray(node.get(LOG_EVENTS_KEY).toString());
+  }
+
+  @NotNull
+  private static List extractLogsFromArray(String content) throws JsonProcessingException {
+    return JSON_PARSER
+        .readValue(content, new TypeReference>>() {})
+        .stream()
+        .map(
+            json -> {
+              try {
+                return JSON_PARSER.writeValueAsString(json);
+              } catch (JsonProcessingException e) {
+                return null;
+              }
+            })
+        .filter(Objects::nonNull)
+        .collect(Collectors.toList());
+  }
+
+  private Iterable extractLogsWithJsonLinesFormat(FullHttpRequest request)
+      throws IOException {
+    List lines = new ArrayList<>();
+    MappingIterator it =
+        JSON_PARSER
+            .readerFor(JsonNode.class)
+            .readValues(request.content().toString(CharsetUtil.UTF_8));
+    while (it.hasNextValue()) {
+      lines.add(JSON_PARSER.writeValueAsString(it.nextValue()));
+    }
+    return lines;
+  }
+
+  @NotNull
+  private static Iterable extractLogsWithJsonArrayFormat(FullHttpRequest request)
+      throws IOException {
+    return extractLogsFromArray(request.content().toString(CharsetUtil.UTF_8));
+  }
+
+  /**
+   * Handles an incoming plain text (string) message. By default simply passes a string to {@link
+   * #processLine(ChannelHandlerContext, String, DataFormat)} method.
+   */
+  @Override
+  protected void handlePlainTextMessage(
+      final ChannelHandlerContext ctx, @Nonnull final String message) {
+    String trimmedMessage = message.trim();
+    if (trimmedMessage.isEmpty()) return;
+    processLine(ctx, trimmedMessage, null);
+  }
+
+  /**
+   * Detect data format for an incoming HTTP request, if possible.
+   *
+   * @param httpRequest http request.
+   * @return Detected data format or null if unknown.
+   */
+  @Nullable
+  protected abstract DataFormat getFormat(FullHttpRequest httpRequest);
+
+  /**
+   * Process a single line for a line-based stream.
+   *
+   * @param ctx Channel handler context.
+   * @param message Message to process.
+   * @param format Data format, if known
+   */
+  protected abstract void processLine(
+      final ChannelHandlerContext ctx, @Nonnull final String message, @Nullable DataFormat format);
+
+  protected void processBatchMetrics(
+      final ChannelHandlerContext ctx, final FullHttpRequest request, @Nullable DataFormat format) {
+    if (LOGS_DATA_FORMATS.contains(format)) {
+      Histogram receivedLogsBatches =
+          getOrCreateLogsHistogramFromRegistry(
+              Metrics.defaultRegistry(), format, "logs." + handle, "received" + ".batches");
+      receivedLogsBatches.update(request.content().toString(CharsetUtil.UTF_8).length());
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/AbstractPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/AbstractPortUnificationHandler.java
new file mode 100644
index 000000000..f487c3dbc
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/AbstractPortUnificationHandler.java
@@ -0,0 +1,291 @@
+package com.wavefront.agent.listeners;
+
+import static com.wavefront.agent.channel.ChannelUtils.errorMessageWithRootCause;
+import static com.wavefront.agent.channel.ChannelUtils.formatErrorMessage;
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+import static com.wavefront.common.Utils.lazySupplier;
+import static org.apache.commons.lang3.ObjectUtils.firstNonNull;
+
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.channel.NoopHealthCheckManager;
+import com.wavefront.common.TaggedMetricName;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Histogram;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.TooLongFrameException;
+import io.netty.handler.codec.compression.DecompressionException;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.codec.http.HttpVersion;
+import io.netty.util.CharsetUtil;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.commons.lang3.ObjectUtils;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+
+/**
+ * This is a base class for the majority of proxy's listeners. Handles an incoming message of either
+ * String or FullHttpRequest type, all other types are ignored. Has ability to support health checks
+ * and authentication of incoming HTTP requests. Designed to be used with {@link
+ * com.wavefront.agent.channel.PlainTextOrHttpFrameDecoder}.
+ *
+ * @author vasily@wavefront.com
+ */
+@SuppressWarnings("SameReturnValue")
+@ChannelHandler.Sharable
+public abstract class AbstractPortUnificationHandler extends SimpleChannelInboundHandler {
+  private static final Logger logger =
+      Logger.getLogger(AbstractPortUnificationHandler.class.getCanonicalName());
+
+  protected final Supplier httpRequestHandleDuration;
+  protected final Supplier requestsDiscarded;
+  protected final Supplier pointsDiscarded;
+  protected final Supplier> httpRequestsInFlightGauge;
+  protected final AtomicLong httpRequestsInFlight = new AtomicLong();
+
+  protected final String handle;
+  protected final TokenAuthenticator tokenAuthenticator;
+  protected final HealthCheckManager healthCheck;
+
+  /**
+   * Create new instance.
+   *
+   * @param tokenAuthenticator {@link TokenAuthenticator} for incoming requests.
+   * @param healthCheckManager shared health check endpoint handler.
+   * @param handle handle/port number.
+   */
+  public AbstractPortUnificationHandler(
+      @Nullable final TokenAuthenticator tokenAuthenticator,
+      @Nullable final HealthCheckManager healthCheckManager,
+      @Nullable final String handle) {
+    this.tokenAuthenticator =
+        ObjectUtils.firstNonNull(tokenAuthenticator, TokenAuthenticator.DUMMY_AUTHENTICATOR);
+    this.healthCheck =
+        healthCheckManager == null ? new NoopHealthCheckManager() : healthCheckManager;
+    this.handle = firstNonNull(handle, "unknown");
+    String portNumber = this.handle.replaceAll("^\\d", "");
+    if (NumberUtils.isNumber(portNumber)) {
+      healthCheck.setHealthy(Integer.parseInt(portNumber));
+    }
+
+    this.httpRequestHandleDuration =
+        lazySupplier(
+            () ->
+                Metrics.newHistogram(
+                    new TaggedMetricName(
+                        "listeners", "http-requests.duration-nanos", "port", this.handle)));
+    this.requestsDiscarded =
+        lazySupplier(
+            () ->
+                Metrics.newCounter(
+                    new TaggedMetricName(
+                        "listeners", "http-requests.discarded", "port", this.handle)));
+    this.pointsDiscarded =
+        lazySupplier(
+            () ->
+                Metrics.newCounter(
+                    new TaggedMetricName("listeners", "items-discarded", "port", this.handle)));
+    this.httpRequestsInFlightGauge =
+        lazySupplier(
+            () ->
+                Metrics.newGauge(
+                    new TaggedMetricName("listeners", "http-requests.active", "port", this.handle),
+                    new Gauge() {
+                      @Override
+                      public Long value() {
+                        return httpRequestsInFlight.get();
+                      }
+                    }));
+  }
+
+  /**
+   * Process incoming HTTP request.
+   *
+   * @param ctx Channel handler's context
+   * @param request HTTP request to process
+   * @throws URISyntaxException in case of a malformed URL
+   */
+  protected abstract void handleHttpMessage(
+      final ChannelHandlerContext ctx, final FullHttpRequest request) throws URISyntaxException;
+
+  /**
+   * Process incoming plaintext string.
+   *
+   * @param ctx Channel handler's context
+   * @param message Plaintext message to process
+   */
+  protected abstract void handlePlainTextMessage(
+      final ChannelHandlerContext ctx, @Nonnull final String message);
+
+  @Override
+  public void channelReadComplete(ChannelHandlerContext ctx) {
+    ctx.flush();
+  }
+
+  @Override
+  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+    if (cause instanceof TooLongFrameException) {
+      logWarning(
+          "Received line is too long, consider increasing pushListenerMaxReceivedLength",
+          cause,
+          ctx);
+      return;
+    }
+    if (cause instanceof DecompressionException) {
+      logWarning("Decompression error", cause, ctx);
+      writeHttpResponse(
+          ctx, HttpResponseStatus.BAD_REQUEST, "Decompression error: " + cause.getMessage(), false);
+      return;
+    }
+    if (cause instanceof IOException && cause.getMessage().contains("Connection reset by peer")) {
+      // These errors are caused by the client and are safe to ignore
+      return;
+    }
+    logWarning("Handler failed", cause, ctx);
+    logger.log(Level.WARNING, "Unexpected error: ", cause);
+  }
+
+  protected String extractToken(final FullHttpRequest request) {
+    String token =
+        firstNonNull(
+                request.headers().getAsString("X-AUTH-TOKEN"),
+                request.headers().getAsString("Authorization"),
+                "")
+            .replaceAll("^Bearer ", "")
+            .trim();
+    Optional tokenParam =
+        URLEncodedUtils.parse(URI.create(request.uri()), CharsetUtil.UTF_8).stream()
+            .filter(
+                x ->
+                    x.getName().equals("t")
+                        || x.getName().equals("token")
+                        || x.getName().equals("api_key"))
+            .findFirst();
+    if (tokenParam.isPresent()) {
+      token = tokenParam.get().getValue();
+    }
+    return token;
+  }
+
+  protected boolean authorized(final ChannelHandlerContext ctx, final FullHttpRequest request) {
+    if (tokenAuthenticator.authRequired()) {
+      String token = extractToken(request);
+      if (!tokenAuthenticator.authorize(token)) { // 401 if no token or auth fails
+        writeHttpResponse(ctx, HttpResponseStatus.UNAUTHORIZED, "401 Unauthorized\n", request);
+        return false;
+      }
+    }
+    return true;
+  }
+
+  @Override
+  protected void channelRead0(final ChannelHandlerContext ctx, final Object message) {
+    if (message instanceof String) {
+      try {
+        if (tokenAuthenticator.authRequired()) {
+          // plaintext is disabled with auth enabled
+          pointsDiscarded.get().inc();
+          logger.warning(
+              "Input discarded: plaintext protocol is not supported on port "
+                  + handle
+                  + " (authentication enabled)");
+          return;
+        }
+        handlePlainTextMessage(ctx, (String) message);
+      } catch (final Exception e) {
+        e.printStackTrace();
+        logWarning("Failed to handle message", e, ctx);
+      }
+    } else if (message instanceof FullHttpRequest) {
+      FullHttpRequest request = (FullHttpRequest) message;
+      try {
+        HttpResponse healthCheckResponse = healthCheck.getHealthCheckResponse(ctx, request);
+        if (healthCheckResponse != null) {
+          writeHttpResponse(ctx, healthCheckResponse, request);
+          return;
+        }
+        if (!getHttpEnabled()) {
+          requestsDiscarded.get().inc();
+          logger.warning("Inbound HTTP request discarded: HTTP disabled on port " + handle);
+          return;
+        }
+        if (authorized(ctx, request)) {
+          httpRequestsInFlightGauge.get();
+          httpRequestsInFlight.incrementAndGet();
+          long startTime = System.nanoTime();
+          if (request.method() == HttpMethod.OPTIONS) {
+            writeHttpResponse(
+                ctx,
+                new DefaultFullHttpResponse(
+                    HttpVersion.HTTP_1_1, HttpResponseStatus.NO_CONTENT, Unpooled.EMPTY_BUFFER),
+                request);
+            return;
+          }
+          try {
+            handleHttpMessage(ctx, request);
+          } finally {
+            httpRequestsInFlight.decrementAndGet();
+          }
+          httpRequestHandleDuration.get().update(System.nanoTime() - startTime);
+        }
+      } catch (URISyntaxException e) {
+        writeHttpResponse(
+            ctx, HttpResponseStatus.BAD_REQUEST, errorMessageWithRootCause(e), request);
+        logger.warning(
+            formatErrorMessage(
+                "WF-300: Request URI '" + request.uri() + "' cannot be parsed", e, ctx));
+      } catch (final Exception e) {
+        e.printStackTrace();
+        logWarning("Failed to handle message", e, ctx);
+      }
+    } else {
+      logWarning(
+          "Received unexpected message type "
+              + (message == null ? "" : message.getClass().getName()),
+          null,
+          ctx);
+    }
+  }
+
+  /**
+   * Checks whether HTTP protocol is enabled on this port
+   *
+   * @return whether HTTP protocol is enabled
+   */
+  protected boolean getHttpEnabled() {
+    return true;
+  }
+
+  /**
+   * Log a detailed error message with remote IP address
+   *
+   * @param message the error message
+   * @param e the exception (optional) that caused the message to be blocked
+   * @param ctx ChannelHandlerContext (optional) to extract remote client ip
+   */
+  protected void logWarning(
+      final String message,
+      @Nullable final Throwable e,
+      @Nullable final ChannelHandlerContext ctx) {
+    logger.warning(formatErrorMessage(message, e, ctx));
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/AdminPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/AdminPortUnificationHandler.java
new file mode 100644
index 000000000..4d053e141
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/AdminPortUnificationHandler.java
@@ -0,0 +1,133 @@
+package com.wavefront.agent.listeners;
+
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
+
+/**
+ * Admin API for managing proxy-wide healthchecks. Access can be restricted by a client's IP address
+ * (must match provided allow list regex). Exposed endpoints: - GET /status/{port} check current
+ * status for {port}, returns 200 if enabled / 503 if not. - POST /enable/{port} mark port {port} as
+ * healthy. - POST /disable/{port} mark port {port} as unhealthy. - POST /enable mark all
+ * healthcheck-enabled ports as healthy. - POST /disable mark all healthcheck-enabled ports as
+ * unhealthy.
+ *
+ * @author vasily@wavefront.com
+ */
+@ChannelHandler.Sharable
+public class AdminPortUnificationHandler extends AbstractHttpOnlyHandler {
+  private static final Logger logger =
+      Logger.getLogger(AdminPortUnificationHandler.class.getCanonicalName());
+
+  private static final Pattern PATH = Pattern.compile("/(enable|disable|status)/?(\\d*)/?");
+
+  private final String remoteIpAllowRegex;
+
+  /**
+   * Create new instance.
+   *
+   * @param tokenAuthenticator {@link TokenAuthenticator} for incoming requests.
+   * @param healthCheckManager shared health check endpoint handler.
+   * @param handle handle/port number.
+   */
+  public AdminPortUnificationHandler(
+      @Nullable TokenAuthenticator tokenAuthenticator,
+      @Nullable HealthCheckManager healthCheckManager,
+      @Nullable String handle,
+      @Nullable String remoteIpAllowRegex) {
+    super(tokenAuthenticator, healthCheckManager, handle);
+    this.remoteIpAllowRegex = remoteIpAllowRegex;
+  }
+
+  @Override
+  protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request)
+      throws URISyntaxException {
+    StringBuilder output = new StringBuilder();
+    String remoteIp =
+        ((InetSocketAddress) ctx.channel().remoteAddress()).getAddress().getHostAddress();
+    if (remoteIpAllowRegex != null
+        && !Pattern.compile(remoteIpAllowRegex).matcher(remoteIp).matches()) {
+      logger.warning("Incoming request from non-allowed remote address " + remoteIp + " rejected!");
+      writeHttpResponse(ctx, HttpResponseStatus.UNAUTHORIZED, output, request);
+      return;
+    }
+    URI uri = new URI(request.uri());
+    HttpResponseStatus status;
+    Matcher path = PATH.matcher(uri.getPath());
+    if (path.matches()) {
+      String strPort = path.group(2);
+      Integer port = NumberUtils.isNumber(strPort) ? Integer.parseInt(strPort) : null;
+      if (StringUtils.isBlank(strPort) || port != null) {
+        switch (path.group(1)) {
+          case "status":
+            if (request.method().equals(HttpMethod.GET)) {
+              if (port == null) {
+                output.append("Status check requires a specific port");
+                status = HttpResponseStatus.BAD_REQUEST;
+              } else {
+                // return 200 if status check ok, 503 if not
+                status =
+                    healthCheck.isHealthy(port)
+                        ? HttpResponseStatus.OK
+                        : HttpResponseStatus.SERVICE_UNAVAILABLE;
+                output.append(status.reasonPhrase());
+              }
+            } else {
+              status = HttpResponseStatus.METHOD_NOT_ALLOWED;
+            }
+            break;
+          case "enable":
+            if (request.method().equals(HttpMethod.POST)) {
+              if (port == null) {
+                logger.info("Request to mark all HTTP ports as healthy from remote: " + remoteIp);
+                healthCheck.setAllHealthy();
+              } else {
+                logger.info("Marking HTTP port " + port + " as healthy, remote: " + remoteIp);
+                healthCheck.setHealthy(port);
+              }
+              status = HttpResponseStatus.OK;
+            } else {
+              status = HttpResponseStatus.METHOD_NOT_ALLOWED;
+            }
+            break;
+          case "disable":
+            if (request.method().equals(HttpMethod.POST)) {
+              if (port == null) {
+                logger.info("Request to mark all HTTP ports as unhealthy from remote: " + remoteIp);
+                healthCheck.setAllUnhealthy();
+              } else {
+                logger.info("Marking HTTP port " + port + " as unhealthy, remote: " + remoteIp);
+                healthCheck.setUnhealthy(port);
+              }
+              status = HttpResponseStatus.OK;
+            } else {
+              status = HttpResponseStatus.METHOD_NOT_ALLOWED;
+            }
+            break;
+          default:
+            status = HttpResponseStatus.BAD_REQUEST;
+        }
+      } else {
+        status = HttpResponseStatus.BAD_REQUEST;
+      }
+    } else {
+      status = HttpResponseStatus.NOT_FOUND;
+    }
+    writeHttpResponse(ctx, status, output, request);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/ChannelByteArrayHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/ChannelByteArrayHandler.java
index 34ccd7369..c84704268 100644
--- a/proxy/src/main/java/com/wavefront/agent/listeners/ChannelByteArrayHandler.java
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/ChannelByteArrayHandler.java
@@ -1,81 +1,81 @@
 package com.wavefront.agent.listeners;
 
 import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-
-import com.wavefront.agent.PointHandler;
-import com.wavefront.agent.PointHandlerImpl;
-import com.wavefront.agent.preprocessor.ReportableEntityPreprocessor;
-import com.wavefront.ingester.Decoder;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
 import com.wavefront.ingester.GraphiteDecoder;
-
+import com.wavefront.ingester.ReportPointSerializer;
+import com.wavefront.ingester.ReportableEntityDecoder;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
 import java.net.InetSocketAddress;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-
 import javax.annotation.Nullable;
-
-import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
 import wavefront.report.ReportPoint;
 
-
 /**
  * Channel handler for byte array data.
+ *
  * @author Mike McLaughlin (mike@wavefront.com)
  */
 @ChannelHandler.Sharable
 public class ChannelByteArrayHandler extends SimpleChannelInboundHandler {
-  private static final Logger logger = Logger.getLogger(ChannelByteArrayHandler.class.getCanonicalName());
-  private static final Logger blockedPointsLogger = Logger.getLogger("RawBlockedPoints");
+  private static final Logger logger =
+      Logger.getLogger(ChannelByteArrayHandler.class.getCanonicalName());
 
-  private final Decoder decoder;
-  private final PointHandler pointHandler;
+  private final ReportableEntityDecoder decoder;
+  private final ReportableEntityHandler pointHandler;
 
-  @Nullable
-  private final ReportableEntityPreprocessor preprocessor;
+  @Nullable private final Supplier preprocessorSupplier;
+  private final Logger blockedItemsLogger;
   private final GraphiteDecoder recoder;
 
-  /**
-   * Constructor.
-   */
-  public ChannelByteArrayHandler(Decoder decoder,
-                          final PointHandler pointHandler,
-                          @Nullable final ReportableEntityPreprocessor preprocessor) {
+  /** Constructor. */
+  public ChannelByteArrayHandler(
+      final ReportableEntityDecoder decoder,
+      final ReportableEntityHandler pointHandler,
+      @Nullable final Supplier preprocessorSupplier,
+      final Logger blockedItemsLogger) {
     this.decoder = decoder;
     this.pointHandler = pointHandler;
-    this.preprocessor = preprocessor;
+    this.preprocessorSupplier = preprocessorSupplier;
+    this.blockedItemsLogger = blockedItemsLogger;
     this.recoder = new GraphiteDecoder(Collections.emptyList());
   }
 
   @Override
-  protected void channelRead0(ChannelHandlerContext ctx, byte[] msg) throws Exception {
+  protected void channelRead0(ChannelHandlerContext ctx, byte[] msg) {
     // ignore empty lines.
     if (msg == null || msg.length == 0) {
       return;
     }
 
-    List points = Lists.newArrayListWithExpectedSize(1);
+    ReportableEntityPreprocessor preprocessor =
+        preprocessorSupplier == null ? null : preprocessorSupplier.get();
+
+    List points = new ArrayList<>(1);
     try {
-      decoder.decodeReportPoints(msg, points, "dummy");
-      for (ReportPoint point: points) {
-        if (preprocessor != null && preprocessor.forPointLine().hasTransformers()) {
-          String pointLine = PointHandlerImpl.pointToString(point);
+      decoder.decode(msg, points, "dummy");
+      for (ReportPoint point : points) {
+        if (preprocessor != null && !preprocessor.forPointLine().getTransformers().isEmpty()) {
+          String pointLine = ReportPointSerializer.pointToString(point);
           pointLine = preprocessor.forPointLine().transform(pointLine);
-          List parsedPoints = Lists.newArrayListWithExpectedSize(1);
+          List parsedPoints = new ArrayList<>(1);
           recoder.decodeReportPoints(pointLine, parsedPoints, "dummy");
-          parsedPoints.forEach(this::preprocessAndReportPoint);
+          parsedPoints.forEach(x -> preprocessAndReportPoint(x, preprocessor));
         } else {
-          preprocessAndReportPoint(point);
+          preprocessAndReportPoint(point, preprocessor);
         }
       }
     } catch (final Exception e) {
       final Throwable rootCause = Throwables.getRootCause(e);
-      String errMsg = "WF-300 Cannot parse: \"" +
-          "\", reason: \"" + e.getMessage() + "\"";
+      String errMsg = "WF-300 Cannot parse: \"" + "\", reason: \"" + e.getMessage() + "\"";
       if (rootCause != null && rootCause.getMessage() != null) {
         errMsg = errMsg + ", root cause: \"" + rootCause.getMessage() + "\"";
       }
@@ -84,47 +84,48 @@ protected void channelRead0(ChannelHandlerContext ctx, byte[] msg) throws Except
         errMsg += "; remote: " + remoteAddress.getHostString();
       }
       logger.log(Level.WARNING, errMsg, e);
-      pointHandler.handleBlockedPoint(errMsg);
+      pointHandler.block(null, errMsg);
     }
   }
 
-  private void preprocessAndReportPoint(ReportPoint point) {
+  private void preprocessAndReportPoint(
+      ReportPoint point, ReportableEntityPreprocessor preprocessor) {
+    String[] messageHolder = new String[1];
     if (preprocessor == null) {
-      pointHandler.reportPoint(point, point.getMetric());
+      pointHandler.report(point);
       return;
     }
     // backwards compatibility: apply "pointLine" rules to metric name
-    if (!preprocessor.forPointLine().filter(point.getMetric())) {
-      if (preprocessor.forPointLine().getLastFilterResult() != null) {
-        blockedPointsLogger.warning(PointHandlerImpl.pointToString(point));
+    if (!preprocessor.forPointLine().filter(point.getMetric(), messageHolder)) {
+      if (messageHolder[0] != null) {
+        blockedItemsLogger.warning(ReportPointSerializer.pointToString(point));
       } else {
-        blockedPointsLogger.info(PointHandlerImpl.pointToString(point));
+        blockedItemsLogger.info(ReportPointSerializer.pointToString(point));
       }
-      pointHandler.handleBlockedPoint(preprocessor.forPointLine().getLastFilterResult());
+      pointHandler.block(point, messageHolder[0]);
       return;
     }
     preprocessor.forReportPoint().transform(point);
-    if (!preprocessor.forReportPoint().filter(point)) {
-      if (preprocessor.forReportPoint().getLastFilterResult() != null) {
-        blockedPointsLogger.warning(PointHandlerImpl.pointToString(point));
+    if (!preprocessor.forReportPoint().filter(point, messageHolder)) {
+      if (messageHolder[0] != null) {
+        blockedItemsLogger.warning(ReportPointSerializer.pointToString(point));
       } else {
-        blockedPointsLogger.info(PointHandlerImpl.pointToString(point));
+        blockedItemsLogger.info(ReportPointSerializer.pointToString(point));
       }
-      pointHandler.handleBlockedPoint(preprocessor.forReportPoint().getLastFilterResult());
+      pointHandler.block(point, messageHolder[0]);
       return;
     }
-    pointHandler.reportPoint(point, point.getMetric());
+    pointHandler.report(point);
   }
 
   @Override
-  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
     if (cause.getMessage().contains("Connection reset by peer")) {
       // These errors are caused by the client and are safe to ignore
       return;
     }
     final Throwable rootCause = Throwables.getRootCause(cause);
-    String message = "WF-301 Error while receiving data, reason: \""
-        + cause.getMessage() + "\"";
+    String message = "WF-301 Error while receiving data, reason: \"" + cause.getMessage() + "\"";
     if (rootCause != null && rootCause.getMessage() != null) {
       message += ", root cause: \"" + rootCause.getMessage() + "\"";
     }
@@ -132,6 +133,6 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E
     if (remoteAddress != null) {
       message += "; remote: " + remoteAddress.getHostString();
     }
-    pointHandler.handleBlockedPoint(message);
+    logger.warning(message);
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/ChannelStringHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/ChannelStringHandler.java
deleted file mode 100644
index 0f034117c..000000000
--- a/proxy/src/main/java/com/wavefront/agent/listeners/ChannelStringHandler.java
+++ /dev/null
@@ -1,199 +0,0 @@
-package com.wavefront.agent.listeners;
-
-import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-
-import com.wavefront.agent.PointHandler;
-import com.wavefront.agent.PointHandlerImpl;
-import com.wavefront.agent.preprocessor.ReportableEntityPreprocessor;
-import com.wavefront.ingester.Decoder;
-
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import javax.annotation.Nullable;
-
-import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-
-import org.apache.commons.lang.math.NumberUtils;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Parses points from a channel using the given decoder and send it off to the AgentAPI interface.
- *
- * @author Clement Pang (clement@wavefront.com).
- */
-@Deprecated
-@ChannelHandler.Sharable
-public class ChannelStringHandler extends SimpleChannelInboundHandler {
-
-  private static final Logger blockedPointsLogger = Logger.getLogger("RawBlockedPoints");
-  private static final Logger rawDataLogger = Logger.getLogger("RawDataLogger");
-
-  private final Decoder decoder;
-  private static final Random RANDOM = new Random();
-
-  /**
-   * Transformer to transform each line.
-   */
-  @Nullable
-  private final ReportableEntityPreprocessor preprocessor;
-  private final PointHandler pointHandler;
-
-  /**
-   * Value of system property wavefront.proxy.lograwdata (for backwards compatibility)
-   */
-  private final boolean logRawDataFlag;
-  private double logRawDataRate;
-  private volatile long logRawUpdatedMillis = 0L;
-  private boolean logRawData = false;
-
-  public ChannelStringHandler(Decoder decoder,
-                              final PointHandler pointhandler,
-                              @Nullable final ReportableEntityPreprocessor preprocessor) {
-    this.decoder = decoder;
-    this.pointHandler = pointhandler;
-    this.preprocessor = preprocessor;
-
-    // check the property setting for logging raw data
-    @Nullable String logRawDataProperty = System.getProperty("wavefront.proxy.lograwdata");
-    logRawDataFlag = logRawDataProperty != null && logRawDataProperty.equalsIgnoreCase("true");
-    @Nullable String logRawDataSampleRateProperty = System.getProperty("wavefront.proxy.lograwdata.sample-rate");
-    this.logRawDataRate = logRawDataSampleRateProperty != null &&
-        NumberUtils.isNumber(logRawDataSampleRateProperty) ? Double.parseDouble(logRawDataSampleRateProperty) : 1.0d;
-
-    // make sure the rate fits between 0.0d - 1.0d
-    if (logRawDataRate < 0.0d) {
-      rawDataLogger.info("Invalid log raw data rate:" + logRawDataRate + ", adjusted to 0.0");
-      logRawDataRate = 0.0d;
-    } else if (logRawDataRate > 1.0d) {
-      rawDataLogger.info("Invalid log raw data rate:" + logRawDataRate + ", adjusted to 1.0");
-      logRawDataRate = 1.0d;
-    }
-  }
-
-  @Override
-  protected void channelRead0(ChannelHandlerContext ctx, String msg) throws Exception {
-    // use data rate to determine sampling rate
-    // logging includes the source host and port
-    if (logRawUpdatedMillis + TimeUnit.SECONDS.toMillis(1) < System.currentTimeMillis()) {
-      if (logRawData != rawDataLogger.isLoggable(Level.FINEST)) {
-        logRawData = !logRawData;
-        rawDataLogger.info("Raw data logging is now " + (logRawData ?
-            "enabled with " + (logRawDataRate * 100) + "% sampling" :
-            "disabled"));
-      }
-      logRawUpdatedMillis = System.currentTimeMillis();
-    }
-
-    if ((logRawData || logRawDataFlag) &&
-        (logRawDataRate >= 1.0d || (logRawDataRate > 0.0d && RANDOM.nextDouble() < logRawDataRate))) {
-      if (ctx.channel().remoteAddress() != null) {
-        String hostAddress = ((InetSocketAddress) ctx.channel().remoteAddress()).getAddress().getHostAddress();
-        int localPort = ((InetSocketAddress) ctx.channel().localAddress()).getPort();
-        rawDataLogger.info("[" + hostAddress + ">" + localPort + "]" + msg);
-      } else {
-        int localPort = ((InetSocketAddress) ctx.channel().localAddress()).getPort();
-        rawDataLogger.info("[>" + localPort + "]" + msg);
-        
-      }
-    }
-    processPointLine(msg, decoder, pointHandler, preprocessor, ctx);
-  }
-
-  /**
-   * This probably belongs in a base class.  It's only done like this so it can be easily re-used. This should be
-   * refactored when it's clear where it belongs.
-   */
-  public static void processPointLine(final String message,
-                                      Decoder decoder,
-                                      final PointHandler pointHandler,
-                                      @Nullable final ReportableEntityPreprocessor preprocessor,
-                                      @Nullable final ChannelHandlerContext ctx) {
-    // ignore empty lines.
-    if (message == null) return;
-    String pointLine = message.trim();
-    if (pointLine.isEmpty()) return;
-
-    // transform the line if needed
-    if (preprocessor != null) {
-      pointLine = preprocessor.forPointLine().transform(pointLine);
-      // apply white/black lists after formatting
-      if (!preprocessor.forPointLine().filter(pointLine)) {
-        if (preprocessor.forPointLine().getLastFilterResult() != null) {
-          blockedPointsLogger.warning(pointLine);
-        } else {
-          blockedPointsLogger.info(pointLine);
-        }
-        pointHandler.handleBlockedPoint(preprocessor.forPointLine().getLastFilterResult());
-        return;
-      }
-    }
-
-    // decode the line into report points
-    List points = Lists.newArrayListWithExpectedSize(1);
-    try {
-      decoder.decodeReportPoints(pointLine, points, "dummy");
-    } catch (Exception e) {
-      final Throwable rootCause = Throwables.getRootCause(e);
-      String errMsg = "WF-300 Cannot parse: \"" + pointLine +
-          "\", reason: \"" + e.getMessage() + "\"";
-      if (rootCause != null && rootCause.getMessage() != null && rootCause != e) {
-        errMsg = errMsg + ", root cause: \"" + rootCause.getMessage() + "\"";
-      }
-      if (ctx != null) {
-        InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress();
-        if (remoteAddress != null) {
-          errMsg += "; remote: " + remoteAddress.getHostString();
-        }
-      }
-      blockedPointsLogger.warning(pointLine);
-      pointHandler.handleBlockedPoint(errMsg);
-      return;
-    }
-
-    // transform the point after parsing, and apply additional white/black lists if any
-    if (preprocessor != null) {
-      for (ReportPoint point : points) {
-        preprocessor.forReportPoint().transform(point);
-        if (!preprocessor.forReportPoint().filter(point)) {
-          if (preprocessor.forReportPoint().getLastFilterResult() != null) {
-            blockedPointsLogger.warning(PointHandlerImpl.pointToString(point));
-          } else {
-            blockedPointsLogger.info(PointHandlerImpl.pointToString(point));
-          }
-          pointHandler.handleBlockedPoint(preprocessor.forReportPoint().getLastFilterResult());
-          return;
-        }
-      }
-    }
-    pointHandler.reportPoints(points);
-  }
-
-  @Override
-  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
-    if (cause.getMessage().contains("Connection reset by peer")) {
-      // These errors are caused by the client and are safe to ignore
-      return;
-    }
-    final Throwable rootCause = Throwables.getRootCause(cause);
-    String message = "WF-301 Error while receiving data, reason: \""
-        + cause.getMessage() + "\"";
-    if (rootCause != null && rootCause.getMessage() != null) {
-      message += ", root cause: \"" + rootCause.getMessage() + "\"";
-    }
-    InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress();
-    if (remoteAddress != null) {
-      message += "; remote: " + remoteAddress.getHostString();
-    }
-    pointHandler.handleBlockedPoint(message);
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/DataDogPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/DataDogPortUnificationHandler.java
index a074ac910..3545805c7 100644
--- a/proxy/src/main/java/com/wavefront/agent/listeners/DataDogPortUnificationHandler.java
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/DataDogPortUnificationHandler.java
@@ -1,174 +1,291 @@
 package com.wavefront.agent.listeners;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
+/**
+ * How-to create the required protobuf classes:
+ *
+ * 
1. Check out this repo 'https://github.com/DataDog/agent-payload' on
+ * '$GOPATH/src/github.com/DataDog/agent-payload'
+ *
+ * 
2. run protoc -I=. --java_out=[PROXY_SRC]/proxy/src/main/java/
+ * $GOPATH/github.com/DataDog/agent-payload/proto/metrics/agent_payload.proto
+ *
+ * 
3. run protoc -I=. --java_out=[PROXY_SRC]/proxy/src/main/java/
+ * $GOPATH/github.com/gogo/protobuf/gogoproto/gogo.proto
+ *
+ * 
That will generate this 2 classes:
+ *
+ * 
- proxy/src/main/java/com/google/protobuf/GoGoProtos.java
+ *
+ * 
- proxy/src/main/java/datadog/agentpayload/AgentPayload.java
+ */
+import static com.wavefront.agent.channel.ChannelUtils.errorMessageWithRootCause;
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+import static io.netty.handler.codec.http.HttpMethod.POST;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
-import com.wavefront.agent.PointHandlerImpl;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
 import com.wavefront.agent.auth.TokenAuthenticatorBuilder;
-import com.wavefront.agent.auth.TokenValidationMethod;
+import com.wavefront.agent.channel.HealthCheckManager;
 import com.wavefront.agent.handlers.HandlerKey;
 import com.wavefront.agent.handlers.ReportableEntityHandler;
 import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
-import com.wavefront.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
 import com.wavefront.common.Clock;
+import com.wavefront.common.NamedThreadFactory;
 import com.wavefront.common.TaggedMetricName;
 import com.wavefront.data.ReportableEntityType;
+import com.wavefront.ingester.ReportPointSerializer;
 import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
 import com.yammer.metrics.core.Gauge;
 import com.yammer.metrics.core.Histogram;
-
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.util.EntityUtils;
-
+import datadog.agentpayload.AgentPayload;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
-
 import javax.annotation.Nullable;
-
-import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.http.FullHttpRequest;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import io.netty.util.CharsetUtil;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.util.EntityUtils;
 import wavefront.report.ReportPoint;
 
-import static io.netty.handler.codec.http.HttpMethod.POST;
-
 /**
- * This class handles an incoming message of either String or FullHttpRequest type.  All other types are ignored.
+ * Accepts incoming HTTP requests in DataDog JSON format. has the ability to relay them to DataDog.
  *
  * @author vasily@wavefront.com
  */
 @ChannelHandler.Sharable
-public class DataDogPortUnificationHandler extends PortUnificationHandler {
-  private static final Logger logger = Logger.getLogger(DataDogPortUnificationHandler.class.getCanonicalName());
+public class DataDogPortUnificationHandler extends AbstractHttpOnlyHandler {
+  private static final Logger logger =
+      Logger.getLogger(DataDogPortUnificationHandler.class.getCanonicalName());
   private static final Logger blockedPointsLogger = Logger.getLogger("RawBlockedPoints");
   private static final Pattern INVALID_METRIC_CHARACTERS = Pattern.compile("[^-_\\.\\dA-Za-z]");
   private static final Pattern INVALID_TAG_CHARACTERS = Pattern.compile("[^-_:\\.\\\\/\\dA-Za-z]");
 
-  private volatile Histogram httpRequestSize;
+  private static final Map SYSTEM_METRICS =
+      ImmutableMap.builder()
+          .put("system.cpu.guest", "cpuGuest")
+          .put("system.cpu.idle", "cpuIdle")
+          .put("system.cpu.stolen", "cpuStolen")
+          .put("system.cpu.system", "cpuSystem")
+          .put("system.cpu.user", "cpuUser")
+          .put("system.cpu.wait", "cpuWait")
+          .put("system.mem.buffers", "memBuffers")
+          .put("system.mem.cached", "memCached")
+          .put("system.mem.page_tables", "memPageTables")
+          .put("system.mem.shared", "memShared")
+          .put("system.mem.slab", "memSlab")
+          .put("system.mem.free", "memPhysFree")
+          .put("system.mem.pct_usable", "memPhysPctUsable")
+          .put("system.mem.total", "memPhysTotal")
+          .put("system.mem.usable", "memPhysUsable")
+          .put("system.mem.used", "memPhysUsed")
+          .put("system.swap.cached", "memSwapCached")
+          .put("system.swap.free", "memSwapFree")
+          .put("system.swap.pct_free", "memSwapPctFree")
+          .put("system.swap.total", "memSwapTotal")
+          .put("system.swap.used", "memSwapUsed")
+          .build();
+
+  private final Histogram httpRequestSize;
 
   /**
-   * The point handler that takes report metrics one data point at a time and handles batching and retries, etc
+   * The point handler that takes report metrics one data point at a time and handles batching and
+   * retries, etc
    */
-  private final ReportableEntityHandler pointHandler;
+  private final ReportableEntityHandler pointHandler;
+
+  private final boolean synchronousMode;
   private final boolean processSystemMetrics;
   private final boolean processServiceChecks;
-  @Nullable
-  private final HttpClient requestRelayClient;
-  @Nullable
-  private final String requestRelayTarget;
+  @Nullable private final HttpClient requestRelayClient;
+  @Nullable private final String requestRelayTarget;
 
-  @Nullable
-  private final ReportableEntityPreprocessor preprocessor;
+  @Nullable private final Supplier preprocessorSupplier;
 
   private final ObjectMapper jsonParser;
 
-  private final Cache> tagsCache = Caffeine.newBuilder().
-      expireAfterWrite(6, TimeUnit.HOURS).
-      maximumSize(100_000).
-      build();
-
-  @SuppressWarnings("unchecked")
-  public DataDogPortUnificationHandler(final String handle,
-                                       final ReportableEntityHandlerFactory handlerFactory,
-                                       final boolean processSystemMetrics,
-                                       final boolean processServiceChecks,
-                                       @Nullable final HttpClient requestRelayClient,
-                                       @Nullable final String requestRelayTarget,
-                                       @Nullable final ReportableEntityPreprocessor preprocessor) {
-    this(handle, handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle)), processSystemMetrics,
-        processServiceChecks, requestRelayClient, requestRelayTarget, preprocessor);
+  private final Cache> tagsCache =
+      Caffeine.newBuilder().expireAfterWrite(6, TimeUnit.HOURS).maximumSize(100_000).build();
+  private final LoadingCache httpStatusCounterCache;
+  private final ScheduledThreadPoolExecutor threadpool;
+
+  public DataDogPortUnificationHandler(
+      final String handle,
+      final HealthCheckManager healthCheckManager,
+      final ReportableEntityHandlerFactory handlerFactory,
+      final int fanout,
+      final boolean synchronousMode,
+      final boolean processSystemMetrics,
+      final boolean processServiceChecks,
+      @Nullable final HttpClient requestRelayClient,
+      @Nullable final String requestRelayTarget,
+      @Nullable final Supplier preprocessor) {
+    this(
+        handle,
+        healthCheckManager,
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle)),
+        fanout,
+        synchronousMode,
+        processSystemMetrics,
+        processServiceChecks,
+        requestRelayClient,
+        requestRelayTarget,
+        preprocessor);
   }
 
-
   @VisibleForTesting
-  protected DataDogPortUnificationHandler(final String handle,
-                                          final ReportableEntityHandler pointHandler,
-                                          final boolean processSystemMetrics,
-                                          final boolean processServiceChecks,
-                                          @Nullable final HttpClient requestRelayClient,
-                                          @Nullable final String requestRelayTarget,
-                                          @Nullable final ReportableEntityPreprocessor preprocessor) {
-    super(TokenAuthenticatorBuilder.create().setTokenValidationMethod(TokenValidationMethod.NONE).build(), handle,
-        false, true);
+  protected DataDogPortUnificationHandler(
+      final String handle,
+      final HealthCheckManager healthCheckManager,
+      final ReportableEntityHandler pointHandler,
+      final int fanout,
+      final boolean synchronousMode,
+      final boolean processSystemMetrics,
+      final boolean processServiceChecks,
+      @Nullable final HttpClient requestRelayClient,
+      @Nullable final String requestRelayTarget,
+      @Nullable final Supplier preprocessor) {
+    super(TokenAuthenticatorBuilder.create().build(), healthCheckManager, handle);
     this.pointHandler = pointHandler;
+    this.threadpool = new ScheduledThreadPoolExecutor(fanout, new NamedThreadFactory("dd-relay"));
+    this.synchronousMode = synchronousMode;
     this.processSystemMetrics = processSystemMetrics;
     this.processServiceChecks = processServiceChecks;
     this.requestRelayClient = requestRelayClient;
     this.requestRelayTarget = requestRelayTarget;
-    this.preprocessor = preprocessor;
+    this.preprocessorSupplier = preprocessor;
     this.jsonParser = new ObjectMapper();
-    this.httpRequestSize = Metrics.newHistogram(new TaggedMetricName("listeners", "http-requests.payload-points",
-        "port", handle));
-
-    Metrics.newGauge(new TaggedMetricName("listeners", "tags-cache-size",
-        "port", handle), new Gauge() {
-      @Override
-      public Long value() {
-        return tagsCache.estimatedSize();
-      }
-    });
-
+    this.httpRequestSize =
+        Metrics.newHistogram(
+            new TaggedMetricName("listeners", "http-requests.payload-points", "port", handle));
+    this.httpStatusCounterCache =
+        Caffeine.newBuilder()
+            .build(
+                status ->
+                    Metrics.newCounter(
+                        new TaggedMetricName(
+                            "listeners",
+                            "http-relay.status." + status + ".count",
+                            "port",
+                            handle)));
+    Metrics.newGauge(
+        new TaggedMetricName("listeners", "tags-cache-size", "port", handle),
+        new Gauge() {
+          @Override
+          public Long value() {
+            return tagsCache.estimatedSize();
+          }
+        });
+    Metrics.newGauge(
+        new TaggedMetricName("listeners", "http-relay.threadpool.queue-size", "port", handle),
+        new Gauge() {
+          @Override
+          public Integer value() {
+            return threadpool.getQueue().size();
+          }
+        });
   }
 
   @Override
-  protected void handleHttpMessage(final ChannelHandlerContext ctx,
-                                   final FullHttpRequest incomingRequest) {
+  protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request)
+      throws URISyntaxException {
     StringBuilder output = new StringBuilder();
     AtomicInteger pointsPerRequest = new AtomicInteger();
-
-    URI uri = parseUri(ctx, incomingRequest);
-    if (uri == null) return;
-
+    URI uri = new URI(request.uri());
     HttpResponseStatus status = HttpResponseStatus.ACCEPTED;
-    String requestBody = incomingRequest.content().toString(CharsetUtil.UTF_8);
-
-    if (requestRelayClient != null && requestRelayTarget != null && incomingRequest.method() == POST) {
-      Histogram requestRelayDuration = Metrics.newHistogram(new TaggedMetricName("listeners",
-          "http-relay.duration-nanos", "port", handle));
-      Long startNanos = System.nanoTime();
+    byte[] bodyBytes = new byte[request.content().readableBytes()];
+    request.content().readBytes(bodyBytes);
+
+    if (requestRelayClient != null && requestRelayTarget != null && request.method() == POST) {
+      Histogram requestRelayDuration =
+          Metrics.newHistogram(
+              new TaggedMetricName("listeners", "http-relay.duration-nanos", "port", handle));
+      long startNanos = System.nanoTime();
       try {
-        String outgoingUrl = requestRelayTarget.replaceFirst("/*$", "") + incomingRequest.uri();
+        String outgoingUrl = requestRelayTarget.replaceFirst("/*$", "") + request.uri();
         HttpPost outgoingRequest = new HttpPost(outgoingUrl);
-        if (incomingRequest.headers().contains("Content-Type")) {
-          outgoingRequest.addHeader("Content-Type", incomingRequest.headers().get("Content-Type"));
-        }
-        outgoingRequest.setEntity(new StringEntity(requestBody));
-        logger.info("Relaying incoming HTTP request to " + outgoingUrl);
-        HttpResponse response = requestRelayClient.execute(outgoingRequest);
-        int httpStatusCode = response.getStatusLine().getStatusCode();
-        Metrics.newCounter(new TaggedMetricName("listeners", "http-relay.status." + httpStatusCode + ".count",
-            "port", handle)).inc();
-
-        if (httpStatusCode < 200 || httpStatusCode >= 300) {
-          // anything that is not 2xx is relayed as is to the client, don't process the payload
-          writeHttpResponse(ctx, HttpResponseStatus.valueOf(httpStatusCode),
-              EntityUtils.toString(response.getEntity(), "UTF-8"), incomingRequest);
-          return;
-        }
-        EntityUtils.consumeQuietly(response.getEntity());
 
+        request
+            .headers()
+            .forEach(
+                header -> {
+                  if (!header.getKey().equalsIgnoreCase("Content-Length"))
+                    outgoingRequest.addHeader(header.getKey(), header.getValue());
+                });
+
+        outgoingRequest.setEntity(new ByteArrayEntity(bodyBytes));
+        if (synchronousMode) {
+          if (logger.isLoggable(Level.FINE)) {
+            logger.fine("Relaying incoming HTTP request to " + outgoingUrl);
+          }
+          HttpResponse response = requestRelayClient.execute(outgoingRequest);
+          int httpStatusCode = response.getStatusLine().getStatusCode();
+          httpStatusCounterCache.get(httpStatusCode).inc();
+
+          if (httpStatusCode < 200 || httpStatusCode >= 300) {
+            // anything that is not 2xx is relayed as is to the client, don't process
+            // the payload
+            writeHttpResponse(
+                ctx,
+                HttpResponseStatus.valueOf(httpStatusCode),
+                EntityUtils.toString(response.getEntity(), "UTF-8"),
+                request);
+            return;
+          }
+        } else {
+          threadpool.submit(
+              () -> {
+                try {
+                  if (logger.isLoggable(Level.FINE)) {
+                    logger.fine("Relaying incoming HTTP request (async) to " + outgoingUrl);
+                  }
+                  HttpResponse response = requestRelayClient.execute(outgoingRequest);
+                  int httpStatusCode = response.getStatusLine().getStatusCode();
+                  httpStatusCounterCache.get(httpStatusCode).inc();
+                  EntityUtils.consumeQuietly(response.getEntity());
+                } catch (IOException e) {
+                  logger.log(
+                      Level.WARNING,
+                      "Unable to relay request to " + requestRelayTarget + ": " + e.getMessage(),
+                      e);
+                  Metrics.newCounter(
+                          new TaggedMetricName("listeners", "http-relay.failed", "port", handle))
+                      .inc();
+                }
+              });
+        }
       } catch (IOException e) {
         logger.warning("Unable to relay request to " + requestRelayTarget + ": " + e.getMessage());
-        Metrics.newCounter(new TaggedMetricName("listeners", "http-relay.failed",
-            "port", handle)).inc();
-        writeHttpResponse(ctx, HttpResponseStatus.BAD_GATEWAY, "Unable to relay request: " + e.getMessage(),
-            incomingRequest);
+        Metrics.newCounter(new TaggedMetricName("listeners", "http-relay.failed", "port", handle))
+            .inc();
+        writeHttpResponse(
+            ctx,
+            HttpResponseStatus.BAD_GATEWAY,
+            "Unable to relay request: " + e.getMessage(),
+            request);
         return;
       } finally {
         requestRelayDuration.update(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos));
@@ -177,118 +294,175 @@ protected void handleHttpMessage(final ChannelHandlerContext ctx,
 
     String path = uri.getPath().endsWith("/") ? uri.getPath() : uri.getPath() + "/";
     switch (path) {
+      case "/api/v2/series/": // Check doc's on the beginning of this file
+        try {
+          AgentPayload.MetricPayload obj = AgentPayload.MetricPayload.parseFrom(bodyBytes);
+          reportMetrics(obj, pointsPerRequest, output::append);
+        } catch (IOException e) {
+          writeHttpResponse(ctx, HttpResponseStatus.BAD_REQUEST, output, request);
+          throw new RuntimeException(e);
+        }
+        writeHttpResponse(ctx, status, output, request);
+        break;
       case "/api/v1/series/":
         try {
-          if (!reportMetrics(jsonParser.readTree(requestBody), pointsPerRequest)) {
-            status = HttpResponseStatus.BAD_REQUEST;
-            output.append("At least one data point had error.");
-          }
+          String requestBody = new String(bodyBytes, StandardCharsets.UTF_8);
+          status =
+              reportMetrics(jsonParser.readTree(requestBody), pointsPerRequest, output::append);
         } catch (Exception e) {
           status = HttpResponseStatus.BAD_REQUEST;
-          writeExceptionText(e, output);
+          output.append(errorMessageWithRootCause(e));
           logWarning("WF-300: Failed to handle /api/v1/series request", e, ctx);
         }
         httpRequestSize.update(pointsPerRequest.intValue());
-        writeHttpResponse(ctx, status, output, incomingRequest);
+        writeHttpResponse(ctx, status, output, request);
         break;
 
       case "/api/v1/check_run/":
         if (!processServiceChecks) {
-          Metrics.newCounter(new TaggedMetricName("listeners", "http-requests.ignored", "port", handle)).inc();
-          writeHttpResponse(ctx, HttpResponseStatus.ACCEPTED, output, incomingRequest);
+          Metrics.newCounter(
+                  new TaggedMetricName("listeners", "http-requests.ignored", "port", handle))
+              .inc();
+          writeHttpResponse(ctx, HttpResponseStatus.ACCEPTED, output, request);
           return;
         }
         try {
-          if (!reportChecks(jsonParser.readTree(requestBody), pointsPerRequest)) {
-            output.append("One or more checks were not valid.");
-          }
+          String requestBody = new String(bodyBytes, StandardCharsets.UTF_8);
+          reportChecks(jsonParser.readTree(requestBody), pointsPerRequest, output::append);
         } catch (Exception e) {
           status = HttpResponseStatus.BAD_REQUEST;
-          writeExceptionText(e, output);
+          output.append(errorMessageWithRootCause(e));
           logWarning("WF-300: Failed to handle /api/v1/check_run request", e, ctx);
         }
-        writeHttpResponse(ctx, status, output, incomingRequest);
+        writeHttpResponse(ctx, status, output, request);
         break;
 
       case "/api/v1/validate/":
-        writeHttpResponse(ctx, HttpResponseStatus.OK, output, incomingRequest);
+        writeHttpResponse(ctx, HttpResponseStatus.OK, output, request);
         break;
 
       case "/intake/":
-        if (!processSystemMetrics) {
-          Metrics.newCounter(new TaggedMetricName("listeners", "http-requests.ignored", "port", handle)).inc();
-          writeHttpResponse(ctx, HttpResponseStatus.ACCEPTED, output, incomingRequest);
-          return;
-        }
         try {
-          if (!reportSystemMetrics(jsonParser.readTree(requestBody), pointsPerRequest)) {
-            output.append("At least one data point had error.");
-          }
+          String requestBody = new String(bodyBytes, StandardCharsets.UTF_8);
+          status =
+              processMetadataAndSystemMetrics(
+                  jsonParser.readTree(requestBody),
+                  processSystemMetrics,
+                  pointsPerRequest,
+                  output::append);
         } catch (Exception e) {
           status = HttpResponseStatus.BAD_REQUEST;
-          writeExceptionText(e, output);
+          output.append(errorMessageWithRootCause(e));
           logWarning("WF-300: Failed to handle /intake request", e, ctx);
         }
         httpRequestSize.update(pointsPerRequest.intValue());
-        writeHttpResponse(ctx, status, output, incomingRequest);
+        writeHttpResponse(ctx, status, output, request);
         break;
 
       default:
-        writeHttpResponse(ctx, HttpResponseStatus.NO_CONTENT, output, incomingRequest);
-        logWarning("WF-300: Unexpected path '" + incomingRequest.uri() + "', returning HTTP 204", null, ctx);
+        writeHttpResponse(ctx, HttpResponseStatus.NO_CONTENT, output, request);
+        logWarning(
+            "WF-300: Unexpected path '" + request.uri() + "', returning HTTP 204", null, ctx);
         break;
     }
   }
 
-  /**
-   * Handles an incoming plain text (string) message. Handles :
-   */
-  @Override
-  protected void handlePlainTextMessage(final ChannelHandlerContext ctx,
-                                        final String message) throws Exception {
-    if (message == null) {
-      throw new IllegalArgumentException("Message cannot be null");
-    }
-    final ObjectMapper jsonTree = new ObjectMapper();
-    try {
-      reportMetrics(jsonTree.readTree(message), null);
-    } catch (Exception e) {
-      logWarning("WF-300: Unable to parse JSON on plaintext port", e, ctx);
+  private HttpResponseStatus reportMetrics(
+      AgentPayload.MetricPayload payload,
+      AtomicInteger pointCounter,
+      Consumer outputConsumer) {
+    HttpResponseStatus worstStatus = HttpResponseStatus.ACCEPTED;
+    for (final AgentPayload.MetricPayload.MetricSeries metric : payload.getSeriesList()) {
+      HttpResponseStatus latestStatus = reportMetric(metric, pointCounter, outputConsumer);
+      if (latestStatus.compareTo(worstStatus) > 0) {
+        worstStatus = latestStatus;
+      }
     }
+    return worstStatus;
   }
 
-  @Override
-  protected void processLine(final ChannelHandlerContext ctx, final String message) {
-    throw new UnsupportedOperationException("Invalid context for processLine");
+  private HttpResponseStatus reportMetric(
+      AgentPayload.MetricPayload.MetricSeries metric,
+      AtomicInteger pointCounter,
+      Consumer outputConsumer) {
+    if (metric == null) {
+      error("Skipping - series object null.", outputConsumer);
+      return HttpResponseStatus.BAD_REQUEST;
+    }
+    try {
+      Map tags = new HashMap<>();
+      String metricName = INVALID_METRIC_CHARACTERS.matcher(metric.getMetric()).replaceAll("_");
+      String hostName = "unknown";
+      for (AgentPayload.MetricPayload.Resource resource : metric.getResourcesList()) {
+        if (resource.getType().equalsIgnoreCase("host")) {
+          hostName = resource.getName();
+        } else if (resource.getType().equalsIgnoreCase("device")) {
+          tags.put("device", resource.getName());
+        }
+      }
+      Map systemTags;
+      if ((systemTags = tagsCache.getIfPresent(hostName)) != null) {
+        tags.putAll(systemTags);
+      }
+      metric.getTagsList().stream().forEach(tag -> extractTag(tag, tags));
+
+      int interval = 1;
+      if (metric.getTypeValue() == AgentPayload.MetricPayload.MetricType.RATE_VALUE) {
+        interval = Math.toIntExact(metric.getInterval());
+      }
+
+      for (AgentPayload.MetricPayload.MetricPoint point : metric.getPointsList()) {
+        reportValue(
+            metricName,
+            hostName,
+            tags,
+            point.getValue(),
+            point.getTimestamp() * 1000,
+            pointCounter,
+            interval);
+      }
+      return HttpResponseStatus.ACCEPTED;
+    } catch (final Exception e) {
+      logger.log(Level.WARNING, "Failed to add metric", e);
+      outputConsumer.accept("Failed to add metric");
+      return HttpResponseStatus.BAD_REQUEST;
+    }
   }
 
   /**
-   * Parse the metrics JSON and report the metrics found.  There are 2 formats supported: - array of points - single
-   * point
+   * Parse the metrics JSON and report the metrics found. There are 2 formats supported: array of
+   * points and single point
    *
    * @param metrics a DataDog-format payload
    * @param pointCounter counter to track the number of points processed in one request
-   *
-   * @return true if all metrics added successfully; false o/w
-   * @see #reportMetric(JsonNode, AtomicInteger)
+   * @return final HTTP status code to return to the client
+   * @see #reportMetric(JsonNode, AtomicInteger, Consumer)
    */
-  private boolean reportMetrics(final JsonNode metrics, @Nullable final AtomicInteger pointCounter) {
-    if (metrics == null || !metrics.isObject() || !metrics.has("series")) {
-      pointHandler.reject((ReportPoint) null, "WF-300: Payload missing 'series' field");
-      return false;
+  private HttpResponseStatus reportMetrics(
+      final JsonNode metrics,
+      @Nullable final AtomicInteger pointCounter,
+      Consumer outputConsumer) {
+    if (metrics == null || !metrics.isObject()) {
+      error("Empty or malformed /api/v1/series payload - ignoring", outputConsumer);
+      return HttpResponseStatus.BAD_REQUEST;
+    }
+    if (!metrics.has("series")) {
+      error("/api/v1/series payload missing 'series' field", outputConsumer);
+      return HttpResponseStatus.BAD_REQUEST;
     }
     JsonNode series = metrics.get("series");
     if (!series.isArray()) {
-      pointHandler.reject((ReportPoint) null, "WF-300: 'series' field must be an array");
-      return false;
+      error("'series' field must be an array", outputConsumer);
+      return HttpResponseStatus.BAD_REQUEST;
     }
-    boolean successful = true;
+    HttpResponseStatus worstStatus = HttpResponseStatus.ACCEPTED;
     for (final JsonNode metric : series) {
-      if (!reportMetric(metric, pointCounter)) {
-        successful = false;
+      HttpResponseStatus latestStatus = reportMetric(metric, pointCounter, outputConsumer);
+      if (latestStatus.compareTo(worstStatus) > 0) {
+        worstStatus = latestStatus;
       }
     }
-    return successful;
+    return worstStatus;
   }
 
   /**
@@ -296,21 +470,25 @@ private boolean reportMetrics(final JsonNode metrics, @Nullable final AtomicInte
    *
    * @param metric the JSON object representing a single metric
    * @param pointCounter counter to track the number of points processed in one request
-   *
    * @return True if the metric was reported successfully; False o/w
    */
-  private boolean reportMetric(final JsonNode metric, @Nullable final AtomicInteger pointCounter) {
+  private HttpResponseStatus reportMetric(
+      final JsonNode metric,
+      @Nullable final AtomicInteger pointCounter,
+      Consumer outputConsumer) {
     if (metric == null) {
-      pointHandler.reject((ReportPoint) null, "Skipping - series object null.");
-      return false;
+      error("Skipping - series object null.", outputConsumer);
+      return HttpResponseStatus.BAD_REQUEST;
     }
     try {
-      if (metric.get("metric") == null ) {
-        pointHandler.reject((ReportPoint) null, "Skipping - 'metric' field missing.");
-        return false;
+      if (metric.get("metric") == null) {
+        error("Skipping - 'metric' field missing.", outputConsumer);
+        return HttpResponseStatus.BAD_REQUEST;
       }
-      String metricName = INVALID_METRIC_CHARACTERS.matcher(metric.get("metric").textValue()).replaceAll("_");
-      String hostName = metric.get("host") == null ? "unknown" : metric.get("host").textValue().toLowerCase();
+      String metricName =
+          INVALID_METRIC_CHARACTERS.matcher(metric.get("metric").textValue()).replaceAll("_");
+      String hostName =
+          metric.get("host") == null ? "unknown" : metric.get("host").textValue().toLowerCase();
       JsonNode tagsNode = metric.get("tags");
       Map systemTags;
       Map tags = new HashMap<>();
@@ -318,56 +496,86 @@ private boolean reportMetric(final JsonNode metric, @Nullable final AtomicIntege
         tags.putAll(systemTags);
       }
       extractTags(tagsNode, tags); // tags sent with the data override system host-level tags
+      // Include a device= tag on the data if that property exists
+      JsonNode deviceNode = metric.get("device");
+      if (deviceNode != null) {
+        tags.put("device", deviceNode.textValue());
+      }
+      // If the metric is of type rate its value needs to be multiplied by the specified
+      // interval
+      int interval = 1;
+      JsonNode type = metric.get("type");
+      if (type != null) {
+        if (type.textValue().equals("rate")) {
+          JsonNode jsonInterval = metric.get("interval");
+          if (jsonInterval != null && jsonInterval.isNumber()) {
+            interval = jsonInterval.intValue();
+          }
+        }
+      }
       JsonNode pointsNode = metric.get("points");
       if (pointsNode == null) {
-        pointHandler.reject((ReportPoint) null, "Skipping - 'points' field missing.");
-        return false;
+        error("Skipping - 'points' field missing.", outputConsumer);
+        return HttpResponseStatus.BAD_REQUEST;
       }
       for (JsonNode node : pointsNode) {
         if (node.size() == 2) {
-          reportValue(metricName, hostName, tags, node.get(1), node.get(0).longValue() * 1000, pointCounter);
+          reportValue(
+              metricName,
+              hostName,
+              tags,
+              node.get(1),
+              node.get(0).longValue() * 1000,
+              pointCounter,
+              interval);
         } else {
-          pointHandler.reject((ReportPoint) null, "WF-300: Inconsistent point value size (expected: 2)");
+          error("Inconsistent point value size (expected: 2)", outputConsumer);
         }
       }
-      return true;
+      return HttpResponseStatus.ACCEPTED;
     } catch (final Exception e) {
-      logger.log(Level.WARNING, "WF-300: Failed to add metric", e);
-      return false;
+      logger.log(Level.WARNING, "Failed to add metric", e);
+      outputConsumer.accept("Failed to add metric");
+      return HttpResponseStatus.BAD_REQUEST;
     }
   }
 
-  private boolean reportChecks(final JsonNode checkNode, @Nullable final AtomicInteger pointCounter) {
+  private void reportChecks(
+      final JsonNode checkNode,
+      @Nullable final AtomicInteger pointCounter,
+      Consumer outputConsumer) {
     if (checkNode == null) {
-      pointHandler.reject((ReportPoint) null, "Skipping - check object is null.");
-      return false;
+      error("Empty or malformed /api/v1/check_run payload - ignoring", outputConsumer);
+      return;
     }
     if (checkNode.isArray()) {
-      boolean result = true;
       for (JsonNode check : checkNode) {
-        result &= reportCheck(check, pointCounter);
+        reportCheck(check, pointCounter, outputConsumer);
       }
-      return result;
     } else {
-      return reportCheck(checkNode, pointCounter);
+      reportCheck(checkNode, pointCounter, outputConsumer);
     }
   }
 
-  private boolean reportCheck(final JsonNode check, @Nullable final AtomicInteger pointCounter) {
+  private void reportCheck(
+      final JsonNode check,
+      @Nullable final AtomicInteger pointCounter,
+      Consumer outputConsumer) {
     try {
-      if (check.get("check") == null ) {
-        pointHandler.reject((ReportPoint) null, "Skipping - 'check' field missing.");
-        return false;
+      if (check.get("check") == null) {
+        error("Skipping - 'check' field missing.", outputConsumer);
+        return;
       }
-      if (check.get("host_name") == null ) {
-        pointHandler.reject((ReportPoint) null, "Skipping - 'host_name' field missing.");
-        return false;
+      if (check.get("host_name") == null) {
+        error("Skipping - 'host_name' field missing.", outputConsumer);
+        return;
       }
-      if (check.get("status") == null ) {
+      if (check.get("status") == null) {
         // ignore - there is no status to update
-        return true;
+        return;
       }
-      String metricName = INVALID_METRIC_CHARACTERS.matcher(check.get("check").textValue()).replaceAll("_");
+      String metricName =
+          INVALID_METRIC_CHARACTERS.matcher(check.get("check").textValue()).replaceAll("_");
       String hostName = check.get("host_name").textValue().toLowerCase();
       JsonNode tagsNode = check.get("tags");
       Map systemTags;
@@ -377,31 +585,38 @@ private boolean reportCheck(final JsonNode check, @Nullable final AtomicInteger
       }
       extractTags(tagsNode, tags); // tags sent with the data override system host-level tags
 
-      long timestamp = check.get("timestamp") == null ? Clock.now() : check.get("timestamp").asLong() * 1000;
+      long timestamp =
+          check.get("timestamp") == null ? Clock.now() : check.get("timestamp").asLong() * 1000;
       reportValue(metricName, hostName, tags, check.get("status"), timestamp, pointCounter);
-      return true;
     } catch (final Exception e) {
       logger.log(Level.WARNING, "WF-300: Failed to add metric", e);
-      return false;
     }
   }
 
-
-  private boolean reportSystemMetrics(final JsonNode metrics, @Nullable final AtomicInteger pointCounter) {
-    if (metrics == null || !metrics.isObject() || !metrics.has("collection_timestamp")) {
-      pointHandler.reject((ReportPoint) null, "WF-300: Payload missing 'collection_timestamp' field");
-      return false;
+  private HttpResponseStatus processMetadataAndSystemMetrics(
+      final JsonNode metrics,
+      boolean reportSystemMetrics,
+      @Nullable final AtomicInteger pointCounter,
+      Consumer outputConsumer) {
+    if (metrics == null || !metrics.isObject()) {
+      error("Empty or malformed /intake payload", outputConsumer);
+      return HttpResponseStatus.BAD_REQUEST;
     }
-    long timestamp = metrics.get("collection_timestamp").asLong() * 1000;
     if (!metrics.has("internalHostname")) {
-      pointHandler.reject((ReportPoint) null, "WF-300: Payload missing 'internalHostname' field");
-      return false;
+      error("Payload missing 'internalHostname' field, ignoring", outputConsumer);
+      return HttpResponseStatus.ACCEPTED;
     }
+
+    // Some /api/v1/intake requests only contain host-tag metadata so process it first
     String hostName = metrics.get("internalHostname").textValue().toLowerCase();
-    Map systemTags = new HashMap<>();
+    HashMap systemTags = new HashMap<>();
     if (metrics.has("host-tags") && metrics.get("host-tags").get("system") != null) {
       extractTags(metrics.get("host-tags").get("system"), systemTags);
-      tagsCache.put(hostName, systemTags); // cache even if map is empty so we know how many unique hosts report metrics.
+      // cache even if map is empty so we know how many unique hosts report metrics.
+      tagsCache.put(hostName, systemTags);
+      if (logger.isLoggable(Level.FINE)) {
+        logger.fine("Cached system tags for " + hostName + ": " + systemTags.toString());
+      }
     } else {
       Map cachedTags = tagsCache.getIfPresent(hostName);
       if (cachedTags != null) {
@@ -410,57 +625,94 @@ private boolean reportSystemMetrics(final JsonNode metrics, @Nullable final Atom
       }
     }
 
-    // Report "system.io." metrics
-    JsonNode ioStats = metrics.get("ioStats");
-    if (ioStats != null && ioStats.isObject()) {
-      ioStats.fields().forEachRemaining(entry -> {
-        Map deviceTags = ImmutableMap.builder().
-            putAll(systemTags).
-            put("device", entry.getKey()).
-            build();
-        if (entry.getValue() != null && entry.getValue().isObject()) {
-          entry.getValue().fields().forEachRemaining(metricEntry -> {
-            String metric = "system.io." + metricEntry.getKey().replace('%', ' ').replace('/', '_').trim();
-            reportValue(metric, hostName, deviceTags, metricEntry.getValue(), timestamp, pointCounter);
-          });
-        }
-      });
+    if (!reportSystemMetrics) {
+      Metrics.newCounter(new TaggedMetricName("listeners", "http-requests.ignored", "port", handle))
+          .inc();
+      return HttpResponseStatus.ACCEPTED;
     }
 
-    // Report all metrics that already start with "system."
-    metrics.fields().forEachRemaining(entry -> {
-      if (entry.getKey().startsWith("system.")) {
-        reportValue(entry.getKey(), hostName, systemTags, entry.getValue(), timestamp, pointCounter);
+    if (metrics.has("collection_timestamp")) {
+      long timestamp = metrics.get("collection_timestamp").asLong() * 1000;
+
+      // Report "system.io." metrics
+      JsonNode ioStats = metrics.get("ioStats");
+      if (ioStats != null && ioStats.isObject()) {
+        ioStats
+            .fields()
+            .forEachRemaining(
+                entry -> {
+                  Map deviceTags =
+                      ImmutableMap.builder()
+                          .putAll(systemTags)
+                          .put("device", entry.getKey())
+                          .build();
+                  if (entry.getValue() != null && entry.getValue().isObject()) {
+                    entry
+                        .getValue()
+                        .fields()
+                        .forEachRemaining(
+                            metricEntry -> {
+                              String metric =
+                                  "system.io."
+                                      + metricEntry
+                                          .getKey()
+                                          .replace('%', ' ')
+                                          .replace('/', '_')
+                                          .trim();
+                              reportValue(
+                                  metric,
+                                  hostName,
+                                  deviceTags,
+                                  metricEntry.getValue(),
+                                  timestamp,
+                                  pointCounter);
+                            });
+                  }
+                });
       }
-    });
-
-    // Report CPU and memory metrics
-    reportValue("system.cpu.guest", hostName, systemTags, metrics.get("cpuGuest"), timestamp, pointCounter);
-    reportValue("system.cpu.idle", hostName, systemTags, metrics.get("cpuIdle"), timestamp, pointCounter);
-    reportValue("system.cpu.stolen", hostName, systemTags, metrics.get("cpuStolen"), timestamp, pointCounter);
-    reportValue("system.cpu.system", hostName, systemTags, metrics.get("cpuSystem"), timestamp, pointCounter);
-    reportValue("system.cpu.user", hostName, systemTags, metrics.get("cpuUser"), timestamp, pointCounter);
-    reportValue("system.cpu.wait", hostName, systemTags, metrics.get("cpuWait"), timestamp, pointCounter);
-    reportValue("system.mem.buffers", hostName, systemTags, metrics.get("memBuffers"), timestamp, pointCounter);
-    reportValue("system.mem.cached", hostName, systemTags, metrics.get("memCached"), timestamp, pointCounter);
-    reportValue("system.mem.page_tables", hostName, systemTags, metrics.get("memPageTables"), timestamp, pointCounter);
-    reportValue("system.mem.shared", hostName, systemTags, metrics.get("memShared"), timestamp, pointCounter);
-    reportValue("system.mem.slab", hostName, systemTags, metrics.get("memSlab"), timestamp, pointCounter);
-    reportValue("system.mem.free", hostName, systemTags, metrics.get("memPhysFree"), timestamp, pointCounter);
-    reportValue("system.mem.pct_usable", hostName, systemTags, metrics.get("memPhysPctUsable"), timestamp, pointCounter);
-    reportValue("system.mem.total", hostName, systemTags, metrics.get("memPhysTotal"), timestamp, pointCounter);
-    reportValue("system.mem.usable", hostName, systemTags, metrics.get("memPhysUsable"), timestamp, pointCounter);
-    reportValue("system.mem.used", hostName, systemTags, metrics.get("memPhysUsed"), timestamp, pointCounter);
-    reportValue("system.swap.cached", hostName, systemTags, metrics.get("memSwapCached"), timestamp, pointCounter);
-    reportValue("system.swap.free", hostName, systemTags, metrics.get("memSwapFree"), timestamp, pointCounter);
-    reportValue("system.swap.pct_free", hostName, systemTags, metrics.get("memSwapPctFree"), timestamp, pointCounter);
-    reportValue("system.swap.total", hostName, systemTags, metrics.get("memSwapTotal"), timestamp, pointCounter);
-    reportValue("system.swap.used", hostName, systemTags, metrics.get("memSwapUsed"), timestamp, pointCounter);
-    return true;
+
+      // Report all metrics that already start with "system."
+      metrics
+          .fields()
+          .forEachRemaining(
+              entry -> {
+                if (entry.getKey().startsWith("system.")) {
+                  reportValue(
+                      entry.getKey(),
+                      hostName,
+                      systemTags,
+                      entry.getValue(),
+                      timestamp,
+                      pointCounter);
+                }
+              });
+
+      // Report CPU and memory metrics
+      SYSTEM_METRICS.forEach(
+          (key, value) ->
+              reportValue(key, hostName, systemTags, metrics.get(value), timestamp, pointCounter));
+    }
+    return HttpResponseStatus.ACCEPTED;
+  }
+
+  private void reportValue(
+      String metricName,
+      String hostName,
+      Map tags,
+      JsonNode valueNode,
+      long timestamp,
+      AtomicInteger pointCounter) {
+    reportValue(metricName, hostName, tags, valueNode, timestamp, pointCounter, 1);
   }
 
-  private void reportValue(String metricName, String hostName, Map tags, JsonNode valueNode,
-                              long timestamp, AtomicInteger pointCounter) {
+  private void reportValue(
+      String metricName,
+      String hostName,
+      Map tags,
+      JsonNode valueNode,
+      long timestamp,
+      AtomicInteger pointCounter,
+      int interval) {
     if (valueNode == null || valueNode.isNull()) return;
     double value;
     if (valueNode.isTextual()) {
@@ -476,27 +728,45 @@ private void reportValue(String metricName, String hostName, Map
     } else {
       value = valueNode.asLong();
     }
+    reportValue(metricName, hostName, tags, value, timestamp, pointCounter, interval);
+  }
 
-    ReportPoint point = ReportPoint.newBuilder().
-        setTable("dummy").
-        setMetric(metricName).
-        setHost(hostName).
-        setTimestamp(timestamp).
-        setAnnotations(tags).
-        setValue(value).
-        build();
+  private void reportValue(
+      String metricName,
+      String hostName,
+      Map tags,
+      double value,
+      long timestamp,
+      AtomicInteger pointCounter,
+      int interval) {
+
+    // interval will normally be 1 unless the metric was a rate type with a specified interval
+    value = value * interval;
+
+    ReportPoint point =
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric(metricName)
+            .setHost(hostName)
+            .setTimestamp(timestamp)
+            .setAnnotations(tags)
+            .setValue(value)
+            .build();
     if (pointCounter != null) {
       pointCounter.incrementAndGet();
     }
-    if (preprocessor != null) {
+    if (preprocessorSupplier != null) {
+      ReportableEntityPreprocessor preprocessor = preprocessorSupplier.get();
+      String[] messageHolder = new String[1];
       preprocessor.forReportPoint().transform(point);
-      if (!preprocessor.forReportPoint().filter(point)) {
-        if (preprocessor.forReportPoint().getLastFilterResult() != null) {
-          blockedPointsLogger.warning(PointHandlerImpl.pointToString(point));
+      if (!preprocessor.forReportPoint().filter(point, messageHolder)) {
+        if (messageHolder[0] != null) {
+          blockedPointsLogger.warning(ReportPointSerializer.pointToString(point));
+          pointHandler.reject(point, messageHolder[0]);
         } else {
-          blockedPointsLogger.info(PointHandlerImpl.pointToString(point));
+          blockedPointsLogger.info(ReportPointSerializer.pointToString(point));
+          pointHandler.block(point);
         }
-        pointHandler.reject(point, preprocessor.forReportPoint().getLastFilterResult());
         return;
       }
     }
@@ -523,10 +793,17 @@ private void extractTag(String input, final Map tags) {
     if (tagKvIndex > 0) { // first character can't be ':' either
       String tagK = input.substring(0, tagKvIndex);
       if (tagK.toLowerCase().equals("source")) {
-        tags.put("_source", input.substring(tagKvIndex + 1, input.length()));
+        tags.put("_source", input.substring(tagKvIndex + 1));
       } else {
-        tags.put(INVALID_TAG_CHARACTERS.matcher(tagK).replaceAll("_"), input.substring(tagKvIndex + 1, input.length()));
+        tags.put(
+            INVALID_TAG_CHARACTERS.matcher(tagK).replaceAll("_"), input.substring(tagKvIndex + 1));
       }
     }
   }
+
+  private void error(String msg, Consumer outputConsumer) {
+    pointHandler.reject((ReportPoint) null, msg);
+    outputConsumer.accept(msg);
+    outputConsumer.accept("\n");
+  }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/FeatureCheckUtils.java b/proxy/src/main/java/com/wavefront/agent/listeners/FeatureCheckUtils.java
new file mode 100644
index 000000000..33ebbe8fd
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/FeatureCheckUtils.java
@@ -0,0 +1,165 @@
+package com.wavefront.agent.listeners;
+
+import com.wavefront.common.logger.MessageDedupingLogger;
+import com.yammer.metrics.core.Counter;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.util.CharsetUtil;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Constants and utility methods for validating feature subscriptions.
+ *
+ * @author vasily@wavefront.com
+ */
+public abstract class FeatureCheckUtils {
+  private static final Logger logger = Logger.getLogger(FeatureCheckUtils.class.getCanonicalName());
+
+  private static final Logger featureDisabledLogger = new MessageDedupingLogger(logger, 3, 0.2);
+  public static final String HISTO_DISABLED =
+      "Ingested point discarded because histogram "
+          + "feature has not been enabled for your account";
+  public static final String SPAN_DISABLED =
+      "Ingested span discarded because distributed "
+          + "tracing feature has not been enabled for your account.";
+  public static final String SPANLOGS_DISABLED =
+      "Ingested span log discarded because "
+          + "this feature has not been enabled for your account.";
+  public static final String LOGS_DISABLED =
+      "Ingested logs discarded because " + "this feature has not been enabled for your account.";
+
+  public static final String LOGS_SERVER_DETAILS_MISSING =
+      "Ingested logs discarded because the "
+          + "configuration is missing either "
+          + "logServerIngestionToken/logServerIngestionURL or both.";
+
+  /**
+   * Check whether feature disabled flag is set, log a warning message, increment the counter by 1.
+   *
+   * @param featureDisabledFlag Supplier for feature disabled flag.
+   * @param message Warning message to log if feature is disabled.
+   * @param discardedCounter Optional counter for discarded items.
+   * @return true if feature is disabled
+   */
+  public static boolean isFeatureDisabled(
+      Supplier featureDisabledFlag, String message, @Nullable Counter discardedCounter) {
+    return isFeatureDisabled(featureDisabledFlag, message, discardedCounter, null, null);
+  }
+
+  /**
+   * Check whether feature disabled flag is set, log a warning message, increment the counter by 1.
+   *
+   * @param featureDisabledFlag Supplier for feature disabled flag.
+   * @param message Warning message to log if feature is disabled.
+   * @param discardedCounter Optional counter for discarded items.
+   * @param output Optional stringbuilder for messages
+   * @return true if feature is disabled
+   */
+  public static boolean isFeatureDisabled(
+      Supplier featureDisabledFlag,
+      String message,
+      @Nullable Counter discardedCounter,
+      @Nullable StringBuilder output) {
+    return isFeatureDisabled(featureDisabledFlag, message, discardedCounter, output, null);
+  }
+
+  /**
+   * Check whether feature disabled flag is set, log a warning message, increment the counter either
+   * by 1 or by number of \n characters in request payload, if provided.
+   *
+   * @param featureDisabledFlag Supplier for feature disabled flag.
+   * @param message Warning message to log if feature is disabled.
+   * @param discardedCounter Optional counter for discarded items.
+   * @param output Optional stringbuilder for messages
+   * @param request Optional http request to use payload size
+   * @return true if feature is disabled
+   */
+  public static boolean isFeatureDisabled(
+      Supplier featureDisabledFlag,
+      String message,
+      @Nullable Counter discardedCounter,
+      @Nullable StringBuilder output,
+      @Nullable FullHttpRequest request) {
+    return isFeatureDisabled(featureDisabledFlag, message, discardedCounter, 1, output, request);
+  }
+
+  /**
+   * Check whether feature disabled flag is set, log a warning message, increment the counter by
+   * increment.
+   *
+   * @param featureDisabledFlag Supplier for feature disabled flag.
+   * @param message Warning message to log if feature is disabled.
+   * @param discardedCounter Counter for discarded items.
+   * @param increment The amount by which the counter will be increased.
+   * @return true if feature is disabled
+   */
+  public static boolean isFeatureDisabled(
+      Supplier featureDisabledFlag,
+      String message,
+      Counter discardedCounter,
+      long increment) {
+    return isFeatureDisabled(featureDisabledFlag, message, discardedCounter, increment, null, null);
+  }
+
+  /**
+   * Check whether feature disabled flag is set, log a warning message, increment the counter either
+   * by increment or by number of \n characters in request payload, if provided.
+   *
+   * @param featureDisabledFlag Supplier for feature disabled flag.
+   * @param message Warning message to log if feature is disabled.
+   * @param discardedCounter Optional counter for discarded items.
+   * @param increment The amount by which the counter will be increased.
+   * @param output Optional stringbuilder for messages
+   * @param request Optional http request to use payload size
+   * @return true if feature is disabled
+   */
+  public static boolean isFeatureDisabled(
+      Supplier featureDisabledFlag,
+      String message,
+      @Nullable Counter discardedCounter,
+      long increment,
+      @Nullable StringBuilder output,
+      @Nullable FullHttpRequest request) {
+    if (featureDisabledFlag.get()) {
+      featureDisabledLogger.warning(message);
+      if (output != null) {
+        output.append(message);
+      }
+      if (discardedCounter != null) {
+        discardedCounter.inc(
+            request == null
+                ? increment
+                : StringUtils.countMatches(request.content().toString(CharsetUtil.UTF_8), "\n")
+                    + 1);
+      }
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Check whether log server details are missing for a converged CSP tenant.
+   *
+   * @param receivedLogServerDetails boolean that indicates availability of log server URL & token
+   * @param enableHyperlogsConvergedCsp boolean that indicates converged CSP tenant setting
+   * @param message Warning message to log if feature is disabled.
+   * @param discardedCounter Optional counter for discarded items.
+   * @return true if log server details are missing
+   */
+  public static boolean isMissingLogServerInfoForAConvergedCSPTenant(
+      boolean receivedLogServerDetails,
+      boolean enableHyperlogsConvergedCsp,
+      String message,
+      @Nullable Counter discardedCounter) {
+    if (enableHyperlogsConvergedCsp && !receivedLogServerDetails) {
+      featureDisabledLogger.warning(message);
+      if (discardedCounter != null) {
+        discardedCounter.inc();
+      }
+      return true;
+    }
+    return false;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/HttpHealthCheckEndpointHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/HttpHealthCheckEndpointHandler.java
new file mode 100644
index 000000000..117cf1948
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/HttpHealthCheckEndpointHandler.java
@@ -0,0 +1,32 @@
+package com.wavefront.agent.listeners;
+
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+
+import com.wavefront.agent.auth.TokenAuthenticatorBuilder;
+import com.wavefront.agent.channel.HealthCheckManager;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import javax.annotation.Nullable;
+
+/**
+ * A simple healthcheck-only endpoint handler. All other endpoints return a 404.
+ *
+ * @author vasily@wavefront.com
+ */
+@ChannelHandler.Sharable
+public class HttpHealthCheckEndpointHandler extends AbstractHttpOnlyHandler {
+
+  public HttpHealthCheckEndpointHandler(
+      @Nullable final HealthCheckManager healthCheckManager, int port) {
+    super(TokenAuthenticatorBuilder.create().build(), healthCheckManager, String.valueOf(port));
+  }
+
+  @Override
+  protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request) {
+    StringBuilder output = new StringBuilder();
+    HttpResponseStatus status = HttpResponseStatus.NOT_FOUND;
+    writeHttpResponse(ctx, status, output, request);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/JaegerThriftCollectorHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/JaegerThriftCollectorHandler.java
deleted file mode 100644
index 9a66c7fe3..000000000
--- a/proxy/src/main/java/com/wavefront/agent/listeners/JaegerThriftCollectorHandler.java
+++ /dev/null
@@ -1,213 +0,0 @@
-package com.wavefront.agent.listeners;
-
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.util.concurrent.RateLimiter;
-
-import com.uber.tchannel.api.handlers.ThriftRequestHandler;
-import com.uber.tchannel.messages.ThriftRequest;
-import com.uber.tchannel.messages.ThriftResponse;
-import com.wavefront.agent.handlers.HandlerKey;
-import com.wavefront.agent.handlers.ReportableEntityHandler;
-import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
-import com.wavefront.common.TraceConstants;
-import com.wavefront.data.ReportableEntityType;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-
-import io.jaegertracing.thriftjava.Batch;
-import io.jaegertracing.thriftjava.Collector;
-import io.jaegertracing.thriftjava.SpanRef;
-import io.jaegertracing.thriftjava.Tag;
-import io.jaegertracing.thriftjava.TagType;
-import wavefront.report.Annotation;
-import wavefront.report.Span;
-
-import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
-import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
-
-/**
- * Handler that processes trace data in Jaeger Thrift compact format and converts them to Wavefront format
- *
- * @author vasily@wavefront.com
- */
-public class JaegerThriftCollectorHandler extends ThriftRequestHandler {
-  protected static final Logger logger = Logger.getLogger(JaegerThriftCollectorHandler.class.getCanonicalName());
-
-  // TODO: support sampling
-  private final static Set IGNORE_TAGS = ImmutableSet.of("sampler.type", "sampler.param");
-  private final static String DEFAULT_APPLICATION = "Jaeger";
-  private static final Logger jaegerDataLogger = Logger.getLogger("JaegerDataLogger");
-
-  private final String handle;
-  private final ReportableEntityHandler handler;
-  private final AtomicBoolean traceDisabled;
-
-  // log every 5 seconds
-  private final RateLimiter warningLoggerRateLimiter = RateLimiter.create(0.2);
-
-  private final Counter discardedTraces;
-  private final Counter discardedBatches;
-  private final Counter processedBatches;
-  private final Counter failedBatches;
-
-  @SuppressWarnings("unchecked")
-  public JaegerThriftCollectorHandler(String handle, ReportableEntityHandlerFactory handlerFactory,
-                                      AtomicBoolean traceDisabled) {
-    this(handle, handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)), traceDisabled);
-  }
-
-  public JaegerThriftCollectorHandler(String handle, ReportableEntityHandler handler,
-                                      AtomicBoolean traceDisabled) {
-    this.handle = handle;
-    this.handler = handler;
-    this.traceDisabled = traceDisabled;
-    this.discardedTraces = Metrics.newCounter(new MetricName("spans." + handle, "", "discarded"));
-    this.discardedBatches = Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "discarded"));
-    this.processedBatches = Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "processed"));
-    this.failedBatches = Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "failed"));
-  }
-
-  @Override
-  public ThriftResponse handleImpl(
-      ThriftRequest request) {
-    for (Batch batch : request.getBody(Collector.submitBatches_args.class).getBatches()) {
-      try {
-        processBatch(batch);
-        processedBatches.inc();
-      } catch (Exception e) {
-        failedBatches.inc();
-        logger.log(Level.WARNING, "Jaeger Thrift batch processing failed", Throwables.getRootCause(e));
-      }
-    }
-    return new ThriftResponse.Builder(request)
-        .setBody(new Collector.submitBatches_result())
-        .build();
-  }
-
-  private void processBatch(Batch batch) {
-    String serviceName = batch.getProcess().getServiceName();
-    String sourceName = null;
-    if (batch.getProcess().getTags() != null) {
-      for (Tag tag : batch.getProcess().getTags()) {
-        if (tag.getKey().equals("hostname") && tag.getVType() == TagType.STRING) {
-          sourceName = tag.getVStr();
-          break;
-        }
-        if (tag.getKey().equals("ip") && tag.getVType()== TagType.STRING) {
-          sourceName = tag.getVStr();
-        }
-      }
-      if (sourceName == null) {
-        sourceName = "unknown";
-      }
-    }
-    if (traceDisabled.get()) {
-      if (warningLoggerRateLimiter.tryAcquire()) {
-        logger.info("Ingested spans discarded because tracing feature is not enabled on the server");
-      }
-      discardedBatches.inc();
-      discardedTraces.inc(batch.getSpansSize());
-      return;
-    }
-    for (io.jaegertracing.thriftjava.Span span : batch.getSpans()) {
-      processSpan(span, serviceName, sourceName);
-    }
-  }
-
-  private void processSpan(io.jaegertracing.thriftjava.Span span, String serviceName, String sourceName) {
-    List annotations = new ArrayList<>();
-    // serviceName is mandatory in Jaeger
-    annotations.add(new Annotation(SERVICE_TAG_KEY, serviceName));
-    long parentSpanId = span.getParentSpanId();
-    if (parentSpanId != 0) {
-      annotations.add(new Annotation("parent", new UUID(0, parentSpanId).toString()));
-    }
-
-    boolean applicationTagPresent = false;
-    if (span.getTags() != null) {
-      for (Tag tag : span.getTags()) {
-        if (applicationTagPresent || tag.getKey().equals(APPLICATION_TAG_KEY)) {
-          applicationTagPresent = true;
-        }
-        if (IGNORE_TAGS.contains(tag.getKey())) {
-          continue;
-        }
-        Annotation annotation = tagToAnnotation(tag);
-        if (annotation != null) {
-          annotations.add(annotation);
-        }
-      }
-    }
-
-    if (!applicationTagPresent) {
-      // Original Jaeger span did not have application set, will default to 'Jaeger'
-      Annotation annotation = new Annotation(APPLICATION_TAG_KEY, DEFAULT_APPLICATION);
-      annotations.add(annotation);
-    }
-
-    if (span.getReferences() != null) {
-      for (SpanRef reference : span.getReferences()) {
-        switch (reference.refType) {
-          case CHILD_OF:
-            if (reference.getSpanId() != 0 && reference.getSpanId() != parentSpanId) {
-              annotations.add(new Annotation(TraceConstants.PARENT_KEY, new UUID(0, reference.getSpanId()).toString()));
-            }
-          case FOLLOWS_FROM:
-            if (reference.getSpanId() != 0) {
-              annotations.add(new Annotation(TraceConstants.FOLLOWS_FROM_KEY, new UUID(0, reference.getSpanId())
-                  .toString()));
-            }
-          default:
-        }
-      }
-    }
-    Span newSpan = Span.newBuilder()
-        .setCustomer("dummy")
-        .setName(span.getOperationName())
-        .setSource(sourceName)
-        .setSpanId(new UUID(0, span.getSpanId()).toString())
-        .setTraceId(new UUID(span.getTraceIdHigh(), span.getTraceIdLow()).toString())
-        .setStartMillis(span.getStartTime() / 1000)
-        .setDuration(span.getDuration() / 1000)
-        .setAnnotations(annotations)
-        .build();
-
-    // Log Jaeger spans as well as Wavefront spans for debugging purposes.
-    if (jaegerDataLogger.isLoggable(Level.FINEST)) {
-      jaegerDataLogger.info("Inbound Jaeger span: " + span.toString());
-      jaegerDataLogger.info("Converted Wavefront span: " + newSpan.toString());
-    }
-
-    handler.report(newSpan);
-  }
-
-  @Nullable
-  private static Annotation tagToAnnotation(Tag tag) {
-    switch (tag.vType) {
-      case BOOL:
-        return new Annotation(tag.getKey(), String.valueOf(tag.isVBool()));
-      case LONG:
-        return new Annotation(tag.getKey(), String.valueOf(tag.getVLong()));
-      case DOUBLE:
-        return new Annotation(tag.getKey(), String.valueOf(tag.getVDouble()));
-      case STRING:
-        return new Annotation(tag.getKey(), tag.getVStr());
-      case BINARY:
-      default:
-        return null;
-    }
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/JsonMetricsEndpoint.java b/proxy/src/main/java/com/wavefront/agent/listeners/JsonMetricsEndpoint.java
deleted file mode 100644
index d2b684089..000000000
--- a/proxy/src/main/java/com/wavefront/agent/listeners/JsonMetricsEndpoint.java
+++ /dev/null
@@ -1,119 +0,0 @@
-package com.wavefront.agent.listeners;
-
-import com.google.common.collect.Maps;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.wavefront.agent.PointHandler;
-import com.wavefront.agent.PointHandlerImpl;
-import com.wavefront.agent.PostPushDataTimedTask;
-import com.wavefront.agent.preprocessor.ReportableEntityPreprocessor;
-import com.wavefront.common.Clock;
-import com.wavefront.metrics.JsonMetricsParser;
-
-import org.eclipse.jetty.server.Request;
-import org.eclipse.jetty.server.handler.AbstractHandler;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Agent-side JSON metrics endpoint.
- *
- * @author Clement Pang (clement@wavefront.com).
- */
-public class JsonMetricsEndpoint extends AbstractHandler {
-
-  private static final Logger blockedPointsLogger = Logger.getLogger("RawBlockedPoints");
-
-  @Nullable
-  private final String prefix;
-  private final String defaultHost;
-  @Nullable
-  private final ReportableEntityPreprocessor preprocessor;
-  private final PointHandler handler;
-
-  public JsonMetricsEndpoint(final String port, final String host,
-                             @Nullable
-                             final String prefix, final String validationLevel, final int blockedPointsPerBatch,
-                             PostPushDataTimedTask[] postPushDataTimedTasks,
-                             @Nullable final ReportableEntityPreprocessor preprocessor) {
-    this.handler = new PointHandlerImpl(port, validationLevel, blockedPointsPerBatch, postPushDataTimedTasks);
-    this.prefix = prefix;
-    this.defaultHost = host;
-    this.preprocessor = preprocessor;
-  }
-
-  @Override
-  public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
-      throws IOException, ServletException {
-    Map tags = Maps.newHashMap();
-    for (Enumeration parameters = request.getParameterNames(); parameters.hasMoreElements();) {
-      String tagk = parameters.nextElement().trim().toLowerCase();
-      if (tagk.equals("h") || tagk.equals("p") || tagk.equals("d") || tagk.equals("t")) {
-        continue;
-      }
-      if (request.getParameter(tagk) != null && request.getParameter(tagk).length() > 0) {
-        tags.put(tagk, request.getParameter(tagk));
-      }
-    }
-    List points = new ArrayList<>();
-    Long timestamp;
-    if (request.getParameter("d") == null) {
-      timestamp = Clock.now();
-    } else {
-      try {
-        timestamp = Long.parseLong(request.getParameter("d"));
-      } catch (NumberFormatException e) {
-        timestamp = Clock.now();
-      }
-    }
-    String prefix;
-    if (this.prefix != null) {
-      prefix = request.getParameter("p") == null ? this.prefix : this.prefix + "." + request.getParameter("p");
-    } else {
-      prefix = request.getParameter("p");
-    }
-    String host = request.getParameter("h") == null ? defaultHost : request.getParameter("h");
-
-    JsonNode metrics = new ObjectMapper().readTree(request.getReader());
-
-    JsonMetricsParser.report("dummy", prefix, metrics, points, host, timestamp);
-    for (ReportPoint point : points) {
-      if (point.getAnnotations() == null) {
-        point.setAnnotations(tags);
-      } else {
-        Map newAnnotations = Maps.newHashMap(tags);
-        newAnnotations.putAll(point.getAnnotations());
-        point.setAnnotations(newAnnotations);
-      }
-      if (preprocessor != null) {
-        if (!preprocessor.forReportPoint().filter(point)) {
-          if (preprocessor.forReportPoint().getLastFilterResult() != null) {
-            blockedPointsLogger.warning(PointHandlerImpl.pointToString(point));
-          } else {
-            blockedPointsLogger.info(PointHandlerImpl.pointToString(point));
-          }
-          handler.handleBlockedPoint(preprocessor.forReportPoint().getLastFilterResult());
-          continue;
-        }
-        preprocessor.forReportPoint().transform(point);
-      }
-      handler.reportPoint(point, "json: " + PointHandlerImpl.pointToString(point));
-    }
-    response.setContentType("text/html;charset=utf-8");
-    response.setStatus(HttpServletResponse.SC_OK);
-    baseRequest.setHandled(true);
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/JsonMetricsPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/JsonMetricsPortUnificationHandler.java
new file mode 100644
index 000000000..077b7d409
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/JsonMetricsPortUnificationHandler.java
@@ -0,0 +1,173 @@
+package com.wavefront.agent.listeners;
+
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.common.Clock;
+import com.wavefront.common.Pair;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.metrics.JsonMetricsParser;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.util.CharsetUtil;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import wavefront.report.ReportPoint;
+
+/**
+ * Agent-side JSON metrics endpoint.
+ *
+ * @author Clement Pang (clement@wavefront.com).
+ * @author vasily@wavefront.com.
+ */
+@ChannelHandler.Sharable
+public class JsonMetricsPortUnificationHandler extends AbstractHttpOnlyHandler {
+  private static final Set STANDARD_PARAMS = ImmutableSet.of("h", "p", "d", "t");
+
+  /**
+   * The point handler that takes report metrics one data point at a time and handles batching and
+   * retries, etc
+   */
+  private final ReportableEntityHandler pointHandler;
+
+  private final String prefix;
+  private final String defaultHost;
+
+  @Nullable private final Supplier preprocessorSupplier;
+  private final ObjectMapper jsonParser;
+
+  /**
+   * Create a new instance.
+   *
+   * @param handle handle/port number.
+   * @param authenticator token authenticator.
+   * @param healthCheckManager shared health check endpoint handler.
+   * @param handlerFactory factory for ReportableEntityHandler objects.
+   * @param prefix metric prefix.
+   * @param defaultHost default host name to use, if none specified.
+   * @param preprocessor preprocessor.
+   */
+  public JsonMetricsPortUnificationHandler(
+      final String handle,
+      final TokenAuthenticator authenticator,
+      final HealthCheckManager healthCheckManager,
+      final ReportableEntityHandlerFactory handlerFactory,
+      final String prefix,
+      final String defaultHost,
+      @Nullable final Supplier preprocessor) {
+    this(
+        handle,
+        authenticator,
+        healthCheckManager,
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle)),
+        prefix,
+        defaultHost,
+        preprocessor);
+  }
+
+  @VisibleForTesting
+  protected JsonMetricsPortUnificationHandler(
+      final String handle,
+      final TokenAuthenticator authenticator,
+      final HealthCheckManager healthCheckManager,
+      final ReportableEntityHandler pointHandler,
+      final String prefix,
+      final String defaultHost,
+      @Nullable final Supplier preprocessor) {
+    super(authenticator, healthCheckManager, handle);
+    this.pointHandler = pointHandler;
+    this.prefix = prefix;
+    this.defaultHost = defaultHost;
+    this.preprocessorSupplier = preprocessor;
+    this.jsonParser = new ObjectMapper();
+  }
+
+  @Override
+  protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request)
+      throws URISyntaxException {
+    StringBuilder output = new StringBuilder();
+    try {
+      URI uri = new URI(request.uri());
+      Map params =
+          Arrays.stream(uri.getRawQuery().split("&"))
+              .map(x -> new Pair<>(x.split("=")[0].trim().toLowerCase(), x.split("=")[1]))
+              .collect(Collectors.toMap(k -> k._1, v -> v._2));
+
+      String requestBody = request.content().toString(CharsetUtil.UTF_8);
+
+      Map tags = Maps.newHashMap();
+      params.entrySet().stream()
+          .filter(x -> !STANDARD_PARAMS.contains(x.getKey()) && x.getValue().length() > 0)
+          .forEach(x -> tags.put(x.getKey(), x.getValue()));
+      List points = new ArrayList<>();
+      long timestamp;
+      if (params.get("d") == null) {
+        timestamp = Clock.now();
+      } else {
+        try {
+          timestamp = Long.parseLong(params.get("d"));
+        } catch (NumberFormatException e) {
+          timestamp = Clock.now();
+        }
+      }
+      String prefix =
+          this.prefix == null
+              ? params.get("p")
+              : params.get("p") == null ? this.prefix : this.prefix + "." + params.get("p");
+      String host = params.get("h") == null ? defaultHost : params.get("h");
+
+      JsonNode metrics = jsonParser.readTree(requestBody);
+
+      ReportableEntityPreprocessor preprocessor =
+          preprocessorSupplier == null ? null : preprocessorSupplier.get();
+      String[] messageHolder = new String[1];
+      JsonMetricsParser.report("dummy", prefix, metrics, points, host, timestamp);
+      for (ReportPoint point : points) {
+        if (point.getAnnotations().isEmpty()) {
+          point.setAnnotations(tags);
+        } else {
+          Map newAnnotations = Maps.newHashMap(tags);
+          newAnnotations.putAll(point.getAnnotations());
+          point.setAnnotations(newAnnotations);
+        }
+        if (preprocessor != null) {
+          preprocessor.forReportPoint().transform(point);
+          if (!preprocessor.forReportPoint().filter(point, messageHolder)) {
+            if (messageHolder[0] != null) {
+              pointHandler.reject(point, messageHolder[0]);
+            } else {
+              pointHandler.block(point);
+            }
+            continue;
+          }
+        }
+        pointHandler.report(point);
+      }
+      writeHttpResponse(ctx, HttpResponseStatus.OK, output, request);
+    } catch (IOException e) {
+      logWarning("WF-300: Error processing incoming JSON request", e, ctx);
+      writeHttpResponse(ctx, HttpResponseStatus.INTERNAL_SERVER_ERROR, output, request);
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/OpenTSDBPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/OpenTSDBPortUnificationHandler.java
index 77edb24e0..13be226fe 100644
--- a/proxy/src/main/java/com/wavefront/agent/listeners/OpenTSDBPortUnificationHandler.java
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/OpenTSDBPortUnificationHandler.java
@@ -1,108 +1,102 @@
 package com.wavefront.agent.listeners;
 
-import com.google.common.collect.Lists;
+import static com.wavefront.agent.channel.ChannelUtils.errorMessageWithRootCause;
+import static com.wavefront.agent.channel.ChannelUtils.getRemoteAddress;
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.wavefront.agent.auth.TokenAuthenticator;
-import com.wavefront.agent.channel.CachingGraphiteHostAnnotator;
-import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.channel.HealthCheckManager;
 import com.wavefront.agent.handlers.ReportableEntityHandler;
 import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
-import com.wavefront.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
 import com.wavefront.common.Clock;
 import com.wavefront.data.ReportableEntityType;
 import com.wavefront.ingester.ReportableEntityDecoder;
 import com.wavefront.metrics.JsonMetricsParser;
-
-import java.net.URI;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.ResourceBundle;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-
 import io.netty.channel.ChannelFuture;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.util.CharsetUtil;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.ResourceBundle;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import wavefront.report.ReportPoint;
 
 /**
- * This class handles an incoming message of either String or FullHttpRequest type.  All other types are ignored. This
- * will likely be passed to the PlainTextOrHttpFrameDecoder as the handler for messages.
+ * This class handles both OpenTSDB JSON and OpenTSDB plaintext protocol.
  *
  * @author Mike McLaughlin (mike@wavefront.com)
  */
-public class OpenTSDBPortUnificationHandler extends PortUnificationHandler {
-  private static final Logger logger = Logger.getLogger(
-      OpenTSDBPortUnificationHandler.class.getCanonicalName());
-
+public class OpenTSDBPortUnificationHandler extends AbstractPortUnificationHandler {
   /**
-   * The point handler that takes report metrics one data point at a time and handles batching and retries, etc
+   * The point handler that takes report metrics one data point at a time and handles batching and
+   * retries, etc
    */
-  private final ReportableEntityHandler pointHandler;
+  private final ReportableEntityHandler pointHandler;
 
-  /**
-   * OpenTSDB decoder object
-   */
+  /** OpenTSDB decoder object */
   private final ReportableEntityDecoder decoder;
 
-  @Nullable
-  private final ReportableEntityPreprocessor preprocessor;
-
-  @Nullable
-  private final CachingGraphiteHostAnnotator annotator;
+  @Nullable private final Supplier preprocessorSupplier;
 
+  @Nullable private final Function resolver;
 
-  @SuppressWarnings("unchecked")
-  public OpenTSDBPortUnificationHandler(final String handle,
-                                        final TokenAuthenticator tokenAuthenticator,
-                                        final ReportableEntityDecoder decoder,
-                                        final ReportableEntityHandlerFactory handlerFactory,
-                                        @Nullable final ReportableEntityPreprocessor preprocessor,
-                                        @Nullable final CachingGraphiteHostAnnotator annotator) {
-    super(tokenAuthenticator, handle, true, true);
+  public OpenTSDBPortUnificationHandler(
+      final String handle,
+      final TokenAuthenticator tokenAuthenticator,
+      final HealthCheckManager healthCheckManager,
+      final ReportableEntityDecoder decoder,
+      final ReportableEntityHandlerFactory handlerFactory,
+      @Nullable final Supplier preprocessor,
+      @Nullable final Function resolver) {
+    super(tokenAuthenticator, healthCheckManager, handle);
     this.decoder = decoder;
-    this.pointHandler = handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle));
-    this.preprocessor = preprocessor;
-    this.annotator = annotator;
+    this.pointHandler = handlerFactory.getHandler(ReportableEntityType.POINT, handle);
+    this.preprocessorSupplier = preprocessor;
+    this.resolver = resolver;
   }
 
   @Override
-  protected void handleHttpMessage(final ChannelHandlerContext ctx,
-                                   final FullHttpRequest request) {
+  protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request)
+      throws URISyntaxException {
     StringBuilder output = new StringBuilder();
-
-    URI uri = parseUri(ctx, request);
-    if (uri == null) return;
-
+    URI uri = new URI(request.uri());
     switch (uri.getPath()) {
       case "/api/put":
         final ObjectMapper jsonTree = new ObjectMapper();
         HttpResponseStatus status;
         // from the docs:
-        // The put endpoint will respond with a 204 HTTP status code and no content if all data points
-        // were stored successfully. If one or more data points had an error, the API will return a 400.
+        // The put endpoint will respond with a 204 HTTP status code and no content
+        // if all data points were stored successfully. If one or more data points
+        // had an error, the API will return a 400.
         try {
-          if (reportMetrics(jsonTree.readTree(request.content().toString(CharsetUtil.UTF_8)), ctx)) {
+          JsonNode metrics = jsonTree.readTree(request.content().toString(CharsetUtil.UTF_8));
+          if (reportMetrics(metrics, ctx)) {
             status = HttpResponseStatus.NO_CONTENT;
           } else {
             // TODO: improve error message
             // http://opentsdb.net/docs/build/html/api_http/put.html#response
-            // User should understand that successful points are processed and the reason for BAD_REQUEST
-            // is due to at least one failure point.
+            // User should understand that successful points are processed and the
+            // reason
+            // for BAD_REQUEST is due to at least one failure point.
             status = HttpResponseStatus.BAD_REQUEST;
             output.append("At least one data point had error.");
           }
         } catch (Exception e) {
           status = HttpResponseStatus.BAD_REQUEST;
-          writeExceptionText(e, output);
+          output.append(errorMessageWithRootCause(e));
           logWarning("WF-300: Failed to handle /api/put request", e, ctx);
         }
         writeHttpResponse(ctx, status, output, request);
@@ -120,75 +114,25 @@ protected void handleHttpMessage(final ChannelHandlerContext ctx,
     }
   }
 
-  /**
-   * Handles an incoming plain text (string) message.
-   */
-  protected void handlePlainTextMessage(final ChannelHandlerContext ctx,
-                                        String message) throws Exception {
-    if (message == null) {
-      throw new IllegalArgumentException("Message cannot be null");
-    }
-    if (tokenAuthenticator.authRequired()) { // plaintext is disabled with auth enabled
-      pointHandler.reject(message, "Plaintext protocol disabled when authentication is enabled, ignoring");
-      return;
-    }
+  /** Handles an incoming plain text (string) message. */
+  protected void handlePlainTextMessage(final ChannelHandlerContext ctx, @Nonnull String message) {
     if (message.startsWith("version")) {
       ChannelFuture f = ctx.writeAndFlush("Wavefront OpenTSDB Endpoint\n");
       if (!f.isSuccess()) {
-        throw new Exception("Failed to write version response", f.cause());
+        throw new RuntimeException("Failed to write version response", f.cause());
       }
     } else {
-      // transform the line if needed
-      if (preprocessor != null) {
-        message = preprocessor.forPointLine().transform(message);
-
-        // apply white/black lists after formatting
-        if (!preprocessor.forPointLine().filter(message)) {
-          if (preprocessor.forPointLine().getLastFilterResult() != null) {
-            pointHandler.reject((ReportPoint) null, message);
-          } else {
-            pointHandler.block(null, message);
-          }
-          return;
-        }
-      }
-
-      List output = Lists.newArrayListWithCapacity(1);
-      try {
-        decoder.decode(message, output, "dummy");
-      } catch (Exception e) {
-        pointHandler.reject(message, formatErrorMessage("WF-300 Cannot parse: \"" + message + "\"", e, ctx));
-        return;
-      }
-
-      for (ReportPoint object : output) {
-        if (preprocessor != null) {
-          preprocessor.forReportPoint().transform(object);
-          if (!preprocessor.forReportPoint().filter(object)) {
-            if (preprocessor.forReportPoint().getLastFilterResult() != null) {
-              pointHandler.reject(object, preprocessor.forReportPoint().getLastFilterResult());
-            } else {
-              pointHandler.block(object);
-            }
-            return;
-          }
-        }
-        pointHandler.report(object);
-      }
+      WavefrontPortUnificationHandler.preprocessAndHandlePoint(
+          message, decoder, pointHandler, preprocessorSupplier, ctx, "OpenTSDB metric");
     }
   }
 
-  @Override
-  protected void processLine(final ChannelHandlerContext ctx, final String message) {
-    throw new UnsupportedOperationException("Invalid context for processLine");
-  }
-
   /**
-   * Parse the metrics JSON and report the metrics found.  There are 2 formats supported: - array of points - single
-   * point
+   * Parse the metrics JSON and report the metrics found. 2 formats are supported: array of points
+   * and a single point.
    *
    * @param metrics an array of objects or a single object representing a metric
-   * @param ctx     channel handler context (to retrieve remote address)
+   * @param ctx channel handler context (to retrieve remote address)
    * @return true if all metrics added successfully; false o/w
    * @see #reportMetric(JsonNode, ChannelHandlerContext)
    */
@@ -210,9 +154,10 @@ private boolean reportMetrics(final JsonNode metrics, ChannelHandlerContext ctx)
    * Parse the individual metric object and send the metric to on to the point handler.
    *
    * @param metric the JSON object representing a single metric
-   * @param ctx    channel handler context (to retrieve remote address)
+   * @param ctx channel handler context (to retrieve remote address)
    * @return True if the metric was reported successfully; False o/w
-   * @see OpenTSDB /api/put documentation
+   * @see OpenTSDB /api/put
+   *     documentation
    */
   private boolean reportMetric(final JsonNode metric, ChannelHandlerContext ctx) {
     try {
@@ -226,13 +171,12 @@ private boolean reportMetric(final JsonNode metric, ChannelHandlerContext ctx) {
       } else if (wftags.containsKey("source")) {
         hostName = wftags.get("source");
       } else {
-        hostName = annotator == null ? "unknown" : annotator.getRemoteHost(ctx);
+        hostName = resolver == null ? "unknown" : resolver.apply(getRemoteAddress(ctx));
       }
       // remove source/host from the tags list
       Map wftags2 = new HashMap<>();
       for (Map.Entry wftag : wftags.entrySet()) {
-        if (wftag.getKey().equalsIgnoreCase("host") ||
-            wftag.getKey().equalsIgnoreCase("source")) {
+        if (wftag.getKey().equalsIgnoreCase("host") || wftag.getKey().equalsIgnoreCase("source")) {
           continue;
         }
         wftags2.put(wftag.getKey(), wftag.getValue());
@@ -270,11 +214,14 @@ private boolean reportMetric(final JsonNode metric, ChannelHandlerContext ctx) {
       builder.setHost(hostName);
       ReportPoint point = builder.build();
 
+      ReportableEntityPreprocessor preprocessor =
+          preprocessorSupplier == null ? null : preprocessorSupplier.get();
+      String[] messageHolder = new String[1];
       if (preprocessor != null) {
         preprocessor.forReportPoint().transform(point);
-        if (!preprocessor.forReportPoint().filter(point)) {
-          if (preprocessor.forReportPoint().getLastFilterResult() != null) {
-            pointHandler.reject(point, preprocessor.forReportPoint().getLastFilterResult());
+        if (!preprocessor.forReportPoint().filter(point, messageHolder)) {
+          if (messageHolder[0] != null) {
+            pointHandler.reject(point, messageHolder[0]);
             return false;
           } else {
             pointHandler.block(point);
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/PortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/PortUnificationHandler.java
deleted file mode 100644
index 56c6a4033..000000000
--- a/proxy/src/main/java/com/wavefront/agent/listeners/PortUnificationHandler.java
+++ /dev/null
@@ -1,329 +0,0 @@
-package com.wavefront.agent.listeners;
-
-import com.google.common.base.Throwables;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.wavefront.agent.auth.TokenAuthenticator;
-import com.wavefront.common.TaggedMetricName;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.Histogram;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.utils.URLEncodedUtils;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Optional;
-import java.util.function.Supplier;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-import io.netty.buffer.Unpooled;
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-import io.netty.handler.codec.TooLongFrameException;
-import io.netty.handler.codec.compression.DecompressionException;
-import io.netty.handler.codec.http.DefaultFullHttpResponse;
-import io.netty.handler.codec.http.FullHttpRequest;
-import io.netty.handler.codec.http.FullHttpResponse;
-import io.netty.handler.codec.http.HttpHeaderValues;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import io.netty.handler.codec.http.HttpUtil;
-import io.netty.util.CharsetUtil;
-
-import io.netty.handler.codec.http.HttpHeaderNames;
-import io.netty.handler.codec.http.HttpVersion;
-
-import static com.wavefront.agent.Utils.lazySupplier;
-import static org.apache.commons.lang3.ObjectUtils.firstNonNull;
-
-/**
- * This class handles an incoming message of either String or FullHttpRequest type.  All other types are ignored. This
- * will likely be passed to the PlainTextOrHttpFrameDecoder as the handler for messages.
- *
- * @author vasily@wavefront.com
- */
-@ChannelHandler.Sharable
-public abstract class PortUnificationHandler extends SimpleChannelInboundHandler {
-  private static final Logger logger = Logger.getLogger(
-      PortUnificationHandler.class.getCanonicalName());
-
-  protected final Supplier httpRequestHandleDuration;
-  protected final Supplier requestsDiscarded;
-  protected final Supplier pointsDiscarded;
-
-  protected final String handle;
-  protected final TokenAuthenticator tokenAuthenticator;
-  protected final boolean plaintextEnabled;
-  protected final boolean httpEnabled;
-
-  /**
-   * Create new instance.
-   *
-   * @param tokenAuthenticator  tokenAuthenticator for incoming requests.
-   * @param handle              handle/port number.
-   */
-  public PortUnificationHandler(@Nonnull TokenAuthenticator tokenAuthenticator, @Nullable final String handle,
-                                boolean plaintextEnabled, boolean httpEnabled) {
-    this.tokenAuthenticator = tokenAuthenticator;
-    this.handle = firstNonNull(handle, "unknown");
-    this.plaintextEnabled = plaintextEnabled;
-    this.httpEnabled = httpEnabled;
-
-    this.httpRequestHandleDuration = lazySupplier(() -> Metrics.newHistogram(new TaggedMetricName("listeners",
-        "http-requests.duration-nanos", "port", this.handle)));
-    this.requestsDiscarded = lazySupplier(() -> Metrics.newCounter(new TaggedMetricName("listeners",
-        "http-requests.discarded", "port", this.handle)));
-    this.pointsDiscarded = lazySupplier(() -> Metrics.newCounter(new TaggedMetricName("listeners",
-        "items-discarded", "port", this.handle)));
-  }
-
-  /**
-   * Handles an incoming HTTP message. Accepts HTTP POST on all paths
-   */
-  protected void handleHttpMessage(final ChannelHandlerContext ctx,
-                                   final FullHttpRequest request) {
-    StringBuilder output = new StringBuilder();
-
-    HttpResponseStatus status;
-    try {
-      for (String line : StringUtils.split(request.content().toString(CharsetUtil.UTF_8), '\n')) {
-        processLine(ctx, line.trim());
-      }
-      status = HttpResponseStatus.NO_CONTENT;
-    } catch (Exception e) {
-      status = HttpResponseStatus.BAD_REQUEST;
-      writeExceptionText(e, output);
-      logWarning("WF-300: Failed to handle HTTP POST", e, ctx);
-    }
-    writeHttpResponse(ctx, status, output, request);
-  }
-
-  /**
-   * Handles an incoming plain text (string) message. By default simply passes a string to
-   * {@link #processLine(ChannelHandlerContext, String)} method.
-   */
-  protected void handlePlainTextMessage(final ChannelHandlerContext ctx,
-                                        final String message) throws Exception {
-    if (message == null) {
-      throw new IllegalArgumentException("Message cannot be null");
-    }
-    if (!plaintextEnabled || tokenAuthenticator.authRequired()) { // plaintext is disabled with auth enabled
-      pointsDiscarded.get().inc();
-      logger.warning("Input discarded: plaintext protocol is not supported on port " + handle +
-          (tokenAuthenticator.authRequired() ? " (authentication enabled)" : ""));
-      return;
-    }
-    processLine(ctx, message.trim());
-  }
-
-  protected abstract void processLine(final ChannelHandlerContext ctx, final String message);
-
-  @Override
-  public void channelReadComplete(ChannelHandlerContext ctx) {
-    ctx.flush();
-  }
-
-  @Override
-  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
-    if (cause instanceof TooLongFrameException) {
-      logWarning("Received line is too long, consider increasing pushListenerMaxReceivedLength", cause, ctx);
-      return;
-    }
-    if (cause instanceof DecompressionException) {
-      logWarning("Decompression error", cause, ctx);
-      writeHttpResponse(ctx, HttpResponseStatus.BAD_REQUEST, "Decompression error: " + cause.getMessage());
-      return;
-    }
-    if (cause instanceof IOException && cause.getMessage().contains("Connection reset by peer")) {
-      // These errors are caused by the client and are safe to ignore
-      return;
-    }
-    logWarning("Handler failed", cause, ctx);
-    logger.log(Level.WARNING, "Unexpected error: ", cause);
-  }
-
-  protected String extractToken(final ChannelHandlerContext ctx, final FullHttpRequest request) {
-    URI requestUri = parseUri(ctx, request);
-    if (requestUri == null) return null;
-    String token = firstNonNull(request.headers().getAsString("X-AUTH-TOKEN"),
-        request.headers().getAsString("Authorization"), "").replaceAll("^Bearer ", "").trim();
-    Optional tokenParam = URLEncodedUtils.parse(requestUri, CharsetUtil.UTF_8).stream().
-        filter(x -> x.getName().equals("t") || x.getName().equals("token") || x.getName().equals("api_key")).
-        findFirst();
-    if (tokenParam.isPresent()) {
-      token = tokenParam.get().getValue();
-    }
-    return token;
-  }
-
-  protected boolean authorized(final ChannelHandlerContext ctx, final FullHttpRequest request) {
-    if (tokenAuthenticator.authRequired()) {
-      String token = extractToken(ctx, request);
-      if (!tokenAuthenticator.authorize(token)) { // 401 if no token or auth fails
-        writeHttpResponse(ctx, HttpResponseStatus.UNAUTHORIZED, "401 Unauthorized\n");
-        return false;
-      }
-    }
-    return true;
-  }
-
-  @Override
-  protected void channelRead0(final ChannelHandlerContext ctx, final Object message) {
-    try {
-      if (message != null) {
-        if (message instanceof String) {
-          handlePlainTextMessage(ctx, (String) message);
-        } else if (message instanceof FullHttpRequest) {
-          if (!httpEnabled) {
-            requestsDiscarded.get().inc();
-            logger.warning("Inbound HTTP request discarded: HTTP disabled on port " + handle);
-            return;
-          }
-          FullHttpRequest request = (FullHttpRequest) message;
-          if (authorized(ctx, request)) {
-            long startTime = System.nanoTime();
-            handleHttpMessage(ctx, request);
-            httpRequestHandleDuration.get().update(System.nanoTime() - startTime);
-          }
-        } else {
-          logWarning("Received unexpected message type " + message.getClass().getName(), null, ctx);
-        }
-      }
-    } catch (final Exception e) {
-      logWarning("Failed to handle message", e, ctx);
-    }
-  }
-
-  protected URI parseUri(final ChannelHandlerContext ctx, FullHttpRequest request) {
-    try {
-      return new URI(request.uri());
-    } catch (URISyntaxException e) {
-      StringBuilder output = new StringBuilder();
-      writeExceptionText(e, output);
-      writeHttpResponse(ctx, HttpResponseStatus.BAD_REQUEST, output, request);
-      logWarning("WF-300: Request URI '" + request.uri() + "' cannot be parsed", e, ctx);
-      return null;
-    }
-  }
-
-  protected void writeHttpResponse(final ChannelHandlerContext ctx, final HttpResponseStatus status,
-                                   final Object contents) {
-    writeHttpResponse(ctx, status, contents, false);
-  }
-
-  protected void writeHttpResponse(final ChannelHandlerContext ctx, final HttpResponseStatus status,
-                                   final Object contents, final FullHttpRequest request) {
-    writeHttpResponse(ctx, status, contents, HttpUtil.isKeepAlive(request));
-  }
-
-  /**
-   * Writes an HTTP response.
-   */
-  private void writeHttpResponse(final ChannelHandlerContext ctx, final HttpResponseStatus status,
-                                   final Object contents, boolean keepAlive) {
-    final FullHttpResponse response;
-    if (contents instanceof JsonNode) {
-      response = new DefaultFullHttpResponse(
-          HttpVersion.HTTP_1_1, status, Unpooled.copiedBuffer(contents.toString(), CharsetUtil.UTF_8));
-      response.headers().set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.APPLICATION_JSON);
-    } else if (contents instanceof CharSequence) {
-      response = new DefaultFullHttpResponse(
-          HttpVersion.HTTP_1_1, status, Unpooled.copiedBuffer((CharSequence) contents, CharsetUtil.UTF_8));
-      response.headers().set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.TEXT_PLAIN);
-    } else {
-      throw new IllegalArgumentException("Unexpected response content type, JsonNode or CharSequence expected: " +
-          contents.getClass().getName());
-    }
-
-    // Decide whether to close the connection or not.
-    if (keepAlive) {
-      // Add 'Content-Length' header only for a keep-alive connection.
-      response.headers().set(HttpHeaderNames.CONTENT_LENGTH, response.content().readableBytes());
-      // Add keep alive header as per:
-      // - http://www.w3.org/Protocols/HTTP/1.1/draft-ietf-http-v11-spec-01.html#Connection
-      response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE);
-      ctx.write(response);
-    } else {
-      ctx.write(response);
-      ctx.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE);
-    }
-  }
-
-  /**
-   * Log a detailed error message with remote IP address
-   *
-   * @param message   the error message
-   * @param e         the exception (optional) that caused the message to be blocked
-   * @param ctx       ChannelHandlerContext (optional) to extract remote client ip
-   */
-  protected void logWarning(final String message,
-                            @Nullable final Throwable e,
-                            @Nullable final ChannelHandlerContext ctx) {
-    logger.warning(formatErrorMessage(message, e, ctx));
-  }
-
-  /**
-   * Create a detailed error message from an exception.
-   *
-   * @param message   the error message
-   * @param e         the exception (optional) that caused the error
-   * @param ctx       ChannelHandlerContext (optional) to extract remote client ip
-   *
-   * @return formatted error message
-   */
-  protected String formatErrorMessage(final String message,
-                                      @Nullable final Throwable e,
-                                      @Nullable final ChannelHandlerContext ctx) {
-    StringBuilder errMsg = new StringBuilder(message);
-    errMsg.append("; remote: ");
-    errMsg.append(getRemoteName(ctx));
-    if (e != null) {
-      errMsg.append("; ");
-      writeExceptionText(e, errMsg);
-    }
-    return errMsg.toString();
-  }
-
-  /**
-   * Create a error message from an exception.
-   *
-   * @param e   Exceptions thrown
-   * @param msg StringBuilder to write message to
-   */
-  protected void writeExceptionText(@Nonnull final Throwable e, @Nonnull StringBuilder msg) {
-    final Throwable rootCause = Throwables.getRootCause(e);
-    msg.append("reason: \"");
-    msg.append(e.getMessage());
-    msg.append("\"");
-    if (rootCause != null && rootCause != e && rootCause.getMessage() != null) {
-      msg.append(", root cause: \"");
-      msg.append(rootCause.getMessage());
-      msg.append("\"");
-    }
-  }
-
-  /**
-   * Get remote client's address as string (without rDNS lookup) and local port
-   */
-  public static String getRemoteName(@Nullable final ChannelHandlerContext ctx) {
-    if (ctx != null) {
-      InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress();
-      InetSocketAddress localAddress = (InetSocketAddress) ctx.channel().localAddress();
-      if (remoteAddress != null && localAddress != null) {
-        return remoteAddress.getAddress().getHostAddress() + " [" + localAddress.getPort() + "]";
-      }
-    }
-    return "";
-  }
-}
-
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/RawLogsIngesterPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/RawLogsIngesterPortUnificationHandler.java
new file mode 100644
index 000000000..8c420c8c8
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/RawLogsIngesterPortUnificationHandler.java
@@ -0,0 +1,112 @@
+package com.wavefront.agent.listeners;
+
+import static com.wavefront.agent.channel.ChannelUtils.getRemoteAddress;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.formatter.DataFormat;
+import com.wavefront.agent.logsharvesting.LogsIngester;
+import com.wavefront.agent.logsharvesting.LogsMessage;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.DecoderException;
+import io.netty.handler.codec.TooLongFrameException;
+import io.netty.handler.codec.http.FullHttpRequest;
+import java.net.InetAddress;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * Process incoming logs in raw plaintext format.
+ *
+ * @author vasily@wavefront.com
+ */
+public class RawLogsIngesterPortUnificationHandler extends AbstractLineDelimitedHandler {
+  private static final Logger logger =
+      Logger.getLogger(RawLogsIngesterPortUnificationHandler.class.getCanonicalName());
+
+  private final LogsIngester logsIngester;
+  private final Function hostnameResolver;
+  private final Supplier preprocessorSupplier;
+
+  private final Counter received =
+      Metrics.newCounter(new MetricName("logsharvesting", "", "raw-received"));
+
+  /**
+   * Create new instance.
+   *
+   * @param handle handle/port number.
+   * @param ingester log ingester.
+   * @param hostnameResolver rDNS lookup for remote clients ({@link InetAddress} to {@link String}
+   *     resolver)
+   * @param authenticator {@link TokenAuthenticator} for incoming requests.
+   * @param healthCheckManager shared health check endpoint handler.
+   * @param preprocessor preprocessor.
+   */
+  public RawLogsIngesterPortUnificationHandler(
+      String handle,
+      @Nonnull LogsIngester ingester,
+      @Nonnull Function hostnameResolver,
+      @Nullable TokenAuthenticator authenticator,
+      @Nullable HealthCheckManager healthCheckManager,
+      @Nullable Supplier preprocessor) {
+    super(authenticator, healthCheckManager, handle);
+    this.logsIngester = ingester;
+    this.hostnameResolver = hostnameResolver;
+    this.preprocessorSupplier = preprocessor;
+  }
+
+  @Override
+  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+    if (cause instanceof TooLongFrameException) {
+      logWarning(
+          "Received line is too long, consider increasing rawLogsMaxReceivedLength", cause, ctx);
+      return;
+    }
+    if (cause instanceof DecoderException) {
+      logger.log(Level.WARNING, "Unexpected exception in raw logs ingester", cause);
+    }
+    super.exceptionCaught(ctx, cause);
+  }
+
+  @Nullable
+  @Override
+  protected DataFormat getFormat(FullHttpRequest httpRequest) {
+    return null;
+  }
+
+  @VisibleForTesting
+  @Override
+  public void processLine(
+      final ChannelHandlerContext ctx, @Nonnull String message, @Nullable DataFormat format) {
+    received.inc();
+    ReportableEntityPreprocessor preprocessor =
+        preprocessorSupplier == null ? null : preprocessorSupplier.get();
+    String processedMessage =
+        preprocessor == null ? message : preprocessor.forPointLine().transform(message);
+    if (preprocessor != null && !preprocessor.forPointLine().filter(message, null)) return;
+
+    logsIngester.ingestLog(
+        new LogsMessage() {
+          @Override
+          public String getLogLine() {
+            return processedMessage;
+          }
+
+          @Override
+          public String hostOrDefault(String fallbackHost) {
+            String hostname = hostnameResolver.apply(getRemoteAddress(ctx));
+            return StringUtils.isBlank(hostname) ? fallbackHost : hostname;
+          }
+        });
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/RelayPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/RelayPortUnificationHandler.java
index 37f64f41f..6079ccc4d 100644
--- a/proxy/src/main/java/com/wavefront/agent/listeners/RelayPortUnificationHandler.java
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/RelayPortUnificationHandler.java
@@ -1,116 +1,398 @@
 package com.wavefront.agent.listeners;
 
+import static com.wavefront.agent.channel.ChannelUtils.errorMessageWithRootCause;
+import static com.wavefront.agent.channel.ChannelUtils.formatErrorMessage;
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.HISTO_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.LOGS_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPANLOGS_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled;
+import static com.wavefront.agent.listeners.WavefrontPortUnificationHandler.preprocessAndHandlePoint;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
+import com.google.common.base.Splitter;
+import com.google.common.base.Throwables;
+import com.wavefront.agent.ProxyConfig;
+import com.wavefront.agent.TokenManager;
+import com.wavefront.agent.api.APIContainer;
 import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.channel.SharedGraphiteHostAnnotator;
+import com.wavefront.agent.formatter.DataFormat;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
 import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
-import com.wavefront.agent.preprocessor.ReportableEntityPreprocessor;
-import com.wavefront.common.Clock;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.AgentConfiguration;
+import com.wavefront.api.agent.Constants;
+import com.wavefront.common.TaggedMetricName;
+import com.wavefront.common.Utils;
 import com.wavefront.data.ReportableEntityType;
 import com.wavefront.ingester.ReportableEntityDecoder;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.net.URI;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
 import io.netty.channel.ChannelHandler;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.util.CharsetUtil;
+import java.net.URI;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+import wavefront.report.ReportPoint;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
 
 /**
- * Process incoming HTTP requests from other proxies (i.e. act as a relay for proxy chaining).
- * Supports metric and histogram data (no source tag or tracing support at this moment).
- * Intended for internal use.
+ * A unified HTTP endpoint for mixed format data. Can serve as a proxy endpoint and process incoming
+ * HTTP requests from other proxies (i.e. act as a relay for proxy chaining), as well as serve as a
+ * DDI (Direct Data Ingestion) endpoint. All the data received on this endpoint will register as
+ * originating from this proxy. Supports metric, histogram and distributed trace data (no source tag
+ * support or log support at this moment). Intended for internal use.
  *
  * @author vasily@wavefront.com
  */
 @ChannelHandler.Sharable
-public class RelayPortUnificationHandler extends WavefrontPortUnificationHandler {
-  private static final Logger logger = Logger.getLogger(RelayPortUnificationHandler.class.getCanonicalName());
-
-  private static final Pattern PATTERN_CHECKIN = Pattern.compile("/api/daemon/(.*)/checkin");
-  private static final Pattern PATTERN_PUSHDATA = Pattern.compile("/api/daemon/(.*)/pushdata/(.*)");
-
-  private Cache proxyTokenCache = Caffeine.newBuilder()
-      .expireAfterWrite(10, TimeUnit.MINUTES)
-      .maximumSize(10_000)
-      .build();
-
-  public RelayPortUnificationHandler(final String handle,
-                                     final TokenAuthenticator tokenAuthenticator,
-                                     final Map decoders,
-                                     final ReportableEntityHandlerFactory handlerFactory,
-                                     @Nullable final ReportableEntityPreprocessor preprocessor) {
-    super(handle, tokenAuthenticator, decoders, handlerFactory, null, preprocessor);
-  }
+public class RelayPortUnificationHandler extends AbstractHttpOnlyHandler {
+  private static final Logger logger =
+      Logger.getLogger(RelayPortUnificationHandler.class.getCanonicalName());
 
-  @Override
-  protected boolean authorized(final ChannelHandlerContext ctx, final FullHttpRequest request) {
-    if (tokenAuthenticator.authRequired()) {
-      String token = extractToken(ctx, request);
+  private static final ObjectMapper JSON_PARSER = new ObjectMapper();
 
-      URI uri = parseUri(ctx, request);
-      if (uri == null) return false;
+  private final Map> decoders;
+  private final ReportableEntityDecoder wavefrontDecoder;
+  private ProxyConfig proxyConfig;
+  private final ReportableEntityHandler wavefrontHandler;
+  private final Supplier> histogramHandlerSupplier;
+  private final Supplier> spanHandlerSupplier;
+  private final Supplier> spanLogsHandlerSupplier;
+  private final Supplier preprocessorSupplier;
+  private final SharedGraphiteHostAnnotator annotator;
 
-      Matcher patternPushDataMatcher = PATTERN_PUSHDATA.matcher(uri.getPath());
-      if (patternPushDataMatcher.matches()) {
-        // extract proxy ID from the URL and get actual token from cache
-        token = proxyTokenCache.getIfPresent(patternPushDataMatcher.replaceAll("$1"));
-      }
+  private final Supplier histogramDisabled;
+  private final Supplier traceDisabled;
+  private final Supplier spanLogsDisabled;
+  private final Supplier logsDisabled;
 
-      if (!tokenAuthenticator.authorize(token)) { // 401 if no token or auth fails
-        writeHttpResponse(ctx, HttpResponseStatus.UNAUTHORIZED, "401 Unauthorized\n");
-        return false;
-      }
+  private final Supplier discardedHistograms;
+  private final Supplier discardedSpans;
+  private final Supplier discardedSpanLogs;
+  private final Supplier receivedSpansTotal;
 
-      Matcher patternCheckinMatcher = PATTERN_CHECKIN.matcher(uri.getPath());
-      if (patternCheckinMatcher.matches() && token != null) {
-        String proxyId = patternCheckinMatcher.replaceAll("$1");
-        logger.info("Caching auth token for proxy ID " + proxyId);
-        proxyTokenCache.put(proxyId, token);
-      }
-    }
-    return true;
+  private final APIContainer apiContainer;
+  /**
+   * Create new instance with lazy initialization for handlers.
+   *
+   * @param handle handle/port number.
+   * @param tokenAuthenticator tokenAuthenticator for incoming requests.
+   * @param healthCheckManager shared health check endpoint handler.
+   * @param decoders decoders.
+   * @param handlerFactory factory for ReportableEntityHandler objects.
+   * @param preprocessorSupplier preprocessor supplier.
+   * @param histogramDisabled supplier for backend-controlled feature flag for histograms.
+   * @param traceDisabled supplier for backend-controlled feature flag for spans.
+   * @param spanLogsDisabled supplier for backend-controlled feature flag for span logs.
+   * @param logsDisabled supplier for backend-controlled feature flag for logs.
+   */
+  @SuppressWarnings("unchecked")
+  public RelayPortUnificationHandler(
+      final String handle,
+      final TokenAuthenticator tokenAuthenticator,
+      final HealthCheckManager healthCheckManager,
+      final Map> decoders,
+      final ReportableEntityHandlerFactory handlerFactory,
+      @Nullable final Supplier preprocessorSupplier,
+      @Nullable final SharedGraphiteHostAnnotator annotator,
+      final Supplier histogramDisabled,
+      final Supplier traceDisabled,
+      final Supplier spanLogsDisabled,
+      final Supplier logsDisabled,
+      final APIContainer apiContainer,
+      final ProxyConfig proxyConfig) {
+    super(tokenAuthenticator, healthCheckManager, handle);
+    this.decoders = decoders;
+    this.wavefrontDecoder =
+        (ReportableEntityDecoder) decoders.get(ReportableEntityType.POINT);
+    this.proxyConfig = proxyConfig;
+    this.wavefrontHandler =
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle));
+    this.histogramHandlerSupplier =
+        Utils.lazySupplier(
+            () -> handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM, handle)));
+    this.spanHandlerSupplier =
+        Utils.lazySupplier(
+            () -> handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)));
+    this.spanLogsHandlerSupplier =
+        Utils.lazySupplier(
+            () ->
+                handlerFactory.getHandler(
+                    HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, handle)));
+    this.receivedSpansTotal =
+        Utils.lazySupplier(
+            () -> Metrics.newCounter(new MetricName("spans." + handle, "", "received.total")));
+    this.preprocessorSupplier = preprocessorSupplier;
+    this.annotator = annotator;
+    this.histogramDisabled = histogramDisabled;
+    this.traceDisabled = traceDisabled;
+    this.spanLogsDisabled = spanLogsDisabled;
+    this.logsDisabled = logsDisabled;
+
+    this.discardedHistograms =
+        Utils.lazySupplier(
+            () -> Metrics.newCounter(new MetricName("histogram", "", "discarded_points")));
+    this.discardedSpans =
+        Utils.lazySupplier(
+            () -> Metrics.newCounter(new MetricName("spans." + handle, "", "discarded")));
+    this.discardedSpanLogs =
+        Utils.lazySupplier(
+            () -> Metrics.newCounter(new MetricName("spanLogs." + handle, "", "discarded")));
+    this.apiContainer = apiContainer;
   }
 
   @Override
-  protected void handleHttpMessage(final ChannelHandlerContext ctx,
-                                   final FullHttpRequest request) {
+  protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request) {
+    URI uri = URI.create(request.uri());
     StringBuilder output = new StringBuilder();
+    String path = uri.getPath();
+
+    if (path.endsWith("/checkin") && (path.startsWith("/api/daemon") || path.contains("wfproxy"))) {
+      Map query =
+          URLEncodedUtils.parse(uri, Charset.forName("UTF-8")).stream()
+              .collect(Collectors.toMap(NameValuePair::getName, NameValuePair::getValue));
 
-    URI uri = parseUri(ctx, request);
-    if (uri == null) return;
+      String agentMetricsStr = request.content().toString(CharsetUtil.UTF_8);
+      JsonNode agentMetrics;
+      try {
+        agentMetrics = JSON_PARSER.readTree(agentMetricsStr);
+      } catch (JsonProcessingException e) {
+        if (logger.isLoggable(Level.FINE)) {
+          logger.log(Level.WARNING, "Exception: ", e);
+        }
+        agentMetrics = JsonNodeFactory.instance.objectNode();
+      }
 
-    if (uri.getPath().startsWith("/api/daemon") && uri.getPath().endsWith("/checkin")) {
-      // simulate checkin response for proxy chaining
-      ObjectNode jsonResponse = JsonNodeFactory.instance.objectNode();
-      jsonResponse.put("currentTime", Clock.now());
-      jsonResponse.put("allowAnyHostKeys", true);
-      writeHttpResponse(ctx, HttpResponseStatus.OK, jsonResponse, request);
+      try {
+        AgentConfiguration agentConfiguration =
+            apiContainer
+                .getProxyV2APIForTenant(APIContainer.CENTRAL_TENANT_NAME)
+                .proxyCheckin(
+                    UUID.fromString(request.headers().get("X-WF-PROXY-ID")),
+                    "Bearer "
+                        + TokenManager.getMulticastingTenantList()
+                            .get(APIContainer.CENTRAL_TENANT_NAME)
+                            .getBearerToken(),
+                    query.get("hostname"),
+                    query.get("proxyname"),
+                    query.get("version"),
+                    Long.parseLong(query.get("currentMillis")),
+                    agentMetrics,
+                    Boolean.parseBoolean(query.get("ephemeral")));
+        JsonNode node = JSON_PARSER.valueToTree(agentConfiguration);
+        writeHttpResponse(ctx, HttpResponseStatus.OK, node, request);
+      } catch (javax.ws.rs.ProcessingException e) {
+        logger.warning("Problem while checking a chained proxy: " + e);
+        if (logger.isLoggable(Level.FINE)) {
+          logger.log(Level.WARNING, "Exception: ", e);
+        }
+        Throwable rootCause = Throwables.getRootCause(e);
+        String error =
+            "Request processing error: Unable to retrieve proxy configuration from '"
+                + proxyConfig.getServer()
+                + "' :"
+                + rootCause;
+        writeHttpResponse(ctx, new HttpResponseStatus(444, error), error, request);
+      } catch (Throwable e) {
+        logger.warning("Problem while checking a chained proxy: " + e);
+        if (logger.isLoggable(Level.FINE)) {
+          logger.log(Level.WARNING, "Exception: ", e);
+        }
+        String error =
+            "Request processing error: Unable to retrieve proxy configuration from '"
+                + proxyConfig.getServer()
+                + "'";
+        writeHttpResponse(ctx, new HttpResponseStatus(500, error), error, request);
+      }
       return;
     }
 
+    String format =
+        URLEncodedUtils.parse(uri, CharsetUtil.UTF_8).stream()
+            .filter(x -> x.getName().equals("format") || x.getName().equals("f"))
+            .map(NameValuePair::getValue)
+            .findFirst()
+            .orElse(Constants.PUSH_FORMAT_WAVEFRONT);
+
+    // Return HTTP 200 (OK) for payloads received on the proxy endpoint
+    // Return HTTP 202 (ACCEPTED) for payloads received on the DDI endpoint
+    // Return HTTP 204 (NO_CONTENT) for payloads received on all other endpoints
+    final boolean isDirectIngestion = path.startsWith("/report");
+    HttpResponseStatus okStatus;
+    if (isDirectIngestion) {
+      okStatus = HttpResponseStatus.ACCEPTED;
+    } else if (path.contains("/pushdata/") || path.contains("wfproxy/report")) {
+      okStatus = HttpResponseStatus.OK;
+    } else {
+      okStatus = HttpResponseStatus.NO_CONTENT;
+    }
+
     HttpResponseStatus status;
-    try {
-      for (String line : StringUtils.split(request.content().toString(CharsetUtil.UTF_8), '\n')) {
-        processLine(ctx, line.trim());
-      }
-      status = HttpResponseStatus.OK;
-    } catch (Exception e) {
-      status = HttpResponseStatus.BAD_REQUEST;
-      writeExceptionText(e, output);
-      logWarning("WF-300: Failed to handle HTTP POST", e, ctx);
+    switch (format) {
+      case Constants.PUSH_FORMAT_HISTOGRAM:
+        if (isFeatureDisabled(
+            histogramDisabled, HISTO_DISABLED, discardedHistograms.get(), output, request)) {
+          status = HttpResponseStatus.FORBIDDEN;
+          break;
+        }
+      case Constants.PUSH_FORMAT_WAVEFRONT:
+      case Constants.PUSH_FORMAT_GRAPHITE_V2:
+        AtomicBoolean hasSuccessfulPoints = new AtomicBoolean(false);
+        try {
+          //noinspection unchecked
+          ReportableEntityDecoder histogramDecoder =
+              (ReportableEntityDecoder)
+                  decoders.get(ReportableEntityType.HISTOGRAM);
+          Splitter.on('\n')
+              .trimResults()
+              .omitEmptyStrings()
+              .split(request.content().toString(CharsetUtil.UTF_8))
+              .forEach(
+                  message -> {
+                    DataFormat dataFormat = DataFormat.autodetect(message);
+                    switch (dataFormat) {
+                      case EVENT:
+                        wavefrontHandler.reject(
+                            message, "Relay port does not support " + "event-formatted data!");
+                        break;
+                      case SOURCE_TAG:
+                        wavefrontHandler.reject(
+                            message, "Relay port does not support " + "sourceTag-formatted data!");
+                        break;
+                      case HISTOGRAM:
+                        if (isFeatureDisabled(
+                            histogramDisabled, HISTO_DISABLED, discardedHistograms.get(), output)) {
+                          break;
+                        }
+                        preprocessAndHandlePoint(
+                            message,
+                            histogramDecoder,
+                            histogramHandlerSupplier.get(),
+                            preprocessorSupplier,
+                            ctx,
+                            "histogram");
+                        hasSuccessfulPoints.set(true);
+                        break;
+                      default:
+                        // only apply annotator if point received on the DDI
+                        // endpoint
+                        message =
+                            annotator != null && isDirectIngestion
+                                ? annotator.apply(ctx, message)
+                                : message;
+                        preprocessAndHandlePoint(
+                            message,
+                            wavefrontDecoder,
+                            wavefrontHandler,
+                            preprocessorSupplier,
+                            ctx,
+                            "metric");
+                        hasSuccessfulPoints.set(true);
+                        break;
+                    }
+                  });
+          status = hasSuccessfulPoints.get() ? okStatus : HttpResponseStatus.BAD_REQUEST;
+        } catch (Exception e) {
+          status = HttpResponseStatus.BAD_REQUEST;
+          output.append(errorMessageWithRootCause(e));
+          logWarning("WF-300: Failed to handle HTTP POST", e, ctx);
+        }
+        break;
+      case Constants.PUSH_FORMAT_TRACING:
+        if (isFeatureDisabled(
+            traceDisabled, SPAN_DISABLED, discardedSpans.get(), output, request)) {
+          receivedSpansTotal.get().inc(discardedSpans.get().count());
+          status = HttpResponseStatus.FORBIDDEN;
+          break;
+        }
+        List spans = new ArrayList<>();
+        //noinspection unchecked
+        ReportableEntityDecoder spanDecoder =
+            (ReportableEntityDecoder) decoders.get(ReportableEntityType.TRACE);
+        ReportableEntityHandler spanHandler = spanHandlerSupplier.get();
+        Splitter.on('\n')
+            .trimResults()
+            .omitEmptyStrings()
+            .split(request.content().toString(CharsetUtil.UTF_8))
+            .forEach(
+                line -> {
+                  try {
+                    receivedSpansTotal.get().inc();
+                    spanDecoder.decode(line, spans, "dummy");
+                  } catch (Exception e) {
+                    spanHandler.reject(line, formatErrorMessage(line, e, ctx));
+                  }
+                });
+        spans.forEach(spanHandler::report);
+        status = okStatus;
+        break;
+      case Constants.PUSH_FORMAT_TRACING_SPAN_LOGS:
+        if (isFeatureDisabled(
+            spanLogsDisabled, SPANLOGS_DISABLED, discardedSpanLogs.get(), output, request)) {
+          status = HttpResponseStatus.FORBIDDEN;
+          break;
+        }
+        List spanLogs = new ArrayList<>();
+        //noinspection unchecked
+        ReportableEntityDecoder spanLogDecoder =
+            (ReportableEntityDecoder)
+                decoders.get(ReportableEntityType.TRACE_SPAN_LOGS);
+        ReportableEntityHandler spanLogsHandler = spanLogsHandlerSupplier.get();
+        Splitter.on('\n')
+            .trimResults()
+            .omitEmptyStrings()
+            .split(request.content().toString(CharsetUtil.UTF_8))
+            .forEach(
+                line -> {
+                  try {
+                    spanLogDecoder.decode(JSON_PARSER.readTree(line), spanLogs, "dummy");
+                  } catch (Exception e) {
+                    spanLogsHandler.reject(line, formatErrorMessage(line, e, ctx));
+                  }
+                });
+        spanLogs.forEach(spanLogsHandler::report);
+        status = okStatus;
+        break;
+      case Constants.PUSH_FORMAT_LOGS_JSON_ARR:
+      case Constants.PUSH_FORMAT_LOGS_JSON_LINES:
+      case Constants.PUSH_FORMAT_LOGS_JSON_CLOUDWATCH:
+        Supplier discardedLogs =
+            Utils.lazySupplier(
+                () ->
+                    Metrics.newCounter(
+                        new TaggedMetricName("logs." + handle, "discarded", "format", format)));
+
+        if (isFeatureDisabled(logsDisabled, LOGS_DISABLED, discardedLogs.get(), output, request)) {
+          status = HttpResponseStatus.FORBIDDEN;
+          break;
+        }
+      default:
+        status = HttpResponseStatus.BAD_REQUEST;
+        logger.warning("Unexpected format for incoming HTTP request: " + format);
     }
     writeHttpResponse(ctx, status, output, request);
   }
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/TracePortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/TracePortUnificationHandler.java
deleted file mode 100644
index 0c85c3786..000000000
--- a/proxy/src/main/java/com/wavefront/agent/listeners/TracePortUnificationHandler.java
+++ /dev/null
@@ -1,121 +0,0 @@
-package com.wavefront.agent.listeners;
-
-import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-
-import com.wavefront.agent.auth.TokenAuthenticator;
-import com.wavefront.agent.handlers.HandlerKey;
-import com.wavefront.agent.handlers.ReportableEntityHandler;
-import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
-import com.wavefront.agent.preprocessor.ReportableEntityPreprocessor;
-import com.wavefront.data.ReportableEntityType;
-import com.wavefront.ingester.ReportableEntityDecoder;
-import com.wavefront.sdk.entities.tracing.sampling.Sampler;
-
-import java.net.InetSocketAddress;
-import java.util.List;
-import java.util.UUID;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-
-import io.netty.channel.ChannelHandlerContext;
-import wavefront.report.Span;
-
-/**
- * Process incoming trace-formatted data.
- *
- * Accepts incoming messages of either String or FullHttpRequest type: single Span in a string,
- * or multiple points in the HTTP post body, newline-delimited.
- *
- * @author vasily@wavefront.com
- */
-public class TracePortUnificationHandler extends PortUnificationHandler {
-  private static final Logger logger = Logger.getLogger(
-      TracePortUnificationHandler.class.getCanonicalName());
-
-  private final ReportableEntityHandler handler;
-  private final ReportableEntityDecoder decoder;
-  private final ReportableEntityPreprocessor preprocessor;
-  private final Sampler sampler;
-
-  @SuppressWarnings("unchecked")
-  public TracePortUnificationHandler(final String handle,
-                                     final TokenAuthenticator tokenAuthenticator,
-                                     final ReportableEntityDecoder traceDecoder,
-                                     @Nullable final ReportableEntityPreprocessor preprocessor,
-                                     final ReportableEntityHandlerFactory handlerFactory,
-                                     final Sampler sampler) {
-    this(handle, tokenAuthenticator, traceDecoder, preprocessor,
-        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)), sampler);
-  }
-
-  public TracePortUnificationHandler(final String handle,
-                                     final TokenAuthenticator tokenAuthenticator,
-                                     final ReportableEntityDecoder traceDecoder,
-                                     @Nullable final ReportableEntityPreprocessor preprocessor,
-                                     final ReportableEntityHandler handler,
-                                     final Sampler sampler) {
-    super(tokenAuthenticator, handle, true, true);
-    this.decoder = traceDecoder;
-    this.handler = handler;
-    this.preprocessor = preprocessor;
-    this.sampler = sampler;
-  }
-
-  @Override
-  protected void processLine(final ChannelHandlerContext ctx, String message) {
-    // transform the line if needed
-    if (preprocessor != null) {
-      message = preprocessor.forPointLine().transform(message);
-
-      // apply white/black lists after formatting
-      if (!preprocessor.forPointLine().filter(message)) {
-        if (preprocessor.forPointLine().getLastFilterResult() != null) {
-          handler.reject((Span) null, message);
-        } else {
-          handler.block(null, message);
-        }
-        return;
-      }
-    }
-
-    List output = Lists.newArrayListWithCapacity(1);
-    try {
-      decoder.decode(message, output, "dummy");
-    } catch (Exception e) {
-      final Throwable rootCause = Throwables.getRootCause(e);
-      String errMsg = "WF-300 Cannot parse: \"" + message +
-          "\", reason: \"" + e.getMessage() + "\"";
-      if (rootCause != null && rootCause.getMessage() != null && rootCause != e) {
-        errMsg = errMsg + ", root cause: \"" + rootCause.getMessage() + "\"";
-      }
-      if (ctx != null) {
-        InetSocketAddress remoteAddress = (InetSocketAddress) ctx.channel().remoteAddress();
-        if (remoteAddress != null) {
-          errMsg += "; remote: " + remoteAddress.getHostString();
-        }
-      }
-      handler.reject(message, errMsg);
-      return;
-    }
-
-    for (Span object : output) {
-      if (preprocessor != null) {
-        preprocessor.forSpan().transform(object);
-        if (!preprocessor.forSpan().filter((object))) {
-          if (preprocessor.forSpan().getLastFilterResult() != null) {
-            handler.reject(object, preprocessor.forSpan().getLastFilterResult());
-          } else {
-            handler.block(object);
-          }
-          return;
-        }
-      }
-      if (sampler.sample(object.getName(), UUID.fromString(object.getTraceId()).getLeastSignificantBits(),
-          object.getDuration())) {
-        handler.report(object);
-      }
-    }
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/WavefrontPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/WavefrontPortUnificationHandler.java
index 7892c4c7b..72a317f0b 100644
--- a/proxy/src/main/java/com/wavefront/agent/listeners/WavefrontPortUnificationHandler.java
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/WavefrontPortUnificationHandler.java
@@ -1,149 +1,470 @@
 package com.wavefront.agent.listeners;
 
-import com.google.common.collect.Lists;
+import static com.wavefront.agent.LogsUtil.LOGS_DATA_FORMATS;
+import static com.wavefront.agent.channel.ChannelUtils.formatErrorMessage;
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+import static com.wavefront.agent.formatter.DataFormat.*;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.HISTO_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.LOGS_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.LOGS_SERVER_DETAILS_MISSING;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPANLOGS_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.isMissingLogServerInfoForAConvergedCSPTenant;
+import static com.wavefront.agent.listeners.tracing.SpanUtils.handleSpanLogs;
+import static com.wavefront.agent.listeners.tracing.SpanUtils.preprocessAndHandleSpan;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
 import com.wavefront.agent.auth.TokenAuthenticator;
-import com.wavefront.agent.channel.CachingGraphiteHostAnnotator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.channel.SharedGraphiteHostAnnotator;
+import com.wavefront.agent.formatter.DataFormat;
 import com.wavefront.agent.handlers.HandlerKey;
 import com.wavefront.agent.handlers.ReportableEntityHandler;
 import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
-import com.wavefront.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.common.TaggedMetricName;
+import com.wavefront.common.Utils;
 import com.wavefront.data.ReportableEntityType;
-import com.wavefront.ingester.ReportSourceTagDecoder;
+import com.wavefront.dto.SourceTag;
 import com.wavefront.ingester.ReportableEntityDecoder;
-
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.util.CharsetUtil;
+import java.net.URI;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.logging.Logger;
-
+import java.util.function.Supplier;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-
-import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerContext;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+import wavefront.report.ReportEvent;
+import wavefront.report.ReportLog;
 import wavefront.report.ReportPoint;
 import wavefront.report.ReportSourceTag;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
 
 /**
- * Process incoming Wavefront-formatted data. Also allows sourceTag formatted data and histogram-formatted data
- * pass-through with lazy-initialized handlers.
+ * Process incoming Wavefront-formatted data. Also allows sourceTag formatted data and
+ * histogram-formatted data pass-through with lazy-initialized handlers.
  *
- * Accepts incoming messages of either String or FullHttpRequest type: single data point in a string,
- * or multiple points in the HTTP post body, newline-delimited.
+ * 
Accepts incoming messages of either String or FullHttpRequest type: single data point in a
+ * string, or multiple points in the HTTP post body, newline-delimited.
  *
  * @author vasily@wavefront.com
  */
 @ChannelHandler.Sharable
-public class WavefrontPortUnificationHandler extends PortUnificationHandler {
-  private static final Logger logger = Logger.getLogger(WavefrontPortUnificationHandler.class.getCanonicalName());
-
-  @Nullable
-  private final CachingGraphiteHostAnnotator annotator;
-
-  @Nullable
-  private final ReportableEntityPreprocessor preprocessor;
+public class WavefrontPortUnificationHandler extends AbstractLineDelimitedHandler {
+  @Nullable private final SharedGraphiteHostAnnotator annotator;
+  @Nullable private final Supplier preprocessorSupplier;
+  private final ReportableEntityDecoder wavefrontDecoder;
+  private final ReportableEntityDecoder sourceTagDecoder;
+  private final ReportableEntityDecoder eventDecoder;
+  private final ReportableEntityDecoder histogramDecoder;
+  private final ReportableEntityDecoder spanDecoder;
+  private final ReportableEntityDecoder spanLogsDecoder;
+  private final ReportableEntityDecoder logDecoder;
+  private final ReportableEntityHandler wavefrontHandler;
+  private final Supplier> histogramHandlerSupplier;
+  private final Supplier>
+      sourceTagHandlerSupplier;
+  private final Supplier> spanHandlerSupplier;
+  private final Supplier> spanLogsHandlerSupplier;
+  private final Supplier> eventHandlerSupplier;
+  private final Supplier> logHandlerSupplier;
 
-  private final ReportableEntityHandlerFactory handlerFactory;
-  private final Map decoders;
+  private final Supplier histogramDisabled;
+  private final Supplier traceDisabled;
+  private final Supplier spanLogsDisabled;
+  private final Supplier logsDisabled;
+  private final boolean receivedLogServerDetails;
+  private final boolean enableHyperlogsConvergedCsp;
+  private final SpanSampler sampler;
 
-  private final ReportableEntityDecoder wavefrontDecoder;
-  private volatile ReportableEntityDecoder sourceTagDecoder;
-  private volatile ReportableEntityDecoder histogramDecoder;
-  private final ReportableEntityHandler wavefrontHandler;
-  private volatile ReportableEntityHandler sourceTagHandler;
-  private volatile ReportableEntityHandler histogramHandler;
+  private final Supplier receivedSpansTotal;
+  private final Supplier discardedHistograms;
+  private final Supplier discardedSpans;
+  private final Supplier discardedSpanLogs;
+  private final Supplier discardedSpansBySampler;
+  private final Supplier discardedSpanLogsBySampler;
+  private final LoadingCache receivedLogsCounter;
+  private final LoadingCache discardedLogsCounter;
+  private final LoadingCache discardedLogsMissingLogServerInfoCounter;
 
   /**
    * Create new instance with lazy initialization for handlers.
    *
-   * @param handle              handle/port number.
-   * @param tokenAuthenticator  tokenAuthenticator for incoming requests.
-   * @param decoders            decoders.
-   * @param handlerFactory      factory for ReportableEntityHandler objects.
-   * @param annotator           hostAnnotator that makes sure all points have a source= tag.
-   * @param preprocessor        preprocessor.
+   * @param handle handle/port number.
+   * @param tokenAuthenticator tokenAuthenticator for incoming requests.
+   * @param healthCheckManager shared health check endpoint handler.
+   * @param decoders decoders.
+   * @param handlerFactory factory for ReportableEntityHandler objects.
+   * @param annotator hostAnnotator that makes sure all points have a source= tag.
+   * @param preprocessor preprocessor supplier.
+   * @param histogramDisabled supplier for backend-controlled feature flag for histograms.
+   * @param traceDisabled supplier for backend-controlled feature flag for spans.
+   * @param spanLogsDisabled supplier for backend-controlled feature flag for span logs.
+   * @param sampler handles sampling of spans and span logs.
+   * @param logsDisabled supplier for backend-controlled feature flag for logs.
+   * @param receivedLogServerDetails boolean that indicates availability of log server URL & token
+   * @param enableHyperlogsConvergedCsp boolean that indicates converged CSP tenant setting
    */
   @SuppressWarnings("unchecked")
-  public WavefrontPortUnificationHandler(final String handle,
-                                         final TokenAuthenticator tokenAuthenticator,
-                                         final Map decoders,
-                                         final ReportableEntityHandlerFactory handlerFactory,
-                                         @Nullable final CachingGraphiteHostAnnotator annotator,
-                                         @Nullable final ReportableEntityPreprocessor preprocessor) {
-    super(tokenAuthenticator, handle, true, true);
-    this.decoders = decoders;
-    this.wavefrontDecoder = (ReportableEntityDecoder)(decoders.get(ReportableEntityType.POINT));
-    this.handlerFactory = handlerFactory;
+  public WavefrontPortUnificationHandler(
+      final String handle,
+      final TokenAuthenticator tokenAuthenticator,
+      final HealthCheckManager healthCheckManager,
+      final Map> decoders,
+      final ReportableEntityHandlerFactory handlerFactory,
+      @Nullable final SharedGraphiteHostAnnotator annotator,
+      @Nullable final Supplier preprocessor,
+      final Supplier histogramDisabled,
+      final Supplier traceDisabled,
+      final Supplier spanLogsDisabled,
+      final SpanSampler sampler,
+      final Supplier logsDisabled,
+      final boolean receivedLogServerDetails,
+      final boolean enableHyperlogsConvergedCsp) {
+    super(tokenAuthenticator, healthCheckManager, handle);
+    this.wavefrontDecoder =
+        (ReportableEntityDecoder) decoders.get(ReportableEntityType.POINT);
     this.annotator = annotator;
-    this.preprocessor = preprocessor;
-    this.wavefrontHandler = handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle));
+    this.preprocessorSupplier = preprocessor;
+    this.wavefrontHandler =
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle));
+    this.histogramDecoder =
+        (ReportableEntityDecoder) decoders.get(ReportableEntityType.HISTOGRAM);
+    this.sourceTagDecoder =
+        (ReportableEntityDecoder)
+            decoders.get(ReportableEntityType.SOURCE_TAG);
+    this.spanDecoder =
+        (ReportableEntityDecoder) decoders.get(ReportableEntityType.TRACE);
+    this.spanLogsDecoder =
+        (ReportableEntityDecoder)
+            decoders.get(ReportableEntityType.TRACE_SPAN_LOGS);
+    this.eventDecoder =
+        (ReportableEntityDecoder) decoders.get(ReportableEntityType.EVENT);
+    this.logDecoder =
+        (ReportableEntityDecoder) decoders.get(ReportableEntityType.LOGS);
+    this.histogramHandlerSupplier =
+        Utils.lazySupplier(
+            () -> handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM, handle)));
+    this.sourceTagHandlerSupplier =
+        Utils.lazySupplier(
+            () ->
+                handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.SOURCE_TAG, handle)));
+    this.spanHandlerSupplier =
+        Utils.lazySupplier(
+            () -> handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)));
+    this.spanLogsHandlerSupplier =
+        Utils.lazySupplier(
+            () ->
+                handlerFactory.getHandler(
+                    HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, handle)));
+    this.eventHandlerSupplier =
+        Utils.lazySupplier(
+            () -> handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.EVENT, handle)));
+    this.logHandlerSupplier =
+        Utils.lazySupplier(
+            () -> handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.LOGS, handle)));
+    this.histogramDisabled = histogramDisabled;
+    this.traceDisabled = traceDisabled;
+    this.spanLogsDisabled = spanLogsDisabled;
+    this.logsDisabled = logsDisabled;
+    this.receivedLogServerDetails = receivedLogServerDetails;
+    this.enableHyperlogsConvergedCsp = enableHyperlogsConvergedCsp;
+    this.sampler = sampler;
+    this.discardedHistograms =
+        Utils.lazySupplier(
+            () -> Metrics.newCounter(new MetricName("histogram", "", "discarded_points")));
+    this.discardedSpans =
+        Utils.lazySupplier(
+            () -> Metrics.newCounter(new MetricName("spans." + handle, "", "discarded")));
+    this.discardedSpanLogs =
+        Utils.lazySupplier(
+            () -> Metrics.newCounter(new MetricName("spanLogs." + handle, "", "discarded")));
+    this.discardedSpansBySampler =
+        Utils.lazySupplier(
+            () -> Metrics.newCounter(new MetricName("spans." + handle, "", "sampler.discarded")));
+    this.discardedSpanLogsBySampler =
+        Utils.lazySupplier(
+            () ->
+                Metrics.newCounter(new MetricName("spanLogs." + handle, "", "sampler.discarded")));
+    this.receivedSpansTotal =
+        Utils.lazySupplier(
+            () -> Metrics.newCounter(new MetricName("spans." + handle, "", "received.total")));
+    this.receivedLogsCounter =
+        Caffeine.newBuilder()
+            .build(
+                format ->
+                    Metrics.newCounter(
+                        new TaggedMetricName(
+                            "logs." + handle,
+                            "received" + ".total",
+                            "format",
+                            format.name().toLowerCase())));
+    this.discardedLogsCounter =
+        Caffeine.newBuilder()
+            .build(
+                format ->
+                    Metrics.newCounter(
+                        new TaggedMetricName(
+                            "logs." + handle, "discarded", "format", format.name().toLowerCase())));
+    this.discardedLogsMissingLogServerInfoCounter =
+        Caffeine.newBuilder()
+            .build(
+                format ->
+                    Metrics.newCounter(
+                        new TaggedMetricName(
+                            "logs." + handle,
+                            "discarded.log.server.info.missing",
+                            "format",
+                            format.name().toLowerCase())));
+  }
+
+  @Override
+  protected DataFormat getFormat(FullHttpRequest httpRequest) {
+    return DataFormat.parse(
+        URLEncodedUtils.parse(URI.create(httpRequest.uri()), CharsetUtil.UTF_8).stream()
+            .filter(x -> x.getName().equals("format") || x.getName().equals("f"))
+            .map(NameValuePair::getValue)
+            .findFirst()
+            .orElse(null));
+  }
+
+  @Override
+  protected void handleHttpMessage(ChannelHandlerContext ctx, FullHttpRequest request) {
+    StringBuilder out = new StringBuilder();
+    DataFormat format = getFormat(request);
+    if ((format == HISTOGRAM
+            && isFeatureDisabled(
+                histogramDisabled, HISTO_DISABLED, discardedHistograms.get(), out, request))
+        || (format == SPAN_LOG
+            && isFeatureDisabled(
+                spanLogsDisabled, SPANLOGS_DISABLED, discardedSpanLogs.get(), out, request))) {
+      writeHttpResponse(ctx, HttpResponseStatus.FORBIDDEN, out, request);
+      return;
+    } else if (format == SPAN
+        && isFeatureDisabled(traceDisabled, SPAN_DISABLED, discardedSpans.get(), out, request)) {
+      receivedSpansTotal.get().inc(discardedSpans.get().count());
+      writeHttpResponse(ctx, HttpResponseStatus.FORBIDDEN, out, request);
+      return;
+    } else if ((LOGS_DATA_FORMATS.contains(format))
+        && isFeatureDisabled(
+            logsDisabled, LOGS_DISABLED, discardedLogsCounter.get(format), out, request)) {
+      receivedLogsCounter.get(format).inc(discardedLogsCounter.get(format).count());
+      writeHttpResponse(ctx, HttpResponseStatus.FORBIDDEN, out, request);
+      return;
+    }
+    super.handleHttpMessage(ctx, request);
   }
 
   /**
-   *
-   * @param ctx      ChannelHandler context (to retrieve remote client's IP in case of errors)
-   * @param message  line being processed
+   * @param ctx ChannelHandler context (to retrieve remote client's IP in case of errors)
+   * @param message line being processed
    */
   @Override
-  @SuppressWarnings("unchecked")
-  protected void processLine(final ChannelHandlerContext ctx, String message) {
-    if (message.isEmpty()) return;
-
-    if (message.startsWith(ReportSourceTagDecoder.SOURCE_TAG) ||
-        message.startsWith(ReportSourceTagDecoder.SOURCE_DESCRIPTION)) {
-      if (sourceTagHandler == null) {
-        synchronized(this) {
-          if (sourceTagHandler == null && handlerFactory != null && decoders != null) {
-            sourceTagHandler = handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.SOURCE_TAG, handle));
-            sourceTagDecoder = decoders.get(ReportableEntityType.SOURCE_TAG);
+  protected void processLine(
+      final ChannelHandlerContext ctx, @Nonnull String message, @Nullable DataFormat format) {
+
+    if (message.contains("\04")) {
+      wavefrontHandler.reject(message, "'EOT' character is not allowed!");
+    }
+
+    DataFormat dataFormat = format == null ? DataFormat.autodetect(message) : format;
+    switch (dataFormat) {
+      case SOURCE_TAG:
+        ReportableEntityHandler sourceTagHandler =
+            sourceTagHandlerSupplier.get();
+        if (sourceTagHandler == null || sourceTagDecoder == null) {
+          wavefrontHandler.reject(
+              message, "Port is not configured to accept " + "sourceTag-formatted data!");
+          return;
+        }
+        List output = new ArrayList<>(1);
+        try {
+          sourceTagDecoder.decode(message, output, "dummy");
+          for (ReportSourceTag tag : output) {
+            sourceTagHandler.report(tag);
           }
-          if (sourceTagHandler == null || sourceTagDecoder == null) {
-            wavefrontHandler.reject(message, "Port is not configured to accept sourceTag-formatted data!");
-            return;
+        } catch (Exception e) {
+          sourceTagHandler.reject(
+              message,
+              formatErrorMessage("WF-300 Cannot parse sourceTag: \"" + message + "\"", e, ctx));
+        }
+        return;
+      case EVENT:
+        ReportableEntityHandler eventHandler = eventHandlerSupplier.get();
+        if (eventHandler == null || eventDecoder == null) {
+          wavefrontHandler.reject(message, "Port is not configured to accept event data!");
+          return;
+        }
+        List events = new ArrayList<>(1);
+        try {
+          eventDecoder.decode(message, events, "dummy");
+          for (ReportEvent event : events) {
+            eventHandler.report(event);
           }
+        } catch (Exception e) {
+          eventHandler.reject(
+              message,
+              formatErrorMessage("WF-300 Cannot parse event: \"" + message + "\"", e, ctx));
         }
-      }
-      List output = Lists.newArrayListWithCapacity(1);
-      try {
-        sourceTagDecoder.decode(message, output, "dummy");
-        for(ReportSourceTag tag : output) {
-          sourceTagHandler.report(tag);
+        return;
+      case SPAN:
+        ReportableEntityHandler spanHandler = spanHandlerSupplier.get();
+        if (spanHandler == null || spanDecoder == null) {
+          wavefrontHandler.reject(
+              message, "Port is not configured to accept " + "tracing data (spans)!");
+          return;
+        }
+        message = annotator == null ? message : annotator.apply(ctx, message);
+        receivedSpansTotal.get().inc();
+        preprocessAndHandleSpan(
+            message,
+            spanDecoder,
+            spanHandler,
+            spanHandler::report,
+            preprocessorSupplier,
+            ctx,
+            span -> sampler.sample(span, discardedSpansBySampler.get()));
+        return;
+      case SPAN_LOG:
+        if (isFeatureDisabled(spanLogsDisabled, SPANLOGS_DISABLED, discardedSpanLogs.get())) return;
+        ReportableEntityHandler spanLogsHandler = spanLogsHandlerSupplier.get();
+        if (spanLogsHandler == null || spanLogsDecoder == null || spanDecoder == null) {
+          wavefrontHandler.reject(
+              message, "Port is not configured to accept " + "tracing data (span logs)!");
+          return;
+        }
+        handleSpanLogs(
+            message,
+            spanLogsDecoder,
+            spanDecoder,
+            spanLogsHandler,
+            preprocessorSupplier,
+            ctx,
+            span -> sampler.sample(span, discardedSpanLogsBySampler.get()));
+        return;
+      case HISTOGRAM:
+        if (isFeatureDisabled(histogramDisabled, HISTO_DISABLED, discardedHistograms.get())) return;
+        ReportableEntityHandler histogramHandler =
+            histogramHandlerSupplier.get();
+        if (histogramHandler == null || histogramDecoder == null) {
+          wavefrontHandler.reject(
+              message, "Port is not configured to accept " + "histogram-formatted data!");
+          return;
         }
-      } catch (Exception e) {
-        sourceTagHandler.reject(message, formatErrorMessage("WF-300 Cannot parse: \"" + message + "\"", e, ctx));
+        message = annotator == null ? message : annotator.apply(ctx, message);
+        preprocessAndHandlePoint(
+            message, histogramDecoder, histogramHandler, preprocessorSupplier, ctx, "histogram");
+        return;
+      case LOGS_JSON_ARR:
+      case LOGS_JSON_LINES:
+      case LOGS_JSON_CLOUDWATCH:
+        receivedLogsCounter.get(format).inc();
+        if (isFeatureDisabled(logsDisabled, LOGS_DISABLED, discardedLogsCounter.get(format)))
+          return;
+        if (isMissingLogServerInfoForAConvergedCSPTenant(
+            receivedLogServerDetails,
+            enableHyperlogsConvergedCsp,
+            LOGS_SERVER_DETAILS_MISSING,
+            discardedLogsMissingLogServerInfoCounter.get(format))) return;
+        ReportableEntityHandler logHandler = logHandlerSupplier.get();
+        if (logHandler == null || logDecoder == null) {
+          wavefrontHandler.reject(message, "Port is not configured to accept log data!");
+          return;
+        }
+        logHandler.setLogFormat(format);
+        message = annotator == null ? message : annotator.apply(ctx, message, true);
+        preprocessAndHandleLog(message, logDecoder, logHandler, preprocessorSupplier, ctx);
+        return;
+      default:
+        message = annotator == null ? message : annotator.apply(ctx, message);
+        preprocessAndHandlePoint(
+            message, wavefrontDecoder, wavefrontHandler, preprocessorSupplier, ctx, "metric");
+    }
+  }
+
+  public static void preprocessAndHandlePoint(
+      String message,
+      ReportableEntityDecoder decoder,
+      ReportableEntityHandler handler,
+      @Nullable Supplier preprocessorSupplier,
+      @Nullable ChannelHandlerContext ctx,
+      String type) {
+    ReportableEntityPreprocessor preprocessor =
+        preprocessorSupplier == null ? null : preprocessorSupplier.get();
+    String[] messageHolder = new String[1];
+    // transform the line if needed
+    if (preprocessor != null) {
+      message = preprocessor.forPointLine().transform(message);
+
+      // apply white/black lists after formatting
+      if (!preprocessor.forPointLine().filter(message, messageHolder)) {
+        if (messageHolder[0] != null) {
+          handler.reject((ReportPoint) null, message);
+        } else {
+          handler.block(null, message);
+        }
+        return;
       }
+    }
+
+    List output = new ArrayList<>(1);
+    try {
+      decoder.decode(message, output, "dummy");
+    } catch (Exception e) {
+      handler.reject(
+          message,
+          formatErrorMessage("WF-300 Cannot parse " + type + ": \"" + message + "\"", e, ctx));
       return;
     }
 
-    ReportableEntityHandler handler = wavefrontHandler;
-    ReportableEntityDecoder decoder = wavefrontDecoder;
-    message = annotator == null ? message : annotator.apply(ctx, message);
-
-    if (message.startsWith("!M ") || message.startsWith("!H ") || message.startsWith("!D ")) {
-      if (histogramHandler == null) {
-        synchronized(this) {
-          if (histogramHandler == null && handlerFactory != null && decoders != null) {
-            histogramHandler = handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM,
-                handle + "-histograms"));
-            histogramDecoder = decoders.get(ReportableEntityType.HISTOGRAM);
-          }
-          if (histogramHandler == null || histogramDecoder == null) {
-            wavefrontHandler.reject(message, "Port is not configured to accept histogram-formatted data!");
-            return;
+    for (ReportPoint object : output) {
+      if (preprocessor != null) {
+        preprocessor.forReportPoint().transform(object);
+        if (!preprocessor.forReportPoint().filter(object, messageHolder)) {
+          if (messageHolder[0] != null) {
+            handler.reject(object, messageHolder[0]);
+          } else {
+            handler.block(object);
           }
+          return;
         }
       }
-      handler = histogramHandler;
-      decoder = histogramDecoder;
+      handler.report(object);
     }
+  }
+
+  public static void preprocessAndHandleLog(
+      String message,
+      ReportableEntityDecoder decoder,
+      ReportableEntityHandler handler,
+      @Nullable Supplier preprocessorSupplier,
+      @Nullable ChannelHandlerContext ctx) {
+    ReportableEntityPreprocessor preprocessor =
+        preprocessorSupplier == null ? null : preprocessorSupplier.get();
 
+    String[] messageHolder = new String[1];
     // transform the line if needed
     if (preprocessor != null) {
       message = preprocessor.forPointLine().transform(message);
-
       // apply white/black lists after formatting
-      if (!preprocessor.forPointLine().filter(message)) {
-        if (preprocessor.forPointLine().getLastFilterResult() != null) {
-          handler.reject((ReportPoint) null, message);
+      if (!preprocessor.forPointLine().filter(message, messageHolder)) {
+        if (messageHolder[0] != null) {
+          handler.reject((ReportLog) null, message);
         } else {
           handler.block(null, message);
         }
@@ -151,20 +472,27 @@ protected void processLine(final ChannelHandlerContext ctx, String message) {
       }
     }
 
-    List output = Lists.newArrayListWithCapacity(1);
+    List output = new ArrayList<>(1);
     try {
       decoder.decode(message, output, "dummy");
     } catch (Exception e) {
-      handler.reject(message, formatErrorMessage("WF-300 Cannot parse: \"" + message + "\"", e, ctx));
+      handler.reject(
+          message, formatErrorMessage("WF-600 Cannot parse Log: \"" + message + "\"", e, ctx));
       return;
     }
 
-    for (ReportPoint object : output) {
+    if (output.get(0) == null) {
+      handler.reject(
+          message, formatErrorMessage("WF-600 Cannot parse Log: \"" + message + "\"", null, ctx));
+      return;
+    }
+
+    for (ReportLog object : output) {
       if (preprocessor != null) {
-        preprocessor.forReportPoint().transform(object);
-        if (!preprocessor.forReportPoint().filter(object)) {
-          if (preprocessor.forReportPoint().getLastFilterResult() != null) {
-            handler.reject(object, preprocessor.forReportPoint().getLastFilterResult());
+        preprocessor.forReportLog().transform(object);
+        if (!preprocessor.forReportLog().filter(object, messageHolder)) {
+          if (messageHolder[0] != null) {
+            handler.reject(object, messageHolder[0]);
           } else {
             handler.block(object);
           }
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/WriteHttpJsonMetricsEndpoint.java b/proxy/src/main/java/com/wavefront/agent/listeners/WriteHttpJsonMetricsEndpoint.java
deleted file mode 100644
index d062ab431..000000000
--- a/proxy/src/main/java/com/wavefront/agent/listeners/WriteHttpJsonMetricsEndpoint.java
+++ /dev/null
@@ -1,209 +0,0 @@
-package com.wavefront.agent.listeners;
-
-import com.google.common.collect.Lists;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.wavefront.agent.PointHandler;
-import com.wavefront.agent.PointHandlerImpl;
-import com.wavefront.agent.PostPushDataTimedTask;
-import com.wavefront.agent.preprocessor.ReportableEntityPreprocessor;
-import com.wavefront.ingester.GraphiteDecoder;
-
-import org.eclipse.jetty.server.Request;
-import org.eclipse.jetty.server.handler.AbstractHandler;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Agent-side JSON metrics endpoint for parsing JSON from write_http collectd plugin.
- *
- * @see https://collectd.org/wiki/index.php/Plugin:Write_HTTP
- */
-public class WriteHttpJsonMetricsEndpoint extends AbstractHandler {
-
-  protected static final Logger logger = Logger.getLogger("agent");
-  private static final Logger blockedPointsLogger = Logger.getLogger("RawBlockedPoints");
-
-  @Nullable
-  private final String prefix;
-  private final String defaultHost;
-  @Nullable
-  private final ReportableEntityPreprocessor preprocessor;
-  private final PointHandler handler;
-
-  /**
-   *  Graphite decoder to re-parse modified points
-   */
-  private final GraphiteDecoder recoder = new GraphiteDecoder(Collections.emptyList());
-
-
-  public WriteHttpJsonMetricsEndpoint(final String port, final String host,
-                                      @Nullable
-                                      final String prefix, final String validationLevel,
-                                      final int blockedPointsPerBatch, PostPushDataTimedTask[] postPushDataTimedTasks,
-                                      @Nullable final ReportableEntityPreprocessor preprocessor) {
-    this.handler = new PointHandlerImpl(port, validationLevel, blockedPointsPerBatch, postPushDataTimedTasks);
-    this.prefix = prefix;
-    this.defaultHost = host;
-    this.preprocessor = preprocessor;
-  }
-
-  @Override
-  public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
-      throws IOException, ServletException {
-    response.setContentType("text/html;charset=utf-8");
-
-    JsonNode metrics = new ObjectMapper().readTree(request.getReader());
-
-    if (!metrics.isArray()) {
-      logger.warning("metrics is not an array!");
-      handler.handleBlockedPoint("[metrics] is not an array!");
-      response.setStatus(HttpServletResponse.SC_BAD_REQUEST); // return HTTP 400
-      baseRequest.setHandled(true);
-      return;
-    }
-
-    for (final JsonNode metric : metrics) {
-      try {
-        JsonNode host = metric.get("host");
-        String hostName;
-        if (host != null) {
-          hostName = host.textValue();
-          if (hostName == null || hostName.isEmpty()) {
-            hostName = defaultHost;
-          }
-        } else {
-          hostName = defaultHost;
-        }
-
-        JsonNode time = metric.get("time");
-        long ts = 0;
-        if (time != null) {
-          ts = time.asLong() * 1000;
-        }
-        JsonNode values = metric.get("values");
-        if (values == null) {
-          handler.handleBlockedPoint("[values] missing in JSON object");
-          logger.warning("Skipping.  Missing values.");
-          continue;
-        }
-        int index = 0;
-        for (final JsonNode value : values) {
-          String metricName = getMetricName(metric, index);
-          ReportPoint.Builder builder = ReportPoint.newBuilder()
-              .setMetric(metricName)
-              .setTable("dummy")
-              .setTimestamp(ts)
-              .setHost(hostName);
-          if (value.isDouble()) {
-            builder.setValue(value.asDouble());
-          } else {
-            builder.setValue(value.asLong());
-          }
-          List parsedPoints = Lists.newArrayListWithExpectedSize(1);
-          ReportPoint point = builder.build();
-          if (preprocessor != null && preprocessor.forPointLine().hasTransformers()) {
-            //
-            String pointLine = PointHandlerImpl.pointToString(point);
-            pointLine = preprocessor.forPointLine().transform(pointLine);
-            recoder.decodeReportPoints(pointLine, parsedPoints, "dummy");
-          } else {
-            parsedPoints.add(point);
-          }
-          for (ReportPoint parsedPoint : parsedPoints) {
-            if (preprocessor != null) {
-              preprocessor.forReportPoint().transform(parsedPoint);
-              if (!preprocessor.forReportPoint().filter(parsedPoint)) {
-                if (preprocessor.forReportPoint().getLastFilterResult() != null) {
-                  blockedPointsLogger.warning(PointHandlerImpl.pointToString(parsedPoint));
-                } else {
-                  blockedPointsLogger.info(PointHandlerImpl.pointToString(parsedPoint));
-                }
-                handler.handleBlockedPoint(preprocessor.forReportPoint().getLastFilterResult());
-                continue;
-              }
-            }
-            handler.reportPoint(parsedPoint, "write_http json: " + PointHandlerImpl.pointToString(parsedPoint));
-          }
-          index++;
-        }
-      } catch (final Exception e) {
-        handler.handleBlockedPoint("Failed adding metric: " + e);
-        logger.log(Level.WARNING, "Failed adding metric", e);
-        response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
-        baseRequest.setHandled(true);
-        return;
-      }
-    }
-    response.setStatus(HttpServletResponse.SC_OK);
-    baseRequest.setHandled(true);
-  }
-
-  /**
-   * Generates a metric name from json format:
-     {
-     "values": [197141504, 175136768],
-     "dstypes": ["counter", "counter"],
-     "dsnames": ["read", "write"],
-     "time": 1251533299,
-     "interval": 10,
-     "host": "leeloo.lan.home.verplant.org",
-     "plugin": "disk",
-     "plugin_instance": "sda",
-     "type": "disk_octets",
-     "type_instance": ""
-    }
-
-    host "/" plugin ["-" plugin instance] "/" type ["-" type instance] =>
-    {plugin}[.{plugin_instance}].{type}[.{type_instance}]
-  */
-  private String getMetricName(final JsonNode metric, int index) {
-    JsonNode plugin = metric.get("plugin");
-    JsonNode plugin_instance = metric.get("plugin_instance");
-    JsonNode type = metric.get("type");
-    JsonNode type_instance = metric.get("type_instance");
-
-    if (plugin == null || type == null) {
-      throw new IllegalArgumentException("plugin or type is missing");
-    }
-
-    StringBuilder sb = new StringBuilder();
-    sb.append(plugin.textValue());
-    sb.append('.');
-    if (plugin_instance != null) {
-      String value = plugin_instance.textValue();
-      if (value != null && !value.isEmpty()) {
-        sb.append(value);
-        sb.append('.');
-      }
-    }
-    sb.append(type.textValue());
-    sb.append('.');
-    if (type_instance != null) {
-      String value = type_instance.textValue();
-      if (value != null && !value.isEmpty()) {
-        sb.append(value);
-        sb.append('.');
-      }
-    }
-
-    JsonNode dsnames = metric.get("dsnames");
-    if (dsnames == null || !dsnames.isArray() || dsnames.size() <= index) {
-      throw new IllegalArgumentException("dsnames is not set");
-    }
-    sb.append(dsnames.get(index).textValue());
-    return sb.toString();
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/WriteHttpJsonPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/WriteHttpJsonPortUnificationHandler.java
new file mode 100644
index 000000000..3ac964efd
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/WriteHttpJsonPortUnificationHandler.java
@@ -0,0 +1,230 @@
+package com.wavefront.agent.listeners;
+
+import static com.wavefront.agent.channel.ChannelUtils.errorMessageWithRootCause;
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.ingester.GraphiteDecoder;
+import com.wavefront.ingester.ReportPointSerializer;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.util.CharsetUtil;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import wavefront.report.ReportPoint;
+
+/**
+ * This class handles incoming messages in write_http format.
+ *
+ * @author Clement Pang (clement@wavefront.com).
+ * @author vasily@wavefront.com
+ */
+@ChannelHandler.Sharable
+public class WriteHttpJsonPortUnificationHandler extends AbstractHttpOnlyHandler {
+  private static final Logger logger =
+      Logger.getLogger(WriteHttpJsonPortUnificationHandler.class.getCanonicalName());
+
+  /**
+   * The point handler that takes report metrics one data point at a time and handles batching and
+   * retries, etc
+   */
+  private final ReportableEntityHandler pointHandler;
+
+  private final String defaultHost;
+
+  @Nullable private final Supplier preprocessorSupplier;
+  private final ObjectMapper jsonParser;
+  /** Graphite decoder to re-parse modified points. */
+  private final GraphiteDecoder recoder = new GraphiteDecoder(Collections.emptyList());
+
+  /**
+   * Create a new instance.
+   *
+   * @param handle handle/port number.
+   * @param healthCheckManager shared health check endpoint handler.
+   * @param handlerFactory factory for ReportableEntityHandler objects.
+   * @param defaultHost default host name to use, if none specified.
+   * @param preprocessor preprocessor.
+   */
+  public WriteHttpJsonPortUnificationHandler(
+      final String handle,
+      final TokenAuthenticator authenticator,
+      final HealthCheckManager healthCheckManager,
+      final ReportableEntityHandlerFactory handlerFactory,
+      final String defaultHost,
+      @Nullable final Supplier preprocessor) {
+    this(
+        handle,
+        authenticator,
+        healthCheckManager,
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle)),
+        defaultHost,
+        preprocessor);
+  }
+
+  @VisibleForTesting
+  protected WriteHttpJsonPortUnificationHandler(
+      final String handle,
+      final TokenAuthenticator authenticator,
+      final HealthCheckManager healthCheckManager,
+      final ReportableEntityHandler pointHandler,
+      final String defaultHost,
+      @Nullable final Supplier preprocessor) {
+    super(authenticator, healthCheckManager, handle);
+    this.pointHandler = pointHandler;
+    this.defaultHost = defaultHost;
+    this.preprocessorSupplier = preprocessor;
+    this.jsonParser = new ObjectMapper();
+  }
+
+  @Override
+  protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request) {
+    HttpResponseStatus status = HttpResponseStatus.OK;
+    String requestBody = request.content().toString(CharsetUtil.UTF_8);
+    try {
+      JsonNode metrics = jsonParser.readTree(requestBody);
+      if (!metrics.isArray()) {
+        logger.warning("metrics is not an array!");
+        pointHandler.reject((ReportPoint) null, "[metrics] is not an array!");
+        status = HttpResponseStatus.BAD_REQUEST;
+        writeHttpResponse(ctx, status, "", request);
+        return;
+      }
+      reportMetrics(metrics);
+      writeHttpResponse(ctx, status, "", request);
+    } catch (Exception e) {
+      status = HttpResponseStatus.BAD_REQUEST;
+      logWarning("WF-300: Failed to handle incoming write_http request", e, ctx);
+      writeHttpResponse(ctx, status, errorMessageWithRootCause(e), request);
+    }
+  }
+
+  private void reportMetrics(JsonNode metrics) {
+    ReportableEntityPreprocessor preprocessor =
+        preprocessorSupplier == null ? null : preprocessorSupplier.get();
+    String[] messageHolder = new String[1];
+    for (final JsonNode metric : metrics) {
+      JsonNode host = metric.get("host");
+      String hostName;
+      if (host != null) {
+        hostName = host.textValue();
+        if (hostName == null || hostName.isEmpty()) {
+          hostName = defaultHost;
+        }
+      } else {
+        hostName = defaultHost;
+      }
+
+      JsonNode time = metric.get("time");
+      long ts = 0;
+      if (time != null) {
+        ts = time.asLong() * 1000;
+      }
+      JsonNode values = metric.get("values");
+      if (values == null) {
+        pointHandler.reject((ReportPoint) null, "[values] missing in JSON object");
+        logger.warning("Skipping - [values] missing in JSON object.");
+        continue;
+      }
+      int index = 0;
+      for (final JsonNode value : values) {
+        String metricName = getMetricName(metric, index);
+        ReportPoint.Builder builder =
+            ReportPoint.newBuilder()
+                .setMetric(metricName)
+                .setTable("dummy")
+                .setTimestamp(ts)
+                .setHost(hostName);
+        if (value.isDouble()) {
+          builder.setValue(value.asDouble());
+        } else {
+          builder.setValue(value.asLong());
+        }
+        List parsedPoints = new ArrayList<>(1);
+        ReportPoint point = builder.build();
+        if (preprocessor != null && preprocessor.forPointLine().getTransformers().size() > 0) {
+          //
+          String pointLine = ReportPointSerializer.pointToString(point);
+          pointLine = preprocessor.forPointLine().transform(pointLine);
+          recoder.decodeReportPoints(pointLine, parsedPoints, "dummy");
+        } else {
+          parsedPoints.add(point);
+        }
+        for (ReportPoint parsedPoint : parsedPoints) {
+          if (preprocessor != null) {
+            preprocessor.forReportPoint().transform(point);
+            if (!preprocessor.forReportPoint().filter(point, messageHolder)) {
+              if (messageHolder[0] != null) {
+                pointHandler.reject(point, messageHolder[0]);
+              } else {
+                pointHandler.block(point);
+              }
+              continue;
+            }
+          }
+          pointHandler.report(parsedPoint);
+        }
+        index++;
+      }
+    }
+  }
+
+  /**
+   * Generates a metric name from json format: { "values": [197141504, 175136768], "dstypes":
+   * ["counter", "counter"], "dsnames": ["read", "write"], "time": 1251533299, "interval": 10,
+   * "host": "leeloo.lan.home.verplant.org", "plugin": "disk", "plugin_instance": "sda", "type":
+   * "disk_octets", "type_instance": "" }
+   *
+   * 
host "/" plugin ["-" plugin instance] "/" type ["-" type instance] =>
+   * {plugin}[.{plugin_instance}].{type}[.{type_instance}]
+   */
+  private static String getMetricName(final JsonNode metric, int index) {
+    JsonNode plugin = metric.get("plugin");
+    JsonNode plugin_instance = metric.get("plugin_instance");
+    JsonNode type = metric.get("type");
+    JsonNode type_instance = metric.get("type_instance");
+
+    if (plugin == null || type == null) {
+      throw new IllegalArgumentException("plugin or type is missing");
+    }
+
+    StringBuilder sb = new StringBuilder();
+    extractMetricFragment(plugin, plugin_instance, sb);
+    extractMetricFragment(type, type_instance, sb);
+
+    JsonNode dsnames = metric.get("dsnames");
+    if (dsnames == null || !dsnames.isArray() || dsnames.size() <= index) {
+      throw new IllegalArgumentException("dsnames is not set");
+    }
+    sb.append(dsnames.get(index).textValue());
+    return sb.toString();
+  }
+
+  private static void extractMetricFragment(
+      JsonNode node, JsonNode instance_node, StringBuilder sb) {
+    sb.append(node.textValue());
+    sb.append('.');
+    if (instance_node != null) {
+      String value = instance_node.textValue();
+      if (value != null && !value.isEmpty()) {
+        sb.append(value);
+        sb.append('.');
+      }
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/ZipkinPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/ZipkinPortUnificationHandler.java
deleted file mode 100644
index b87c24b0a..000000000
--- a/proxy/src/main/java/com/wavefront/agent/listeners/ZipkinPortUnificationHandler.java
+++ /dev/null
@@ -1,254 +0,0 @@
-package com.wavefront.agent.listeners;
-
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.util.concurrent.RateLimiter;
-
-import com.wavefront.agent.Utils;
-import com.wavefront.agent.auth.TokenAuthenticatorBuilder;
-import com.wavefront.agent.auth.TokenValidationMethod;
-import com.wavefront.agent.handlers.HandlerKey;
-import com.wavefront.agent.handlers.ReportableEntityHandler;
-import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
-import com.wavefront.common.TraceConstants;
-import com.wavefront.data.ReportableEntityType;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
-
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.http.FullHttpRequest;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import wavefront.report.Annotation;
-import wavefront.report.Span;
-import zipkin2.SpanBytesDecoderDetector;
-import zipkin2.codec.BytesDecoder;
-
-import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
-import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
-import static com.wavefront.sdk.common.Constants.SOURCE_KEY;
-
-/**
- * Handler that processes Zipkin trace data over HTTP and converts them to Wavefront format.
- *
- * @author Anil Kodali (akodali@vmware.com)
- */
-public class ZipkinPortUnificationHandler extends PortUnificationHandler {
-  private static final Logger logger = Logger.getLogger(
-      ZipkinPortUnificationHandler.class.getCanonicalName());
-  private final String handle;
-  private final ReportableEntityHandler handler;
-  private final AtomicBoolean traceDisabled;
-  private final RateLimiter warningLoggerRateLimiter = RateLimiter.create(0.2);
-  private final Counter discardedBatches;
-  private final Counter processedBatches;
-  private final Counter failedBatches;
-
-  private final static Set ZIPKIN_VALID_PATHS = ImmutableSet.of(
-      "/api/v1/spans/",
-      "/api/v2/spans/");
-  private final static String ZIPKIN_VALID_HTTP_METHOD = "POST";
-  private final static String DEFAULT_APPLICATION = "Zipkin";
-  private final static String DEFAULT_SOURCE = "zipkin";
-  private final static String DEFAULT_SERVICE = "defaultService";
-  private final static String DEFAULT_SPAN_NAME = "defaultOperation";
-
-  private static final Logger zipkinDataLogger = Logger.getLogger("ZipkinDataLogger");
-
-  public ZipkinPortUnificationHandler(String handle,
-                                      ReportableEntityHandlerFactory handlerFactory,
-                                      AtomicBoolean traceDisabled) {
-    this(handle,
-        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)),
-        traceDisabled);
-  }
-
-  public ZipkinPortUnificationHandler(final String handle,
-                                      ReportableEntityHandler handler,
-                                      AtomicBoolean traceDisabled) {
-    super(TokenAuthenticatorBuilder.create().setTokenValidationMethod(TokenValidationMethod.NONE).build(),
-        handle, false, true);
-    this.handle = handle;
-    this.handler = handler;
-    this.traceDisabled = traceDisabled;
-    this.discardedBatches = Metrics.newCounter(new MetricName(
-        "spans." + handle + ".batches", "", "discarded"));
-    this.processedBatches = Metrics.newCounter(new MetricName(
-        "spans." + handle + ".batches", "", "processed"));
-    this.failedBatches = Metrics.newCounter(new MetricName(
-        "spans." + handle + ".batches", "", "failed"));
-  }
-
-  @Override
-  protected void handleHttpMessage(final ChannelHandlerContext ctx,
-                                   final FullHttpRequest incomingRequest) {
-    URI uri = parseUri(ctx, incomingRequest);
-    if (uri == null) return;
-
-    String path = uri.getPath().endsWith("/") ? uri.getPath() : uri.getPath() + "/";
-
-    // Validate Uri Path and HTTP method of incoming Zipkin spans.
-    if (!ZIPKIN_VALID_PATHS.contains(path)) {
-      writeHttpResponse(ctx, HttpResponseStatus.BAD_REQUEST, "Unsupported URL path.", incomingRequest);
-      logWarning("WF-400: Requested URI path '" + path + "' is not supported.", null, ctx);
-      return;
-    }
-    if (!incomingRequest.method().toString().equalsIgnoreCase(ZIPKIN_VALID_HTTP_METHOD)) {
-      writeHttpResponse(ctx, HttpResponseStatus.BAD_REQUEST, "Unsupported Http method.", incomingRequest);
-      logWarning("WF-400: Requested http method '" + incomingRequest.method().toString() +
-          "' is not supported.", null, ctx);
-      return;
-    }
-
-    HttpResponseStatus status;
-    StringBuilder output = new StringBuilder();
-
-    // Handle case when tracing is disabled, ignore reported spans.
-    if (traceDisabled.get()) {
-      if (warningLoggerRateLimiter.tryAcquire()) {
-        logger.info("Ingested spans discarded because tracing feature is not enabled on the " +
-            "server");
-      }
-      discardedBatches.inc();
-      output.append("Ingested spans discarded because tracing feature is not enabled on the " +
-          "server.");
-      status = HttpResponseStatus.ACCEPTED;
-      writeHttpResponse(ctx, status, output, incomingRequest);
-      return;
-    }
-
-    try {
-      byte[] bytesArray = new byte[incomingRequest.content().nioBuffer().remaining()];
-      incomingRequest.content().nioBuffer().get(bytesArray, 0, bytesArray.length);
-      BytesDecoder decoder = SpanBytesDecoderDetector.decoderForListMessage(bytesArray);
-      List zipkinSpanSink = new ArrayList<>();
-      decoder.decodeList(bytesArray, zipkinSpanSink);
-      processZipkinSpans(zipkinSpanSink);
-      status = HttpResponseStatus.ACCEPTED;
-      processedBatches.inc();
-    } catch (Exception e) {
-      failedBatches.inc();
-      writeExceptionText(e, output);
-      status = HttpResponseStatus.BAD_REQUEST;
-      logger.log(Level.WARNING, "Zipkin batch processing failed", Throwables.getRootCause(e));
-    }
-    writeHttpResponse(ctx, status, output, incomingRequest);
-  }
-
-  private void processZipkinSpans(List zipkinSpans) {
-    for (zipkin2.Span zipkinSpan : zipkinSpans) {
-      processZipkinSpan(zipkinSpan);
-    }
-  }
-
-  private void processZipkinSpan(zipkin2.Span zipkinSpan) {
-    if (zipkinDataLogger.isLoggable(Level.FINEST)) {
-      zipkinDataLogger.info("Inbound Zipkin span: " + zipkinSpan.toString());
-    }
-    // Add application tags, span references , span kind and http uri, responses etc.
-    List annotations = addAnnotations(zipkinSpan);
-
-    /** Add source of the span following the below:
-     *    1. If "source" is provided by span tags , use it else
-     *    2. Set "source" to local service endpoint's ipv4 address, else
-     *    3. Default "source" to "zipkin".
-     */
-    String sourceName = DEFAULT_SOURCE;
-    if (zipkinSpan.tags() != null && zipkinSpan.tags().size() > 0) {
-      if (zipkinSpan.tags().get(SOURCE_KEY) != null) {
-        sourceName = zipkinSpan.tags().get(SOURCE_KEY);
-      } else if (zipkinSpan.localEndpoint() != null && zipkinSpan.localEndpoint().ipv4() != null) {
-        sourceName = zipkinSpan.localEndpoint().ipv4();
-      }
-    }
-    // Set spanName.
-    String spanName = zipkinSpan.name() == null ? DEFAULT_SPAN_NAME : zipkinSpan.name();
-
-    //Build wavefront span
-    Span newSpan = Span.newBuilder().
-        setCustomer("dummy").
-        setName(spanName).
-        setSource(sourceName).
-        setSpanId(Utils.convertToUuidString(zipkinSpan.id())).
-        setTraceId(Utils.convertToUuidString(zipkinSpan.traceId())).
-        setStartMillis(zipkinSpan.timestampAsLong() / 1000).
-        setDuration(zipkinSpan.durationAsLong() / 1000).
-        setAnnotations(annotations).
-        build();
-
-    // Log Zipkin spans as well as Wavefront spans for debugging purposes.
-    if (zipkinDataLogger.isLoggable(Level.FINEST)) {
-      zipkinDataLogger.info("Converted Wavefront span: " + newSpan.toString());
-    }
-
-    handler.report(newSpan);
-  }
-
-  private List addAnnotations(zipkin2.Span zipkinSpan) {
-    List annotations = new ArrayList<>();
-
-    // Set Span's References.
-    if (zipkinSpan.parentId() != null) {
-      annotations.add(new Annotation(TraceConstants.PARENT_KEY,
-          Utils.convertToUuidString(zipkinSpan.parentId())));
-    }
-
-    // Set Span Kind.
-    if (zipkinSpan.kind() != null) {
-      annotations.add(new Annotation("span.kind", zipkinSpan.kind().toString().toLowerCase()));
-    }
-
-    // Set Span's service name.
-    String serviceName = zipkinSpan.localServiceName() == null ? DEFAULT_SERVICE :
-        zipkinSpan.localServiceName();
-    annotations.add(new Annotation(SERVICE_TAG_KEY, serviceName));
-
-    // Set Span's Application Tag.
-    // Mandatory tags are com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY and
-    // com.wavefront.sdk.common.Constants.SOURCE_KEY for which we declare defaults.
-    addTagWithKey(zipkinSpan, annotations, APPLICATION_TAG_KEY, DEFAULT_APPLICATION);
-
-    // Set all other Span Tags.
-    addSpanTags(zipkinSpan, annotations, ImmutableList.of(APPLICATION_TAG_KEY, SOURCE_KEY));
-    return annotations;
-  }
-
-  private static void addSpanTags(zipkin2.Span zipkinSpan,
-                                  List annotations,
-                                  List ignoreKeys) {
-    if (zipkinSpan.tags() != null && zipkinSpan.tags().size() > 0) {
-      for (Map.Entry tag : zipkinSpan.tags().entrySet()) {
-        if (!ignoreKeys.contains(tag.getKey().toLowerCase())) {
-          annotations.add(new Annotation(tag.getKey(), tag.getValue()));
-        }
-      }
-    }
-  }
-
-  private static void addTagWithKey(zipkin2.Span zipkinSpan,
-                                    List annotations,
-                                    String key,
-                                    String defaultValue) {
-    if (zipkinSpan.tags() != null && zipkinSpan.tags().size() > 0 && zipkinSpan.tags().get(key) != null) {
-      annotations.add(new Annotation(key, zipkinSpan.tags().get(key)));
-    } else if (defaultValue != null) {
-      annotations.add(new Annotation(key, defaultValue));
-    }
-  }
-
-  @Override
-  protected void processLine(final ChannelHandlerContext ctx, final String message) {
-    throw new UnsupportedOperationException("Invalid context for processLine");
-  }
-}
-
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpGrpcMetricsHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpGrpcMetricsHandler.java
new file mode 100644
index 000000000..e2e3dfded
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpGrpcMetricsHandler.java
@@ -0,0 +1,80 @@
+package com.wavefront.agent.listeners.otlp;
+
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.data.ReportableEntityType;
+import io.grpc.stub.StreamObserver;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse;
+import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc;
+import java.util.function.Supplier;
+import javax.annotation.Nullable;
+import wavefront.report.ReportPoint;
+
+public class OtlpGrpcMetricsHandler extends MetricsServiceGrpc.MetricsServiceImplBase {
+
+  private final ReportableEntityHandler pointHandler;
+  private final ReportableEntityHandler histogramHandler;
+  private final Supplier preprocessorSupplier;
+  private final String defaultSource;
+  private final boolean includeResourceAttrsForMetrics;
+  private final boolean includeOtlpAppTagsOnMetrics;
+
+  /**
+   * Create new instance.
+   *
+   * @param pointHandler
+   * @param histogramHandler
+   * @param preprocessorSupplier
+   * @param defaultSource
+   * @param includeResourceAttrsForMetrics
+   */
+  public OtlpGrpcMetricsHandler(
+      ReportableEntityHandler pointHandler,
+      ReportableEntityHandler histogramHandler,
+      Supplier preprocessorSupplier,
+      String defaultSource,
+      boolean includeResourceAttrsForMetrics,
+      boolean includeOtlpAppTagsOnMetrics) {
+    super();
+    this.pointHandler = pointHandler;
+    this.histogramHandler = histogramHandler;
+    this.preprocessorSupplier = preprocessorSupplier;
+    this.defaultSource = defaultSource;
+    this.includeResourceAttrsForMetrics = includeResourceAttrsForMetrics;
+    this.includeOtlpAppTagsOnMetrics = includeOtlpAppTagsOnMetrics;
+  }
+
+  public OtlpGrpcMetricsHandler(
+      String handle,
+      ReportableEntityHandlerFactory handlerFactory,
+      @Nullable Supplier preprocessorSupplier,
+      String defaultSource,
+      boolean includeResourceAttrsForMetrics,
+      boolean includeOtlpAppTagsOnMetrics) {
+    this(
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle)),
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM, handle)),
+        preprocessorSupplier,
+        defaultSource,
+        includeResourceAttrsForMetrics,
+        includeOtlpAppTagsOnMetrics);
+  }
+
+  public void export(
+      ExportMetricsServiceRequest request,
+      StreamObserver responseObserver) {
+    OtlpMetricsUtils.exportToWavefront(
+        request,
+        pointHandler,
+        histogramHandler,
+        preprocessorSupplier,
+        defaultSource,
+        includeResourceAttrsForMetrics,
+        includeOtlpAppTagsOnMetrics);
+    responseObserver.onNext(ExportMetricsServiceResponse.getDefaultInstance());
+    responseObserver.onCompleted();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpGrpcTraceHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpGrpcTraceHandler.java
new file mode 100644
index 000000000..1adaf7a53
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpGrpcTraceHandler.java
@@ -0,0 +1,165 @@
+package com.wavefront.agent.listeners.otlp;
+
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportHeartbeats;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.common.NamedThreadFactory;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import io.grpc.Status;
+import io.grpc.stub.StreamObserver;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse;
+import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
+
+public class OtlpGrpcTraceHandler extends TraceServiceGrpc.TraceServiceImplBase
+    implements Closeable, Runnable {
+  protected static final Logger logger =
+      Logger.getLogger(OtlpGrpcTraceHandler.class.getCanonicalName());
+  private final ReportableEntityHandler spanHandler;
+  private final ReportableEntityHandler spanLogsHandler;
+  @Nullable private final WavefrontSender wfSender;
+  @Nullable private final Supplier preprocessorSupplier;
+  private final Pair spanSamplerAndCounter;
+  private final Pair, Counter> spansDisabled;
+  private final Pair, Counter> spanLogsDisabled;
+  private final String defaultSource;
+  @Nullable private final WavefrontInternalReporter internalReporter;
+  private final Set, String>> discoveredHeartbeatMetrics;
+  private final Set traceDerivedCustomTagKeys;
+  private final ScheduledExecutorService scheduledExecutorService;
+  private final Counter receivedSpans;
+
+  @VisibleForTesting
+  public OtlpGrpcTraceHandler(
+      String handle,
+      ReportableEntityHandler spanHandler,
+      ReportableEntityHandler spanLogsHandler,
+      @Nullable WavefrontSender wfSender,
+      @Nullable Supplier preprocessorSupplier,
+      SpanSampler sampler,
+      Supplier spansFeatureDisabled,
+      Supplier spanLogsFeatureDisabled,
+      String defaultSource,
+      Set traceDerivedCustomTagKeys) {
+    this.spanHandler = spanHandler;
+    this.spanLogsHandler = spanLogsHandler;
+    this.wfSender = wfSender;
+    this.preprocessorSupplier = preprocessorSupplier;
+    this.defaultSource = defaultSource;
+    this.traceDerivedCustomTagKeys = traceDerivedCustomTagKeys;
+
+    this.discoveredHeartbeatMetrics = Sets.newConcurrentHashSet();
+    this.receivedSpans =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "received.total"));
+    this.spanSamplerAndCounter =
+        Pair.of(
+            sampler,
+            Metrics.newCounter(new MetricName("spans." + handle, "", "sampler.discarded")));
+    this.spansDisabled =
+        Pair.of(
+            spansFeatureDisabled,
+            Metrics.newCounter(new MetricName("spans." + handle, "", "discarded")));
+    this.spanLogsDisabled =
+        Pair.of(
+            spanLogsFeatureDisabled,
+            Metrics.newCounter(new MetricName("spanLogs." + handle, "", "discarded")));
+
+    this.scheduledExecutorService =
+        Executors.newScheduledThreadPool(1, new NamedThreadFactory("otlp-grpc-heart-beater"));
+    scheduledExecutorService.scheduleAtFixedRate(this, 1, 1, TimeUnit.MINUTES);
+
+    this.internalReporter = OtlpTraceUtils.createAndStartInternalReporter(wfSender);
+  }
+
+  public OtlpGrpcTraceHandler(
+      String handle,
+      ReportableEntityHandlerFactory handlerFactory,
+      @Nullable WavefrontSender wfSender,
+      @Nullable Supplier preprocessorSupplier,
+      SpanSampler sampler,
+      Supplier spansFeatureDisabled,
+      Supplier spanLogsFeatureDisabled,
+      String defaultSource,
+      Set traceDerivedCustomTagKeys) {
+    this(
+        handle,
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)),
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, handle)),
+        wfSender,
+        preprocessorSupplier,
+        sampler,
+        spansFeatureDisabled,
+        spanLogsFeatureDisabled,
+        defaultSource,
+        traceDerivedCustomTagKeys);
+  }
+
+  @Override
+  public void export(
+      ExportTraceServiceRequest request,
+      StreamObserver responseObserver) {
+    long spanCount = OtlpTraceUtils.getSpansCount(request);
+    receivedSpans.inc(spanCount);
+
+    if (isFeatureDisabled(spansDisabled._1, SPAN_DISABLED, spansDisabled._2, spanCount)) {
+      Status grpcError = Status.FAILED_PRECONDITION.augmentDescription(SPAN_DISABLED);
+      responseObserver.onError(grpcError.asException());
+      return;
+    }
+
+    OtlpTraceUtils.exportToWavefront(
+        request,
+        spanHandler,
+        spanLogsHandler,
+        preprocessorSupplier,
+        spanLogsDisabled,
+        spanSamplerAndCounter,
+        defaultSource,
+        discoveredHeartbeatMetrics,
+        internalReporter,
+        traceDerivedCustomTagKeys);
+
+    responseObserver.onNext(ExportTraceServiceResponse.getDefaultInstance());
+    responseObserver.onCompleted();
+  }
+
+  @Override
+  public void run() {
+    try {
+      reportHeartbeats(wfSender, discoveredHeartbeatMetrics, "otlp");
+    } catch (IOException e) {
+      logger.warning("Cannot report heartbeat metric to wavefront");
+    }
+  }
+
+  @Override
+  public void close() {
+    scheduledExecutorService.shutdownNow();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpHttpHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpHttpHandler.java
new file mode 100644
index 000000000..7584b6c49
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpHttpHandler.java
@@ -0,0 +1,224 @@
+package com.wavefront.agent.listeners.otlp;
+
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportHeartbeats;
+
+import com.google.common.collect.Sets;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.rpc.Code;
+import com.google.rpc.Status;
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.agent.listeners.AbstractHttpOnlyHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.common.NamedThreadFactory;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.common.annotation.NonNull;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.DefaultHttpHeaders;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.codec.http.HttpVersion;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import wavefront.report.ReportPoint;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
+
+public class OtlpHttpHandler extends AbstractHttpOnlyHandler implements Closeable, Runnable {
+  private static final Logger logger = Logger.getLogger(OtlpHttpHandler.class.getCanonicalName());
+  private final String defaultSource;
+  private final Set, String>> discoveredHeartbeatMetrics;
+  @Nullable private final WavefrontInternalReporter internalReporter;
+  @Nullable private final Supplier preprocessorSupplier;
+  private final Pair spanSamplerAndCounter;
+  private final ScheduledExecutorService scheduledExecutorService;
+  private final ReportableEntityHandler spanHandler;
+  @Nullable private final WavefrontSender sender;
+  private final ReportableEntityHandler spanLogsHandler;
+  private final Set traceDerivedCustomTagKeys;
+  private final ReportableEntityHandler metricsHandler;
+  private final ReportableEntityHandler histogramHandler;
+  private final Counter receivedSpans;
+  private final Pair, Counter> spansDisabled;
+  private final Pair, Counter> spanLogsDisabled;
+  private final boolean includeResourceAttrsForMetrics;
+  private final boolean includeOtlpAppTagsOnMetrics;
+
+  public OtlpHttpHandler(
+      ReportableEntityHandlerFactory handlerFactory,
+      @Nullable TokenAuthenticator tokenAuthenticator,
+      @Nullable HealthCheckManager healthCheckManager,
+      @NonNull String handle,
+      @Nullable WavefrontSender wfSender,
+      @Nullable Supplier preprocessorSupplier,
+      SpanSampler sampler,
+      Supplier spansFeatureDisabled,
+      Supplier spanLogsFeatureDisabled,
+      String defaultSource,
+      Set traceDerivedCustomTagKeys,
+      boolean includeResourceAttrsForMetrics,
+      boolean includeOtlpAppTagsOnMetrics) {
+    super(tokenAuthenticator, healthCheckManager, handle);
+    this.includeResourceAttrsForMetrics = includeResourceAttrsForMetrics;
+    this.includeOtlpAppTagsOnMetrics = includeOtlpAppTagsOnMetrics;
+    this.spanHandler = handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle));
+    this.spanLogsHandler =
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, handle));
+    this.metricsHandler =
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, handle));
+    this.histogramHandler =
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM, handle));
+    this.sender = wfSender;
+    this.preprocessorSupplier = preprocessorSupplier;
+    this.defaultSource = defaultSource;
+    this.traceDerivedCustomTagKeys = traceDerivedCustomTagKeys;
+
+    this.discoveredHeartbeatMetrics = Sets.newConcurrentHashSet();
+    this.receivedSpans =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "received.total"));
+    this.spanSamplerAndCounter =
+        Pair.of(
+            sampler,
+            Metrics.newCounter(new MetricName("spans." + handle, "", "sampler.discarded")));
+    this.spansDisabled =
+        Pair.of(
+            spansFeatureDisabled,
+            Metrics.newCounter(new MetricName("spans." + handle, "", "discarded")));
+    this.spanLogsDisabled =
+        Pair.of(
+            spanLogsFeatureDisabled,
+            Metrics.newCounter(new MetricName("spanLogs." + handle, "", "discarded")));
+
+    this.scheduledExecutorService =
+        Executors.newScheduledThreadPool(1, new NamedThreadFactory("otlp-http-heart-beater"));
+    scheduledExecutorService.scheduleAtFixedRate(this, 1, 1, TimeUnit.MINUTES);
+
+    this.internalReporter = OtlpTraceUtils.createAndStartInternalReporter(sender);
+  }
+
+  @Override
+  protected void handleHttpMessage(ChannelHandlerContext ctx, FullHttpRequest request)
+      throws URISyntaxException {
+    URI uri = new URI(request.uri());
+    String path = uri.getPath().endsWith("/") ? uri.getPath() : uri.getPath() + "/";
+    try {
+      switch (path) {
+        case "/v1/traces/":
+          ExportTraceServiceRequest traceRequest =
+              ExportTraceServiceRequest.parseFrom(request.content().nioBuffer());
+          long spanCount = OtlpTraceUtils.getSpansCount(traceRequest);
+          receivedSpans.inc(spanCount);
+
+          if (isFeatureDisabled(spansDisabled._1, SPAN_DISABLED, spansDisabled._2, spanCount)) {
+            HttpResponse response = makeErrorResponse(Code.FAILED_PRECONDITION, SPAN_DISABLED);
+            writeHttpResponse(ctx, response, request);
+            return;
+          }
+
+          OtlpTraceUtils.exportToWavefront(
+              traceRequest,
+              spanHandler,
+              spanLogsHandler,
+              preprocessorSupplier,
+              spanLogsDisabled,
+              spanSamplerAndCounter,
+              defaultSource,
+              discoveredHeartbeatMetrics,
+              internalReporter,
+              traceDerivedCustomTagKeys);
+          break;
+        case "/v1/metrics/":
+          ExportMetricsServiceRequest metricRequest =
+              ExportMetricsServiceRequest.parseFrom(request.content().nioBuffer());
+          OtlpMetricsUtils.exportToWavefront(
+              metricRequest,
+              metricsHandler,
+              histogramHandler,
+              preprocessorSupplier,
+              defaultSource,
+              includeResourceAttrsForMetrics,
+              includeOtlpAppTagsOnMetrics);
+          break;
+        default:
+          /*
+           We use HTTP 200 for success and HTTP 400 for errors, mirroring what we found in
+           OTel Collector's OTLP Receiver code.
+          */
+          writeHttpResponse(
+              ctx, HttpResponseStatus.BAD_REQUEST, "unknown endpoint " + path, request);
+          return;
+      }
+
+      writeHttpResponse(ctx, HttpResponseStatus.OK, "", request);
+    } catch (InvalidProtocolBufferException e) {
+      logWarning("WF-300: Failed to handle incoming OTLP request", e, ctx);
+      HttpResponse response = makeErrorResponse(Code.INVALID_ARGUMENT, e.getMessage());
+      writeHttpResponse(ctx, response, request);
+    }
+  }
+
+  @Override
+  public void run() {
+    try {
+      reportHeartbeats(sender, discoveredHeartbeatMetrics, "otlp");
+    } catch (IOException e) {
+      logger.warning("Cannot report heartbeat metric to wavefront");
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    scheduledExecutorService.shutdownNow();
+  }
+
+  /*
+  Build an OTLP HTTP error response per the spec:
+  https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlphttp-response
+   */
+  private HttpResponse makeErrorResponse(Code rpcCode, String msg) {
+    Status pbStatus = Status.newBuilder().setCode(rpcCode.getNumber()).setMessage(msg).build();
+    ByteBuf content = Unpooled.copiedBuffer(pbStatus.toByteArray());
+
+    HttpHeaders headers =
+        new DefaultHttpHeaders()
+            .set(HttpHeaderNames.CONTENT_TYPE, "application/x-protobuf")
+            .set(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes());
+
+    HttpResponseStatus httpStatus =
+        (rpcCode == Code.NOT_FOUND) ? HttpResponseStatus.NOT_FOUND : HttpResponseStatus.BAD_REQUEST;
+
+    return new DefaultFullHttpResponse(
+        HttpVersion.HTTP_1_1, httpStatus, content, headers, new DefaultHttpHeaders());
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpMetricsUtils.java b/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpMetricsUtils.java
new file mode 100644
index 000000000..c6a4f5804
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpMetricsUtils.java
@@ -0,0 +1,730 @@
+package com.wavefront.agent.listeners.otlp;
+
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.common.MetricConstants;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.entities.histograms.HistogramGranularity;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
+import io.opentelemetry.proto.common.v1.AnyValue;
+import io.opentelemetry.proto.common.v1.KeyValue;
+import io.opentelemetry.proto.metrics.v1.AggregationTemporality;
+import io.opentelemetry.proto.metrics.v1.ExponentialHistogram;
+import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
+import io.opentelemetry.proto.metrics.v1.Gauge;
+import io.opentelemetry.proto.metrics.v1.Histogram;
+import io.opentelemetry.proto.metrics.v1.HistogramDataPoint;
+import io.opentelemetry.proto.metrics.v1.Metric;
+import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
+import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
+import io.opentelemetry.proto.metrics.v1.ScopeMetrics;
+import io.opentelemetry.proto.metrics.v1.Sum;
+import io.opentelemetry.proto.metrics.v1.Summary;
+import io.opentelemetry.proto.metrics.v1.SummaryDataPoint;
+import io.opentelemetry.proto.resource.v1.Resource;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+import org.jetbrains.annotations.NotNull;
+import wavefront.report.Annotation;
+import wavefront.report.HistogramType;
+import wavefront.report.ReportPoint;
+
+public class OtlpMetricsUtils {
+  public static final Logger OTLP_DATA_LOGGER = Logger.getLogger("OTLPDataLogger");
+  public static final int MILLIS_IN_MINUTE = 60 * 1000;
+  public static final int MILLIS_IN_HOUR = 60 * 60 * 1000;
+  public static final int MILLIS_IN_DAY = 24 * 60 * 60 * 1000;
+
+  public static void exportToWavefront(
+      ExportMetricsServiceRequest request,
+      ReportableEntityHandler pointHandler,
+      ReportableEntityHandler histogramHandler,
+      @Nullable Supplier preprocessorSupplier,
+      String defaultSource,
+      boolean includeResourceAttrsForMetrics,
+      boolean includeOtlpAppTagsOnMetrics) {
+    ReportableEntityPreprocessor preprocessor = null;
+    if (preprocessorSupplier != null) {
+      preprocessor = preprocessorSupplier.get();
+    }
+
+    for (ReportPoint point :
+        fromOtlpRequest(
+            request,
+            preprocessor,
+            defaultSource,
+            includeResourceAttrsForMetrics,
+            includeOtlpAppTagsOnMetrics)) {
+      if (point.getValue() instanceof wavefront.report.Histogram) {
+        if (!wasFilteredByPreprocessor(point, histogramHandler, preprocessor)) {
+          histogramHandler.report(point);
+        }
+      } else {
+        if (!wasFilteredByPreprocessor(point, pointHandler, preprocessor)) {
+          pointHandler.report(point);
+        }
+      }
+    }
+  }
+
+  private static List fromOtlpRequest(
+      ExportMetricsServiceRequest request,
+      @Nullable ReportableEntityPreprocessor preprocessor,
+      String defaultSource,
+      boolean includeResourceAttrsForMetrics,
+      boolean includeAppTagsOnMetrics) {
+    List wfPoints = Lists.newArrayList();
+
+    for (ResourceMetrics resourceMetrics : request.getResourceMetricsList()) {
+      Resource resource = resourceMetrics.getResource();
+      OTLP_DATA_LOGGER.finest(() -> "Inbound OTLP Resource: " + resource);
+      Pair> sourceAndResourceAttrs =
+          OtlpTraceUtils.sourceFromAttributes(resource.getAttributesList(), defaultSource);
+      String source = sourceAndResourceAttrs._1;
+
+      List resourceAttributes = Collections.EMPTY_LIST;
+      if (includeResourceAttrsForMetrics) {
+        resourceAttributes = replaceServiceNameKeyWithServiceKey(sourceAndResourceAttrs._2);
+      } else if (includeAppTagsOnMetrics) {
+        resourceAttributes = appTagsFromResourceAttrs(sourceAndResourceAttrs._2);
+      }
+
+      for (ScopeMetrics scopeMetrics : resourceMetrics.getScopeMetricsList()) {
+        OTLP_DATA_LOGGER.finest(
+            () -> "Inbound OTLP Instrumentation Scope: " + scopeMetrics.getScope());
+        for (Metric otlpMetric : scopeMetrics.getMetricsList()) {
+          OTLP_DATA_LOGGER.finest(() -> "Inbound OTLP Metric: " + otlpMetric);
+          List points =
+              transform(otlpMetric, resourceAttributes, preprocessor, source);
+          OTLP_DATA_LOGGER.finest(() -> "Converted Wavefront Metric: " + points);
+
+          wfPoints.addAll(points);
+        }
+      }
+    }
+    return wfPoints;
+  }
+
+  @VisibleForTesting
+  static List replaceServiceNameKeyWithServiceKey(List attributes) {
+    KeyValue serviceNameAttr =
+        OtlpTraceUtils.getAttrByKey(attributes, OtlpTraceUtils.OTEL_SERVICE_NAME_KEY);
+    KeyValue serviceAttr = OtlpTraceUtils.getAttrByKey(attributes, SERVICE_TAG_KEY);
+    if (serviceNameAttr != null && serviceAttr == null) {
+      List attributesWithServiceKey = new ArrayList<>(attributes);
+      attributesWithServiceKey.remove(serviceNameAttr);
+      attributesWithServiceKey.add(
+          OtlpTraceUtils.buildKeyValue(
+              SERVICE_TAG_KEY, serviceNameAttr.getValue().getStringValue()));
+      return attributesWithServiceKey;
+    }
+    return attributes;
+  }
+
+  /*MONIT-30703: adding application & system.name tags to a metric*/
+  @VisibleForTesting
+  static List appTagsFromResourceAttrs(List resourceAttrs) {
+    List attrList = new ArrayList<>();
+    attrList.add(OtlpTraceUtils.getAttrByKey(resourceAttrs, APPLICATION_TAG_KEY));
+    KeyValue serviceAttr = OtlpTraceUtils.getAttrByKey(resourceAttrs, SERVICE_TAG_KEY);
+    if (serviceAttr != null) {
+      attrList.add(serviceAttr);
+    } else {
+      attrList.add(
+          OtlpTraceUtils.getAttrByKey(resourceAttrs, OtlpTraceUtils.OTEL_SERVICE_NAME_KEY));
+    }
+    attrList.add(OtlpTraceUtils.getAttrByKey(resourceAttrs, CLUSTER_TAG_KEY));
+    attrList.add(OtlpTraceUtils.getAttrByKey(resourceAttrs, SHARD_TAG_KEY));
+    attrList.removeAll(Collections.singleton(null));
+    attrList = replaceServiceNameKeyWithServiceKey(attrList);
+    return attrList;
+  }
+
+  @VisibleForTesting
+  static boolean wasFilteredByPreprocessor(
+      ReportPoint wfReportPoint,
+      ReportableEntityHandler pointHandler,
+      @Nullable ReportableEntityPreprocessor preprocessor) {
+    if (preprocessor == null) {
+      return false;
+    }
+
+    String[] messageHolder = new String[1];
+    if (!preprocessor.forReportPoint().filter(wfReportPoint, messageHolder)) {
+      if (messageHolder[0] != null) {
+        pointHandler.reject(wfReportPoint, messageHolder[0]);
+      } else {
+        pointHandler.block(wfReportPoint);
+      }
+      return true;
+    }
+
+    return false;
+  }
+
+  @VisibleForTesting
+  public static List transform(
+      Metric otlpMetric,
+      List resourceAttrs,
+      ReportableEntityPreprocessor preprocessor,
+      String source) {
+    List points = new ArrayList<>();
+    if (otlpMetric.hasGauge()) {
+      points.addAll(transformGauge(otlpMetric.getName(), otlpMetric.getGauge(), resourceAttrs));
+    } else if (otlpMetric.hasSum()) {
+      points.addAll(transformSum(otlpMetric.getName(), otlpMetric.getSum(), resourceAttrs));
+    } else if (otlpMetric.hasSummary()) {
+      points.addAll(transformSummary(otlpMetric.getName(), otlpMetric.getSummary(), resourceAttrs));
+    } else if (otlpMetric.hasHistogram()) {
+      points.addAll(
+          transformHistogram(
+              otlpMetric.getName(),
+              fromOtelHistogram(otlpMetric.getName(), otlpMetric.getHistogram()),
+              otlpMetric.getHistogram().getAggregationTemporality(),
+              resourceAttrs));
+    } else if (otlpMetric.hasExponentialHistogram()) {
+      points.addAll(
+          transformHistogram(
+              otlpMetric.getName(),
+              fromOtelExponentialHistogram(otlpMetric.getExponentialHistogram()),
+              otlpMetric.getExponentialHistogram().getAggregationTemporality(),
+              resourceAttrs));
+    } else {
+      throw new IllegalArgumentException(
+          "Otel: unsupported metric type for " + otlpMetric.getName());
+    }
+
+    for (ReportPoint point : points) {
+      point.setHost(source);
+      // preprocessor rule transformations should run last
+      if (preprocessor != null) {
+        preprocessor.forReportPoint().transform(point);
+      }
+    }
+    return points;
+  }
+
+  private static List transformSummary(
+      String name, Summary summary, List resourceAttrs) {
+    List points = new ArrayList<>(summary.getDataPointsCount());
+    for (SummaryDataPoint p : summary.getDataPointsList()) {
+      points.addAll(transformSummaryDataPoint(name, p, resourceAttrs));
+    }
+    return points;
+  }
+
+  private static List transformSum(
+      String name, Sum sum, List resourceAttrs) {
+    if (sum.getDataPointsCount() == 0) {
+      throw new IllegalArgumentException("OTel: sum with no data points");
+    }
+
+    String prefix = "";
+    switch (sum.getAggregationTemporality()) {
+      case AGGREGATION_TEMPORALITY_CUMULATIVE:
+        // no prefix
+        break;
+      case AGGREGATION_TEMPORALITY_DELTA:
+        prefix = MetricConstants.DELTA_PREFIX;
+        break;
+      default:
+        throw new IllegalArgumentException(
+            "OTel: sum with unsupported aggregation temporality "
+                + sum.getAggregationTemporality().name());
+    }
+
+    List points = new ArrayList<>(sum.getDataPointsCount());
+    for (NumberDataPoint p : sum.getDataPointsList()) {
+      points.add(transformNumberDataPoint(prefix + name, p, resourceAttrs));
+    }
+    return points;
+  }
+
+  private static List transformHistogram(
+      String name,
+      List dataPoints,
+      AggregationTemporality aggregationTemporality,
+      List resourceAttrs) {
+
+    switch (aggregationTemporality) {
+      case AGGREGATION_TEMPORALITY_CUMULATIVE:
+        return transformCumulativeHistogram(name, dataPoints, resourceAttrs);
+      case AGGREGATION_TEMPORALITY_DELTA:
+        return transformDeltaHistogram(name, dataPoints, resourceAttrs);
+      default:
+        throw new IllegalArgumentException(
+            "OTel: histogram with unsupported aggregation temporality "
+                + aggregationTemporality.name());
+    }
+  }
+
+  private static List transformDeltaHistogram(
+      String name, List dataPoints, List resourceAttrs) {
+    List reportPoints = new ArrayList<>();
+    for (BucketHistogramDataPoint dataPoint : dataPoints) {
+      reportPoints.addAll(transformDeltaHistogramDataPoint(name, dataPoint, resourceAttrs));
+    }
+
+    return reportPoints;
+  }
+
+  private static List transformCumulativeHistogram(
+      String name, List dataPoints, List resourceAttrs) {
+
+    List reportPoints = new ArrayList<>();
+    for (BucketHistogramDataPoint dataPoint : dataPoints) {
+      reportPoints.addAll(transformCumulativeHistogramDataPoint(name, dataPoint, resourceAttrs));
+    }
+
+    return reportPoints;
+  }
+
+  private static List transformDeltaHistogramDataPoint(
+      String name, BucketHistogramDataPoint point, List resourceAttrs) {
+    List reportPoints = new ArrayList<>();
+    BinsAndCounts binsAndCounts = point.asDelta();
+
+    for (HistogramGranularity granularity : HistogramGranularity.values()) {
+      int duration;
+      switch (granularity) {
+        case MINUTE:
+          duration = MILLIS_IN_MINUTE;
+          break;
+        case HOUR:
+          duration = MILLIS_IN_HOUR;
+          break;
+        case DAY:
+          duration = MILLIS_IN_DAY;
+          break;
+        default:
+          throw new IllegalArgumentException("Unknown granularity: " + granularity);
+      }
+
+      wavefront.report.Histogram histogram =
+          wavefront.report.Histogram.newBuilder()
+              .setType(HistogramType.TDIGEST)
+              .setBins(binsAndCounts.getBins())
+              .setCounts(binsAndCounts.getCounts())
+              .setDuration(duration)
+              .build();
+
+      ReportPoint rp =
+          pointWithAnnotations(
+                  name, point.getAttributesList(), resourceAttrs, point.getTimeUnixNano())
+              .setValue(histogram)
+              .build();
+      reportPoints.add(rp);
+    }
+    return reportPoints;
+  }
+
+  private static List transformCumulativeHistogramDataPoint(
+      String name, BucketHistogramDataPoint point, List resourceAttrs) {
+    List buckets = point.asCumulative();
+    List reportPoints = new ArrayList<>(buckets.size());
+    for (CumulativeBucket bucket : buckets) {
+      // we have to create a new builder every time as the annotations are getting appended
+      // after
+      // each iteration
+      ReportPoint rp =
+          pointWithAnnotations(
+                  name, point.getAttributesList(), resourceAttrs, point.getTimeUnixNano())
+              .setValue(bucket.getCount())
+              .build();
+      handleDupAnnotation(rp);
+      rp.getAnnotations().put("le", bucket.getTag());
+      reportPoints.add(rp);
+    }
+    return reportPoints;
+  }
+
+  private static void handleDupAnnotation(ReportPoint rp) {
+    if (rp.getAnnotations().containsKey("le")) {
+      String val = rp.getAnnotations().get("le");
+      rp.getAnnotations().remove("le");
+      rp.getAnnotations().put("_le", val);
+    }
+  }
+
+  private static Collection transformGauge(
+      String name, Gauge gauge, List resourceAttrs) {
+    if (gauge.getDataPointsCount() == 0) {
+      throw new IllegalArgumentException("OTel: gauge with no data points");
+    }
+
+    List points = new ArrayList<>(gauge.getDataPointsCount());
+    for (NumberDataPoint p : gauge.getDataPointsList()) {
+      points.add(transformNumberDataPoint(name, p, resourceAttrs));
+    }
+    return points;
+  }
+
+  @NotNull
+  private static ReportPoint transformNumberDataPoint(
+      String name, NumberDataPoint point, List resourceAttrs) {
+    ReportPoint.Builder rp =
+        pointWithAnnotations(
+            name, point.getAttributesList(), resourceAttrs, point.getTimeUnixNano());
+
+    if (point.hasAsInt()) {
+      return rp.setValue(point.getAsInt()).build();
+    } else {
+      return rp.setValue(point.getAsDouble()).build();
+    }
+  }
+
+  @NotNull
+  private static List transformSummaryDataPoint(
+      String name, SummaryDataPoint point, List resourceAttrs) {
+    List toReturn = new ArrayList<>();
+    List pointAttributes = replaceQuantileTag(point.getAttributesList());
+    toReturn.add(
+        pointWithAnnotations(name + "_sum", pointAttributes, resourceAttrs, point.getTimeUnixNano())
+            .setValue(point.getSum())
+            .build());
+    toReturn.add(
+        pointWithAnnotations(
+                name + "_count", pointAttributes, resourceAttrs, point.getTimeUnixNano())
+            .setValue(point.getCount())
+            .build());
+    for (SummaryDataPoint.ValueAtQuantile q : point.getQuantileValuesList()) {
+      List attributes = new ArrayList<>(pointAttributes);
+      KeyValue quantileTag =
+          KeyValue.newBuilder()
+              .setKey("quantile")
+              .setValue(AnyValue.newBuilder().setDoubleValue(q.getQuantile()).build())
+              .build();
+      attributes.add(quantileTag);
+      toReturn.add(
+          pointWithAnnotations(name, attributes, resourceAttrs, point.getTimeUnixNano())
+              .setValue(q.getValue())
+              .build());
+    }
+    return toReturn;
+  }
+
+  @NotNull
+  private static List replaceQuantileTag(List pointAttributes) {
+    if (pointAttributes.isEmpty()) return pointAttributes;
+
+    List modifiableAttributes = new ArrayList<>();
+    for (KeyValue pointAttribute : pointAttributes) {
+      if (pointAttribute.getKey().equals("quantile")) {
+        modifiableAttributes.add(
+            KeyValue.newBuilder().setKey("_quantile").setValue(pointAttribute.getValue()).build());
+      } else {
+        modifiableAttributes.add(pointAttribute);
+      }
+    }
+    return modifiableAttributes;
+  }
+
+  @NotNull
+  private static ReportPoint.Builder pointWithAnnotations(
+      String name, List pointAttributes, List resourceAttrs, long timeInNs) {
+    ReportPoint.Builder builder = ReportPoint.newBuilder().setMetric(name);
+    Map annotations = new HashMap<>();
+    List otlpAttributes =
+        Stream.of(resourceAttrs, pointAttributes)
+            .flatMap(Collection::stream)
+            .collect(Collectors.toList());
+
+    for (Annotation a : OtlpTraceUtils.annotationsFromAttributes(otlpAttributes)) {
+      annotations.put(a.getKey(), a.getValue());
+    }
+    builder.setAnnotations(annotations);
+    builder.setTimestamp(TimeUnit.NANOSECONDS.toMillis(timeInNs));
+    return builder;
+  }
+
+  static List fromOtelHistogram(String name, Histogram histogram) {
+    List result = new ArrayList<>(histogram.getDataPointsCount());
+    for (HistogramDataPoint dataPoint : histogram.getDataPointsList()) {
+      result.add(fromOtelHistogramDataPoint(name, dataPoint));
+    }
+    return result;
+  }
+
+  static BucketHistogramDataPoint fromOtelHistogramDataPoint(
+      String name, HistogramDataPoint dataPoint) {
+    if (dataPoint.getExplicitBoundsCount() != dataPoint.getBucketCountsCount() - 1) {
+      throw new IllegalArgumentException(
+          "OTel: histogram "
+              + name
+              + ": Explicit bounds count "
+              + "should be one less than bucket count. ExplicitBounds: "
+              + dataPoint.getExplicitBoundsCount()
+              + ", BucketCounts: "
+              + dataPoint.getBucketCountsCount());
+    }
+    return new BucketHistogramDataPoint(
+        dataPoint.getBucketCountsList(),
+        dataPoint.getExplicitBoundsList(),
+        dataPoint.getAttributesList(),
+        dataPoint.getTimeUnixNano(),
+        false);
+  }
+
+  static List fromOtelExponentialHistogram(
+      ExponentialHistogram histogram) {
+    List result = new ArrayList<>(histogram.getDataPointsCount());
+    for (ExponentialHistogramDataPoint dataPoint : histogram.getDataPointsList()) {
+      result.add(fromOtelExponentialHistogramDataPoint(dataPoint));
+    }
+    return result;
+  }
+
+  static BucketHistogramDataPoint fromOtelExponentialHistogramDataPoint(
+      ExponentialHistogramDataPoint dataPoint) {
+    // base is the factor by which explicit bounds increase from bucket to bucket. This formula
+    // comes from the documentation here:
+    // https://github.com/open-telemetry/opentelemetry-proto/blob/8ba33cceb4a6704af68a4022d17868a7ac1d94f4/opentelemetry/proto/metrics/v1/metrics.proto#L487
+    double base = Math.pow(2.0, Math.pow(2.0, -dataPoint.getScale()));
+
+    // ExponentialHistogramDataPoints have buckets with negative explicit bounds, buckets with
+    // positive explicit bounds, and a "zero" bucket. Our job is to merge these bucket groups
+    // into
+    // a single list of buckets and explicit bounds.
+    List negativeBucketCounts = dataPoint.getNegative().getBucketCountsList();
+    List positiveBucketCounts = dataPoint.getPositive().getBucketCountsList();
+
+    // The total number of buckets is the number of negative buckets + the number of positive
+    // buckets + 1 for the zero bucket + 1 bucket for negative infinity up to smallest negative
+    // explicit bound + 1 bucket for the largest positive explicit bound up to positive
+    // infinity.
+    int numBucketCounts = 1 + negativeBucketCounts.size() + 1 + positiveBucketCounts.size() + 1;
+
+    List bucketCounts = new ArrayList<>(numBucketCounts);
+
+    // The number of explicit bounds is always 1 less than the number of buckets. This is how
+    // explicit bounds work. If you have 2 explicit bounds say {2.0, 5.0} then you have 3
+    // buckets:
+    // one for values less than 2.0; one for values between 2.0 and 5.0; and one for values
+    // greater
+    // than 5.0.
+    List explicitBounds = new ArrayList<>(numBucketCounts - 1);
+
+    appendNegativeBucketsAndExplicitBounds(
+        dataPoint.getNegative().getOffset(),
+        base,
+        negativeBucketCounts,
+        bucketCounts,
+        explicitBounds);
+    appendZeroBucketAndExplicitBound(
+        dataPoint.getPositive().getOffset(),
+        base,
+        dataPoint.getZeroCount(),
+        bucketCounts,
+        explicitBounds);
+    appendPositiveBucketsAndExplicitBounds(
+        dataPoint.getPositive().getOffset(),
+        base,
+        positiveBucketCounts,
+        bucketCounts,
+        explicitBounds);
+    return new BucketHistogramDataPoint(
+        bucketCounts,
+        explicitBounds,
+        dataPoint.getAttributesList(),
+        dataPoint.getTimeUnixNano(),
+        true);
+  }
+
+  // appendNegativeBucketsAndExplicitBounds appends negative buckets and explicit bounds to
+  // bucketCounts and explicitBounds respectively.
+  static void appendNegativeBucketsAndExplicitBounds(
+      int negativeOffset,
+      double base,
+      List negativeBucketCounts,
+      List bucketCounts,
+      List explicitBounds) {
+    // The count in the first bucket which includes negative infinity is always 0.
+    bucketCounts.add(0L);
+
+    // The smallest negative explicit bound
+    double le = -Math.pow(base, ((double) negativeOffset) + ((double) negativeBucketCounts.size()));
+    explicitBounds.add(le);
+
+    // The first negativeBucketCount has a negative explicit bound with the smallest magnitude;
+    // the last negativeBucketCount has a negative explicit bound with the largest magnitude.
+    // Therefore, to go in order from smallest to largest explicit bound, we have to start with
+    // the last element in the negativeBucketCounts array.
+    for (int i = negativeBucketCounts.size() - 1; i >= 0; i--) {
+      bucketCounts.add(negativeBucketCounts.get(i));
+      le /= base; // We divide by base because our explicit bounds are getting smaller in
+      // magnitude as
+      // we go
+      explicitBounds.add(le);
+    }
+  }
+
+  // appendZeroBucketAndExplicitBound appends the "zero" bucket and explicit bound to bucketCounts
+  // and explicitBounds respectively. The smallest positive explicit bound is base^positiveOffset.
+  static void appendZeroBucketAndExplicitBound(
+      int positiveOffset,
+      double base,
+      long zeroBucketCount,
+      List bucketCounts,
+      List explicitBounds) {
+    bucketCounts.add(zeroBucketCount);
+
+    // The explicit bound of the zeroBucketCount is the smallest positive explicit bound
+    explicitBounds.add(Math.pow(base, positiveOffset));
+  }
+
+  // appendPositiveBucketsAndExplicitBounds appends positive buckets and explicit bounds to
+  // bucketCounts and explicitBounds respectively. The smallest positive explicit bound is
+  // base^positiveOffset.
+  static void appendPositiveBucketsAndExplicitBounds(
+      int positiveOffset,
+      double base,
+      List positiveBucketCounts,
+      List bucketCounts,
+      List explicitBounds) {
+    double le = Math.pow(base, positiveOffset);
+    for (Long positiveBucketCount : positiveBucketCounts) {
+      bucketCounts.add(positiveBucketCount);
+      le *= base;
+      explicitBounds.add(le);
+    }
+    // Last bucket for positive infinity is always 0.
+    bucketCounts.add(0L);
+  }
+
+  static class CumulativeBucket {
+    private final String tag;
+    private final long count;
+
+    CumulativeBucket(String tag, long count) {
+      this.tag = tag;
+      this.count = count;
+    }
+
+    String getTag() {
+      return tag;
+    }
+
+    long getCount() {
+      return count;
+    }
+  }
+
+  static class BinsAndCounts {
+    private final List bins;
+    private final List counts;
+
+    BinsAndCounts(List bins, List counts) {
+      this.bins = bins;
+      this.counts = counts;
+    }
+
+    List getBins() {
+      return bins;
+    }
+
+    List getCounts() {
+      return counts;
+    }
+  }
+
+  static class BucketHistogramDataPoint {
+    private final List bucketCounts;
+    private final List explicitBounds;
+    private final List attributesList;
+    private final long timeUnixNano;
+    private final boolean isExponential;
+
+    private BucketHistogramDataPoint(
+        List bucketCounts,
+        List explicitBounds,
+        List attributesList,
+        long timeUnixNano,
+        boolean isExponential) {
+      this.bucketCounts = bucketCounts;
+      this.explicitBounds = explicitBounds;
+      this.attributesList = attributesList;
+      this.timeUnixNano = timeUnixNano;
+      this.isExponential = isExponential;
+    }
+
+    List asCumulative() {
+      if (isExponential) {
+        return asCumulative(1, bucketCounts.size());
+      }
+      return asCumulative(0, bucketCounts.size());
+    }
+
+    BinsAndCounts asDelta() {
+      if (isExponential) {
+        return asDelta(1, bucketCounts.size() - 1);
+      }
+      return asDelta(0, bucketCounts.size());
+    }
+
+    private List asCumulative(int startBucketIndex, int endBucketIndex) {
+      List result = new ArrayList<>(endBucketIndex - startBucketIndex);
+      long leCount = 0;
+      for (int i = startBucketIndex; i < endBucketIndex; i++) {
+        leCount += bucketCounts.get(i);
+        result.add(new CumulativeBucket(leTagValue(i), leCount));
+      }
+      return result;
+    }
+
+    private String leTagValue(int index) {
+      if (index == explicitBounds.size()) {
+        return "+Inf";
+      }
+      return String.valueOf(explicitBounds.get(index));
+    }
+
+    private BinsAndCounts asDelta(int startBucketIndex, int endBucketIndex) {
+      List bins = new ArrayList<>(endBucketIndex - startBucketIndex);
+      List counts = new ArrayList<>(endBucketIndex - startBucketIndex);
+      for (int i = startBucketIndex; i < endBucketIndex; i++) {
+        bins.add(centroidValue(i));
+        counts.add(bucketCounts.get(i).intValue());
+      }
+      return new BinsAndCounts(bins, counts);
+    }
+
+    private double centroidValue(int index) {
+      int length = explicitBounds.size();
+      if (length == 0) {
+        return 0.0;
+      }
+      if (index == 0) {
+        return explicitBounds.get(0);
+      }
+      if (index == length) {
+        return explicitBounds.get(length - 1);
+      }
+      return (explicitBounds.get(index - 1) + explicitBounds.get(index)) / 2.0;
+    }
+
+    List getAttributesList() {
+      return attributesList;
+    }
+
+    long getTimeUnixNano() {
+      return timeUnixNano;
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpTraceUtils.java b/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpTraceUtils.java
new file mode 100644
index 000000000..d0d9cc1ad
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/otlp/OtlpTraceUtils.java
@@ -0,0 +1,548 @@
+package com.wavefront.agent.listeners.otlp;
+
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPANLOGS_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled;
+import static com.wavefront.common.TraceConstants.PARENT_KEY;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.ERROR_SPAN_TAG_VAL;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.TRACING_DERIVED_PREFIX;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportWavefrontGeneratedData;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.ERROR_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.NULL_TAG_VAL;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SOURCE_KEY;
+import static com.wavefront.sdk.common.Constants.SPAN_LOG_KEY;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.ByteString;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.listeners.tracing.SpanUtils;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.yammer.metrics.core.Counter;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.common.v1.AnyValue;
+import io.opentelemetry.proto.common.v1.InstrumentationScope;
+import io.opentelemetry.proto.common.v1.KeyValue;
+import io.opentelemetry.proto.resource.v1.Resource;
+import io.opentelemetry.proto.trace.v1.ResourceSpans;
+import io.opentelemetry.proto.trace.v1.ScopeSpans;
+import io.opentelemetry.proto.trace.v1.Span.SpanKind;
+import io.opentelemetry.proto.trace.v1.Status;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
+
+/**
+ * @author Xiaochen Wang (xiaochenw@vmware.com).
+ * @author Glenn Oppegard (goppegard@vmware.com).
+ */
+public class OtlpTraceUtils {
+  // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk_exporters/non-otlp.md#span-status
+  public static final String OTEL_DROPPED_ATTRS_KEY = "otel.dropped_attributes_count";
+  public static final String OTEL_DROPPED_EVENTS_KEY = "otel.dropped_events_count";
+  public static final String OTEL_DROPPED_LINKS_KEY = "otel.dropped_links_count";
+  public static final String OTEL_SERVICE_NAME_KEY = "service.name";
+  public static final String OTEL_STATUS_DESCRIPTION_KEY = "otel.status_description";
+  private static final String DEFAULT_APPLICATION_NAME = "defaultApplication";
+  private static final String DEFAULT_SERVICE_NAME = "defaultService";
+  private static final Logger OTLP_DATA_LOGGER = Logger.getLogger("OTLPDataLogger");
+  private static final String SPAN_EVENT_TAG_KEY = "name";
+  private static final String SPAN_KIND_TAG_KEY = "span.kind";
+  private static final HashMap SPAN_KIND_ANNOTATION_HASH_MAP =
+      new HashMap() {
+        {
+          put(SpanKind.SPAN_KIND_CLIENT, new Annotation(SPAN_KIND_TAG_KEY, "client"));
+          put(SpanKind.SPAN_KIND_CONSUMER, new Annotation(SPAN_KIND_TAG_KEY, "consumer"));
+          put(SpanKind.SPAN_KIND_INTERNAL, new Annotation(SPAN_KIND_TAG_KEY, "internal"));
+          put(SpanKind.SPAN_KIND_PRODUCER, new Annotation(SPAN_KIND_TAG_KEY, "producer"));
+          put(SpanKind.SPAN_KIND_SERVER, new Annotation(SPAN_KIND_TAG_KEY, "server"));
+          put(SpanKind.SPAN_KIND_UNSPECIFIED, new Annotation(SPAN_KIND_TAG_KEY, "unspecified"));
+          put(SpanKind.UNRECOGNIZED, new Annotation(SPAN_KIND_TAG_KEY, "unknown"));
+        }
+      };
+
+  public static KeyValue getAttrByKey(List attributesList, String key) {
+    return attributesList.stream().filter(kv -> key.equals(kv.getKey())).findFirst().orElse(null);
+  }
+
+  public static KeyValue buildKeyValue(String key, String value) {
+    return KeyValue.newBuilder()
+        .setKey(key)
+        .setValue(AnyValue.newBuilder().setStringValue(value).build())
+        .build();
+  }
+
+  static class WavefrontSpanAndLogs {
+    Span span;
+    SpanLogs spanLogs;
+
+    public WavefrontSpanAndLogs(Span span, SpanLogs spanLogs) {
+      this.span = span;
+      this.spanLogs = spanLogs;
+    }
+
+    public Span getSpan() {
+      return span;
+    }
+
+    public SpanLogs getSpanLogs() {
+      return spanLogs;
+    }
+  }
+
+  public static void exportToWavefront(
+      ExportTraceServiceRequest request,
+      ReportableEntityHandler spanHandler,
+      ReportableEntityHandler spanLogsHandler,
+      @Nullable Supplier preprocessorSupplier,
+      Pair, Counter> spanLogsDisabled,
+      Pair samplerAndCounter,
+      String defaultSource,
+      Set, String>> discoveredHeartbeatMetrics,
+      WavefrontInternalReporter internalReporter,
+      Set traceDerivedCustomTagKeys) {
+    ReportableEntityPreprocessor preprocessor = null;
+    if (preprocessorSupplier != null) {
+      preprocessor = preprocessorSupplier.get();
+    }
+
+    for (WavefrontSpanAndLogs spanAndLogs : fromOtlpRequest(request, preprocessor, defaultSource)) {
+      Span span = spanAndLogs.getSpan();
+      SpanLogs spanLogs = spanAndLogs.getSpanLogs();
+
+      if (wasFilteredByPreprocessor(span, spanHandler, preprocessor)) continue;
+
+      if (samplerAndCounter._1.sample(span, samplerAndCounter._2)) {
+        spanHandler.report(span);
+
+        if (shouldReportSpanLogs(spanLogs.getLogs().size(), spanLogsDisabled)) {
+          SpanUtils.addSpanLine(span, spanLogs);
+          spanLogsHandler.report(spanLogs);
+        }
+      }
+
+      // always report RED metrics irrespective of span sampling
+      discoveredHeartbeatMetrics.add(
+          reportREDMetrics(span, internalReporter, traceDerivedCustomTagKeys));
+    }
+  }
+
+  // TODO: consider transforming a single span and returning it for immedidate reporting in
+  //   wfSender. This could be more efficient, and also more reliable in the event the loops
+  //   below throw an error and we don't report any of the list.
+  @VisibleForTesting
+  static List fromOtlpRequest(
+      ExportTraceServiceRequest request,
+      @Nullable ReportableEntityPreprocessor preprocessor,
+      String defaultSource) {
+    List wfSpansAndLogs = new ArrayList<>();
+
+    for (ResourceSpans rSpans : request.getResourceSpansList()) {
+      Resource resource = rSpans.getResource();
+      OTLP_DATA_LOGGER.finest(() -> "Inbound OTLP Resource: " + resource);
+
+      for (ScopeSpans scopeSpans : rSpans.getScopeSpansList()) {
+        InstrumentationScope scope = scopeSpans.getScope();
+        OTLP_DATA_LOGGER.finest(() -> "Inbound OTLP Instrumentation Scope: " + scope);
+
+        for (io.opentelemetry.proto.trace.v1.Span otlpSpan : scopeSpans.getSpansList()) {
+          OTLP_DATA_LOGGER.finest(() -> "Inbound OTLP Span: " + otlpSpan);
+
+          wfSpansAndLogs.add(
+              transformAll(
+                  otlpSpan, resource.getAttributesList(), scope, preprocessor, defaultSource));
+        }
+      }
+    }
+    return wfSpansAndLogs;
+  }
+
+  @VisibleForTesting
+  static boolean wasFilteredByPreprocessor(
+      Span wfSpan,
+      ReportableEntityHandler spanHandler,
+      @Nullable ReportableEntityPreprocessor preprocessor) {
+    if (preprocessor == null) {
+      return false;
+    }
+
+    String[] messageHolder = new String[1];
+    if (!preprocessor.forSpan().filter(wfSpan, messageHolder)) {
+      if (messageHolder[0] != null) {
+        spanHandler.reject(wfSpan, messageHolder[0]);
+      } else {
+        spanHandler.block(wfSpan);
+      }
+      return true;
+    }
+
+    return false;
+  }
+
+  @VisibleForTesting
+  static WavefrontSpanAndLogs transformAll(
+      io.opentelemetry.proto.trace.v1.Span otlpSpan,
+      List resourceAttributes,
+      InstrumentationScope scope,
+      @Nullable ReportableEntityPreprocessor preprocessor,
+      String defaultSource) {
+    Span span = transformSpan(otlpSpan, resourceAttributes, scope, preprocessor, defaultSource);
+    SpanLogs logs = transformEvents(otlpSpan, span);
+    if (!logs.getLogs().isEmpty()) {
+      span.getAnnotations().add(new Annotation(SPAN_LOG_KEY, "true"));
+    }
+
+    OTLP_DATA_LOGGER.finest(() -> "Converted Wavefront Span: " + span);
+    if (!logs.getLogs().isEmpty()) {
+      OTLP_DATA_LOGGER.finest(() -> "Converted Wavefront SpanLogs: " + logs);
+    }
+
+    return new WavefrontSpanAndLogs(span, logs);
+  }
+
+  @VisibleForTesting
+  static Span transformSpan(
+      io.opentelemetry.proto.trace.v1.Span otlpSpan,
+      List resourceAttrs,
+      InstrumentationScope scope,
+      ReportableEntityPreprocessor preprocessor,
+      String defaultSource) {
+    Pair> sourceAndResourceAttrs =
+        sourceFromAttributes(resourceAttrs, defaultSource);
+    String source = sourceAndResourceAttrs._1;
+    resourceAttrs = sourceAndResourceAttrs._2;
+
+    // Order of arguments to Stream.of() matters: when a Resource Attribute and a Span Attribute
+    // happen to share the same key, we want the Span Attribute to "win" and be preserved.
+    List otlpAttributes =
+        Stream.of(resourceAttrs, otlpSpan.getAttributesList())
+            .flatMap(Collection::stream)
+            .collect(Collectors.toList());
+
+    List wfAnnotations = annotationsFromAttributes(otlpAttributes);
+
+    wfAnnotations.add(SPAN_KIND_ANNOTATION_HASH_MAP.get(otlpSpan.getKind()));
+    wfAnnotations.addAll(annotationsFromStatus(otlpSpan.getStatus()));
+    wfAnnotations.addAll(annotationsFromInstrumentationScope(scope));
+    wfAnnotations.addAll(annotationsFromDroppedCounts(otlpSpan));
+    wfAnnotations.addAll(annotationsFromTraceState(otlpSpan.getTraceState()));
+    wfAnnotations.addAll(annotationsFromParentSpanID(otlpSpan.getParentSpanId()));
+
+    String wfSpanId = SpanUtils.toStringId(otlpSpan.getSpanId());
+    String wfTraceId = SpanUtils.toStringId(otlpSpan.getTraceId());
+    long startTimeMs = TimeUnit.NANOSECONDS.toMillis(otlpSpan.getStartTimeUnixNano());
+    long durationMs =
+        otlpSpan.getEndTimeUnixNano() == 0
+            ? 0
+            : TimeUnit.NANOSECONDS.toMillis(
+                otlpSpan.getEndTimeUnixNano() - otlpSpan.getStartTimeUnixNano());
+
+    wavefront.report.Span toReturn =
+        wavefront.report.Span.newBuilder()
+            .setName(otlpSpan.getName())
+            .setSpanId(wfSpanId)
+            .setTraceId(wfTraceId)
+            .setStartMillis(startTimeMs)
+            .setDuration(durationMs)
+            .setAnnotations(wfAnnotations)
+            .setSource(source)
+            .setCustomer("dummy")
+            .build();
+
+    // apply preprocessor
+    if (preprocessor != null) {
+      preprocessor.forSpan().transform(toReturn);
+    }
+
+    // After preprocessor has run `transform()`, set required WF tags that may still be missing
+    List processedAnnotationList = setRequiredTags(toReturn.getAnnotations());
+    toReturn.setAnnotations(processedAnnotationList);
+    return toReturn;
+  }
+
+  @VisibleForTesting
+  static SpanLogs transformEvents(io.opentelemetry.proto.trace.v1.Span otlpSpan, Span wfSpan) {
+    ArrayList logs = new ArrayList<>();
+
+    for (io.opentelemetry.proto.trace.v1.Span.Event event : otlpSpan.getEventsList()) {
+      SpanLog log = new SpanLog();
+      log.setTimestamp(TimeUnit.NANOSECONDS.toMicros(event.getTimeUnixNano()));
+      Map fields = mapFromAttributes(event.getAttributesList());
+      fields.put(SPAN_EVENT_TAG_KEY, event.getName());
+      if (event.getDroppedAttributesCount() != 0) {
+        fields.put(OTEL_DROPPED_ATTRS_KEY, String.valueOf(event.getDroppedAttributesCount()));
+      }
+      log.setFields(fields);
+      logs.add(log);
+    }
+
+    return SpanLogs.newBuilder()
+        .setLogs(logs)
+        .setSpanId(wfSpan.getSpanId())
+        .setTraceId(wfSpan.getTraceId())
+        .setCustomer(wfSpan.getCustomer())
+        .build();
+  }
+
+  // Returns a String of the source value and the original List attributes except
+  // with the removal of the KeyValue determined to be the source.
+  @VisibleForTesting
+  static Pair> sourceFromAttributes(
+      List otlpAttributes, String defaultSource) {
+    // Order of keys in List matters: it determines precedence when multiple candidates exist.
+    List candidateKeys = Arrays.asList(SOURCE_KEY, "host.name", "hostname", "host.id");
+    Comparator keySorter = Comparator.comparing(kv -> candidateKeys.indexOf(kv.getKey()));
+
+    Optional sourceAttr =
+        otlpAttributes.stream()
+            .filter(kv -> candidateKeys.contains(kv.getKey()))
+            .sorted(keySorter)
+            .findFirst();
+
+    if (sourceAttr.isPresent()) {
+      List attributesWithoutSource = new ArrayList<>(otlpAttributes);
+      attributesWithoutSource.remove(sourceAttr.get());
+
+      return new Pair<>(fromAnyValue(sourceAttr.get().getValue()), attributesWithoutSource);
+    } else {
+      return new Pair<>(defaultSource, otlpAttributes);
+    }
+  }
+
+  @VisibleForTesting
+  static List annotationsFromAttributes(List attributesList) {
+    List annotations = new ArrayList<>();
+    for (KeyValue attribute : attributesList) {
+      String key = attribute.getKey().equals(SOURCE_KEY) ? "_source" : attribute.getKey();
+      Annotation.Builder annotationBuilder = Annotation.newBuilder().setKey(key);
+
+      if (!attribute.hasValue()) {
+        annotationBuilder.setValue("");
+      } else {
+        annotationBuilder.setValue(fromAnyValue(attribute.getValue()));
+      }
+      annotations.add(annotationBuilder.build());
+    }
+
+    return annotations;
+  }
+
+  @VisibleForTesting
+  static List annotationsFromInstrumentationScope(InstrumentationScope scope) {
+    if (scope == null || scope.getName().isEmpty()) return Collections.emptyList();
+
+    List annotations = new ArrayList<>();
+
+    // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk_exporters/non-otlp.md
+    annotations.add(new Annotation("otel.scope.name", scope.getName()));
+    if (!scope.getVersion().isEmpty()) {
+      annotations.add(new Annotation("otel.scope.version", scope.getVersion()));
+    }
+
+    return annotations;
+  }
+
+  @VisibleForTesting
+  static List annotationsFromDroppedCounts(
+      io.opentelemetry.proto.trace.v1.Span otlpSpan) {
+    List annotations = new ArrayList<>();
+    if (otlpSpan.getDroppedAttributesCount() != 0) {
+      annotations.add(
+          new Annotation(
+              OTEL_DROPPED_ATTRS_KEY, String.valueOf(otlpSpan.getDroppedAttributesCount())));
+    }
+    if (otlpSpan.getDroppedEventsCount() != 0) {
+      annotations.add(
+          new Annotation(
+              OTEL_DROPPED_EVENTS_KEY, String.valueOf(otlpSpan.getDroppedEventsCount())));
+    }
+    if (otlpSpan.getDroppedLinksCount() != 0) {
+      annotations.add(
+          new Annotation(OTEL_DROPPED_LINKS_KEY, String.valueOf(otlpSpan.getDroppedLinksCount())));
+    }
+
+    return annotations;
+  }
+
+  @VisibleForTesting
+  static Pair, String> reportREDMetrics(
+      Span span,
+      WavefrontInternalReporter internalReporter,
+      Set traceDerivedCustomTagKeys) {
+    Map annotations = mapFromAnnotations(span.getAnnotations());
+    List> spanTags =
+        span.getAnnotations().stream()
+            .map(a -> Pair.of(a.getKey(), a.getValue()))
+            .collect(Collectors.toList());
+
+    return reportWavefrontGeneratedData(
+        internalReporter,
+        span.getName(),
+        annotations.get(APPLICATION_TAG_KEY),
+        annotations.get(SERVICE_TAG_KEY),
+        annotations.get(CLUSTER_TAG_KEY),
+        annotations.get(SHARD_TAG_KEY),
+        span.getSource(),
+        annotations.getOrDefault(COMPONENT_TAG_KEY, NULL_TAG_VAL),
+        Boolean.parseBoolean(annotations.get(ERROR_TAG_KEY)),
+        TimeUnit.MILLISECONDS.toMicros(span.getDuration()),
+        traceDerivedCustomTagKeys,
+        spanTags);
+  }
+
+  @VisibleForTesting
+  static List setRequiredTags(List annotationList) {
+    Map tags = mapFromAnnotations(annotationList);
+    List requiredTags = new ArrayList<>();
+
+    if (!tags.containsKey(SERVICE_TAG_KEY)) {
+      tags.put(SERVICE_TAG_KEY, tags.getOrDefault(OTEL_SERVICE_NAME_KEY, DEFAULT_SERVICE_NAME));
+    }
+    tags.remove(OTEL_SERVICE_NAME_KEY);
+
+    tags.putIfAbsent(APPLICATION_TAG_KEY, DEFAULT_APPLICATION_NAME);
+    tags.putIfAbsent(CLUSTER_TAG_KEY, NULL_TAG_VAL);
+    tags.putIfAbsent(SHARD_TAG_KEY, NULL_TAG_VAL);
+
+    for (Map.Entry tagEntry : tags.entrySet()) {
+      requiredTags.add(
+          Annotation.newBuilder().setKey(tagEntry.getKey()).setValue(tagEntry.getValue()).build());
+    }
+
+    return requiredTags;
+  }
+
+  static long getSpansCount(ExportTraceServiceRequest request) {
+    return request.getResourceSpansList().stream()
+        .flatMapToLong(r -> r.getScopeSpansList().stream().mapToLong(ScopeSpans::getSpansCount))
+        .sum();
+  }
+
+  @VisibleForTesting
+  static boolean shouldReportSpanLogs(
+      int logsCount, Pair, Counter> spanLogsDisabled) {
+    return logsCount > 0
+        && !isFeatureDisabled(
+            spanLogsDisabled._1, SPANLOGS_DISABLED, spanLogsDisabled._2, logsCount);
+  }
+
+  @Nullable
+  static WavefrontInternalReporter createAndStartInternalReporter(
+      @Nullable WavefrontSender sender) {
+    if (sender == null) return null;
+
+    /*
+    Internal reporter should have a custom source identifying where the internal metrics came from.
+    This mirrors the behavior in the Custom Tracing Listener and Jaeger Listeners.
+     */
+    WavefrontInternalReporter reporter =
+        new WavefrontInternalReporter.Builder()
+            .prefixedWith(TRACING_DERIVED_PREFIX)
+            .withSource("otlp")
+            .reportMinuteDistribution()
+            .build(sender);
+    reporter.start(1, TimeUnit.MINUTES);
+    return reporter;
+  }
+
+  /**
+   * Converts an OTLP AnyValue object to its equivalent String representation. The implementation
+   * mimics {@code AsString()} from the OpenTelemetry Collector:
+   * https://github.com/open-telemetry/opentelemetry-collector/blob/cffbecb2ac9ee98e6a60d22f910760be48a94c55/model/pdata/common.go#L384
+   *
+   * 
We do not handle {@code KvlistValue} because the OpenTelemetry Specification for Attributes
+   * does not include maps as an allowed type of value:
+   * https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/common.md#attributes
+   *
+   * @param anyValue OTLP Attributes value in {@link AnyValue} format
+   * @return String representation of the {@link AnyValue}
+   */
+  static String fromAnyValue(AnyValue anyValue) {
+    if (anyValue.hasStringValue()) {
+      return anyValue.getStringValue();
+    } else if (anyValue.hasBoolValue()) {
+      return Boolean.toString(anyValue.getBoolValue());
+    } else if (anyValue.hasIntValue()) {
+      return Long.toString(anyValue.getIntValue());
+    } else if (anyValue.hasDoubleValue()) {
+      return Double.toString(anyValue.getDoubleValue());
+    } else if (anyValue.hasArrayValue()) {
+      List values = anyValue.getArrayValue().getValuesList();
+      return values.stream()
+          .map(OtlpTraceUtils::fromAnyValue)
+          .collect(Collectors.joining(", ", "[", "]"));
+    } else if (anyValue.hasKvlistValue()) {
+      OTLP_DATA_LOGGER.finest(() -> "Encountered KvlistValue but cannot convert to String");
+    } else if (anyValue.hasBytesValue()) {
+      return Base64.getEncoder().encodeToString(anyValue.getBytesValue().toByteArray());
+    }
+    return "";
+  }
+
+  static Map mapFromAttributes(List attributes) {
+    Map map = new HashMap<>();
+    for (KeyValue attribute : attributes) {
+      map.put(attribute.getKey(), fromAnyValue(attribute.getValue()));
+    }
+    return map;
+  }
+
+  private static Map mapFromAnnotations(List annotations) {
+    Map map = new HashMap<>();
+    for (Annotation annotation : annotations) {
+      map.put(annotation.getKey(), annotation.getValue());
+    }
+    return map;
+  }
+
+  private static List annotationsFromStatus(Status otlpStatus) {
+    if (otlpStatus.getCode() != Status.StatusCode.STATUS_CODE_ERROR) return Collections.emptyList();
+
+    List statusAnnotations = new ArrayList<>();
+    statusAnnotations.add(new Annotation(ERROR_TAG_KEY, ERROR_SPAN_TAG_VAL));
+
+    if (!otlpStatus.getMessage().isEmpty()) {
+      statusAnnotations.add(new Annotation(OTEL_STATUS_DESCRIPTION_KEY, otlpStatus.getMessage()));
+    }
+    return statusAnnotations;
+  }
+
+  private static List annotationsFromTraceState(String state) {
+    if (state == null || state.isEmpty()) return Collections.emptyList();
+
+    return Collections.singletonList(new Annotation("w3c.tracestate", state));
+  }
+
+  private static List annotationsFromParentSpanID(ByteString parentSpanId) {
+    if (parentSpanId == null || parentSpanId.equals(ByteString.EMPTY))
+      return Collections.emptyList();
+
+    return Collections.singletonList(
+        new Annotation(PARENT_KEY, SpanUtils.toStringId(parentSpanId)));
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/tracing/CustomTracingPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/CustomTracingPortUnificationHandler.java
new file mode 100644
index 000000000..8beb1b420
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/CustomTracingPortUnificationHandler.java
@@ -0,0 +1,224 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportHeartbeats;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportWavefrontGeneratedData;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.ERROR_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.NULL_TAG_VAL;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static org.apache.commons.lang3.ObjectUtils.firstNonNull;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.ingester.ReportableEntityDecoder;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import io.netty.channel.ChannelHandler;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
+
+/**
+ * Handler that process trace data sent from tier 1 SDK.
+ *
+ * @author djia@vmware.com
+ */
+@ChannelHandler.Sharable
+public class CustomTracingPortUnificationHandler extends TracePortUnificationHandler {
+  private static final Logger logger =
+      Logger.getLogger(CustomTracingPortUnificationHandler.class.getCanonicalName());
+  @Nullable private final WavefrontSender wfSender;
+  private final WavefrontInternalReporter wfInternalReporter;
+  private final Set, String>> discoveredHeartbeatMetrics;
+  private final Set traceDerivedCustomTagKeys;
+  private final String proxyLevelApplicationName;
+  private final String proxyLevelServiceName;
+
+  /**
+   * @param handle handle/port number.
+   * @param tokenAuthenticator {@link TokenAuthenticator} for incoming requests.
+   * @param healthCheckManager shared health check endpoint handler.
+   * @param traceDecoder trace decoders.
+   * @param spanLogsDecoder span logs decoders.
+   * @param preprocessor preprocessor.
+   * @param handlerFactory factory for ReportableEntityHandler objects.
+   * @param sampler sampler.
+   * @param traceDisabled supplier for backend-controlled feature flag for spans.
+   * @param spanLogsDisabled supplier for backend-controlled feature flag for span logs.
+   * @param wfSender sender to send trace to Wavefront.
+   * @param traceDerivedCustomTagKeys custom tags added to derived RED metrics.
+   */
+  public CustomTracingPortUnificationHandler(
+      String handle,
+      TokenAuthenticator tokenAuthenticator,
+      HealthCheckManager healthCheckManager,
+      ReportableEntityDecoder traceDecoder,
+      ReportableEntityDecoder spanLogsDecoder,
+      @Nullable Supplier preprocessor,
+      ReportableEntityHandlerFactory handlerFactory,
+      SpanSampler sampler,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      @Nullable WavefrontSender wfSender,
+      @Nullable WavefrontInternalReporter wfInternalReporter,
+      Set traceDerivedCustomTagKeys,
+      @Nullable String customTracingApplicationName,
+      @Nullable String customTracingServiceName) {
+    this(
+        handle,
+        tokenAuthenticator,
+        healthCheckManager,
+        traceDecoder,
+        spanLogsDecoder,
+        preprocessor,
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)),
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, handle)),
+        sampler,
+        traceDisabled,
+        spanLogsDisabled,
+        wfSender,
+        wfInternalReporter,
+        traceDerivedCustomTagKeys,
+        customTracingApplicationName,
+        customTracingServiceName);
+  }
+
+  @VisibleForTesting
+  public CustomTracingPortUnificationHandler(
+      String handle,
+      TokenAuthenticator tokenAuthenticator,
+      HealthCheckManager healthCheckManager,
+      ReportableEntityDecoder traceDecoder,
+      ReportableEntityDecoder spanLogsDecoder,
+      @Nullable Supplier preprocessor,
+      final ReportableEntityHandler handler,
+      final ReportableEntityHandler spanLogsHandler,
+      SpanSampler sampler,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      @Nullable WavefrontSender wfSender,
+      @Nullable WavefrontInternalReporter wfInternalReporter,
+      Set traceDerivedCustomTagKeys,
+      @Nullable String customTracingApplicationName,
+      @Nullable String customTracingServiceName) {
+    super(
+        handle,
+        tokenAuthenticator,
+        healthCheckManager,
+        traceDecoder,
+        spanLogsDecoder,
+        preprocessor,
+        handler,
+        spanLogsHandler,
+        sampler,
+        traceDisabled,
+        spanLogsDisabled);
+    this.wfSender = wfSender;
+    this.wfInternalReporter = wfInternalReporter;
+    this.discoveredHeartbeatMetrics = Sets.newConcurrentHashSet();
+    this.traceDerivedCustomTagKeys = traceDerivedCustomTagKeys;
+    this.proxyLevelApplicationName =
+        StringUtils.isBlank(customTracingApplicationName)
+            ? "defaultApp"
+            : customTracingApplicationName;
+    this.proxyLevelServiceName =
+        StringUtils.isBlank(customTracingServiceName) ? "defaultService" : customTracingServiceName;
+  }
+
+  @Override
+  protected void report(Span object) {
+    // report converted metrics/histograms from the span
+    String applicationName = null;
+    String serviceName = null;
+    String cluster = NULL_TAG_VAL;
+    String shard = NULL_TAG_VAL;
+    String componentTagValue = NULL_TAG_VAL;
+    String isError = "false";
+    List annotations = object.getAnnotations();
+    for (Annotation annotation : annotations) {
+      switch (annotation.getKey()) {
+        case APPLICATION_TAG_KEY:
+          applicationName = annotation.getValue();
+          continue;
+        case SERVICE_TAG_KEY:
+          serviceName = annotation.getValue();
+          continue;
+        case CLUSTER_TAG_KEY:
+          cluster = annotation.getValue();
+          continue;
+        case SHARD_TAG_KEY:
+          shard = annotation.getValue();
+          continue;
+        case COMPONENT_TAG_KEY:
+          componentTagValue = annotation.getValue();
+          continue;
+        case ERROR_TAG_KEY:
+          isError = annotation.getValue();
+          continue;
+      }
+    }
+    if (applicationName == null || serviceName == null) {
+      logger.warning(
+          "Ingested spans discarded because span application/service name is " + "missing.");
+      discardedSpans.inc();
+      return;
+    }
+    handler.report(object);
+    // update application and service for red metrics
+    applicationName = firstNonNull(applicationName, proxyLevelApplicationName);
+    serviceName = firstNonNull(serviceName, proxyLevelServiceName);
+    if (wfInternalReporter != null) {
+      List> spanTags =
+          annotations.stream()
+              .map(a -> new Pair<>(a.getKey(), a.getValue()))
+              .collect(Collectors.toList());
+      discoveredHeartbeatMetrics.add(
+          reportWavefrontGeneratedData(
+              wfInternalReporter,
+              object.getName(),
+              applicationName,
+              serviceName,
+              cluster,
+              shard,
+              object.getSource(),
+              componentTagValue,
+              Boolean.parseBoolean(isError),
+              millisToMicros(object.getDuration()),
+              traceDerivedCustomTagKeys,
+              spanTags,
+              true));
+      try {
+        reportHeartbeats(wfSender, discoveredHeartbeatMetrics, "wavefront-generated");
+      } catch (IOException e) {
+        logger.log(Level.WARNING, "Cannot report heartbeat metric to wavefront");
+      }
+    }
+  }
+
+  private long millisToMicros(long millis) {
+    return millis * 1000;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerGrpcCollectorHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerGrpcCollectorHandler.java
new file mode 100644
index 000000000..c5ea4ec1f
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerGrpcCollectorHandler.java
@@ -0,0 +1,195 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.agent.listeners.tracing.JaegerProtobufUtils.processBatch;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.TRACING_DERIVED_PREFIX;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportHeartbeats;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.Sets;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.common.NamedThreadFactory;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import io.grpc.stub.StreamObserver;
+import io.opentelemetry.exporter.jaeger.proto.api_v2.Collector;
+import io.opentelemetry.exporter.jaeger.proto.api_v2.CollectorServiceGrpc;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
+
+/**
+ * Handler that processes trace data in Jaeger ProtoBuf format and converts them to Wavefront format
+ *
+ * @author Hao Song (songhao@vmware.com)
+ */
+public class JaegerGrpcCollectorHandler extends CollectorServiceGrpc.CollectorServiceImplBase
+    implements Runnable, Closeable {
+  protected static final Logger logger =
+      Logger.getLogger(JaegerTChannelCollectorHandler.class.getCanonicalName());
+
+  private static final String JAEGER_COMPONENT = "jaeger";
+  private static final String DEFAULT_SOURCE = "jaeger";
+
+  private final ReportableEntityHandler spanHandler;
+  private final ReportableEntityHandler spanLogsHandler;
+  @Nullable private final WavefrontSender wfSender;
+  @Nullable private final WavefrontInternalReporter wfInternalReporter;
+  private final Supplier traceDisabled;
+  private final Supplier spanLogsDisabled;
+  private final Supplier preprocessorSupplier;
+  private final SpanSampler sampler;
+  private final String proxyLevelApplicationName;
+  private final Set traceDerivedCustomTagKeys;
+
+  private final Counter receivedSpansTotal;
+  private final Counter discardedTraces;
+  private final Counter discardedBatches;
+  private final Counter processedBatches;
+  private final Counter failedBatches;
+  private final Counter discardedSpansBySampler;
+  private final Set, String>> discoveredHeartbeatMetrics;
+  private final ScheduledExecutorService scheduledExecutorService;
+
+  public JaegerGrpcCollectorHandler(
+      String handle,
+      ReportableEntityHandlerFactory handlerFactory,
+      @Nullable WavefrontSender wfSender,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      @Nullable Supplier preprocessor,
+      SpanSampler sampler,
+      @Nullable String traceJaegerApplicationName,
+      Set traceDerivedCustomTagKeys) {
+    this(
+        handle,
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)),
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, handle)),
+        wfSender,
+        traceDisabled,
+        spanLogsDisabled,
+        preprocessor,
+        sampler,
+        traceJaegerApplicationName,
+        traceDerivedCustomTagKeys);
+  }
+
+  public JaegerGrpcCollectorHandler(
+      String handle,
+      ReportableEntityHandler spanHandler,
+      ReportableEntityHandler spanLogsHandler,
+      @Nullable WavefrontSender wfSender,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      @Nullable Supplier preprocessor,
+      SpanSampler sampler,
+      @Nullable String traceJaegerApplicationName,
+      Set traceDerivedCustomTagKeys) {
+    this.spanHandler = spanHandler;
+    this.spanLogsHandler = spanLogsHandler;
+    this.wfSender = wfSender;
+    this.traceDisabled = traceDisabled;
+    this.spanLogsDisabled = spanLogsDisabled;
+    this.preprocessorSupplier = preprocessor;
+    this.sampler = sampler;
+    this.proxyLevelApplicationName =
+        StringUtils.isBlank(traceJaegerApplicationName)
+            ? "Jaeger"
+            : traceJaegerApplicationName.trim();
+    this.traceDerivedCustomTagKeys = traceDerivedCustomTagKeys;
+    this.discardedTraces = Metrics.newCounter(new MetricName("spans." + handle, "", "discarded"));
+    this.discardedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "discarded"));
+    this.processedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "processed"));
+    this.failedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "failed"));
+    this.discardedSpansBySampler =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "sampler.discarded"));
+    this.receivedSpansTotal =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "received.total"));
+    this.discoveredHeartbeatMetrics = Sets.newConcurrentHashSet();
+    this.scheduledExecutorService =
+        Executors.newScheduledThreadPool(1, new NamedThreadFactory("jaeger-heart-beater"));
+    scheduledExecutorService.scheduleAtFixedRate(this, 1, 1, TimeUnit.MINUTES);
+
+    if (wfSender != null) {
+      wfInternalReporter =
+          new WavefrontInternalReporter.Builder()
+              .prefixedWith(TRACING_DERIVED_PREFIX)
+              .withSource(DEFAULT_SOURCE)
+              .reportMinuteDistribution()
+              .build(wfSender);
+      // Start the reporter
+      wfInternalReporter.start(1, TimeUnit.MINUTES);
+    } else {
+      wfInternalReporter = null;
+    }
+  }
+
+  @Override
+  public void postSpans(
+      Collector.PostSpansRequest request,
+      StreamObserver responseObserver) {
+    try {
+      processBatch(
+          request.getBatch(),
+          null,
+          DEFAULT_SOURCE,
+          proxyLevelApplicationName,
+          spanHandler,
+          spanLogsHandler,
+          wfInternalReporter,
+          traceDisabled,
+          spanLogsDisabled,
+          preprocessorSupplier,
+          sampler,
+          traceDerivedCustomTagKeys,
+          discardedTraces,
+          discardedBatches,
+          discardedSpansBySampler,
+          discoveredHeartbeatMetrics,
+          receivedSpansTotal);
+      processedBatches.inc();
+    } catch (Exception e) {
+      failedBatches.inc();
+      logger.log(
+          Level.WARNING, "Jaeger Protobuf batch processing failed", Throwables.getRootCause(e));
+    }
+    responseObserver.onNext(Collector.PostSpansResponse.newBuilder().build());
+    responseObserver.onCompleted();
+  }
+
+  @Override
+  public void run() {
+    try {
+      reportHeartbeats(wfSender, discoveredHeartbeatMetrics, JAEGER_COMPONENT);
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Cannot report heartbeat metric to wavefront");
+    }
+  }
+
+  @Override
+  public void close() {
+    scheduledExecutorService.shutdownNow();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerPortUnificationHandler.java
new file mode 100644
index 000000000..fd3e4c040
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerPortUnificationHandler.java
@@ -0,0 +1,242 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.agent.channel.ChannelUtils.errorMessageWithRootCause;
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+import static com.wavefront.agent.listeners.tracing.JaegerThriftUtils.processBatch;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.TRACING_DERIVED_PREFIX;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportHeartbeats;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Sets;
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.agent.listeners.AbstractHttpOnlyHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.common.NamedThreadFactory;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import io.jaegertracing.thriftjava.Batch;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+import org.apache.thrift.TDeserializer;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
+
+/**
+ * Handler that processes Jaeger Thrift trace data over HTTP and converts them to Wavefront format.
+ *
+ * @author Han Zhang (zhanghan@vmware.com)
+ */
+public class JaegerPortUnificationHandler extends AbstractHttpOnlyHandler
+    implements Runnable, Closeable {
+  protected static final Logger logger =
+      Logger.getLogger(JaegerPortUnificationHandler.class.getCanonicalName());
+
+  private static final String JAEGER_COMPONENT = "jaeger";
+  private static final String DEFAULT_SOURCE = "jaeger";
+
+  private final ReportableEntityHandler spanHandler;
+  private final ReportableEntityHandler spanLogsHandler;
+  @Nullable private final WavefrontSender wfSender;
+  @Nullable private final WavefrontInternalReporter wfInternalReporter;
+  private final Supplier traceDisabled;
+  private final Supplier spanLogsDisabled;
+  private final Supplier preprocessorSupplier;
+  private final SpanSampler sampler;
+  private final String proxyLevelApplicationName;
+  private final Set traceDerivedCustomTagKeys;
+
+  private final Counter receivedSpansTotal;
+  private final Counter discardedTraces;
+  private final Counter discardedBatches;
+  private final Counter processedBatches;
+  private final Counter failedBatches;
+  private final Counter discardedSpansBySampler;
+  private final Set, String>> discoveredHeartbeatMetrics;
+  private final ScheduledExecutorService scheduledExecutorService;
+
+  private static final String JAEGER_VALID_PATH = "/api/traces/";
+  private static final String JAEGER_VALID_HTTP_METHOD = "POST";
+
+  public JaegerPortUnificationHandler(
+      String handle,
+      final TokenAuthenticator tokenAuthenticator,
+      final HealthCheckManager healthCheckManager,
+      ReportableEntityHandlerFactory handlerFactory,
+      @Nullable WavefrontSender wfSender,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      @Nullable Supplier preprocessor,
+      SpanSampler sampler,
+      @Nullable String traceJaegerApplicationName,
+      Set traceDerivedCustomTagKeys) {
+    this(
+        handle,
+        tokenAuthenticator,
+        healthCheckManager,
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)),
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, handle)),
+        wfSender,
+        traceDisabled,
+        spanLogsDisabled,
+        preprocessor,
+        sampler,
+        traceJaegerApplicationName,
+        traceDerivedCustomTagKeys);
+  }
+
+  @VisibleForTesting
+  JaegerPortUnificationHandler(
+      String handle,
+      final TokenAuthenticator tokenAuthenticator,
+      final HealthCheckManager healthCheckManager,
+      ReportableEntityHandler spanHandler,
+      ReportableEntityHandler spanLogsHandler,
+      @Nullable WavefrontSender wfSender,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      @Nullable Supplier preprocessor,
+      SpanSampler sampler,
+      @Nullable String traceJaegerApplicationName,
+      Set traceDerivedCustomTagKeys) {
+    super(tokenAuthenticator, healthCheckManager, handle);
+    this.spanHandler = spanHandler;
+    this.spanLogsHandler = spanLogsHandler;
+    this.wfSender = wfSender;
+    this.traceDisabled = traceDisabled;
+    this.spanLogsDisabled = spanLogsDisabled;
+    this.preprocessorSupplier = preprocessor;
+    this.sampler = sampler;
+    this.proxyLevelApplicationName =
+        StringUtils.isBlank(traceJaegerApplicationName)
+            ? "Jaeger"
+            : traceJaegerApplicationName.trim();
+    this.traceDerivedCustomTagKeys = traceDerivedCustomTagKeys;
+    this.discardedTraces = Metrics.newCounter(new MetricName("spans." + handle, "", "discarded"));
+    this.discardedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "discarded"));
+    this.processedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "processed"));
+    this.failedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "failed"));
+    this.discardedSpansBySampler =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "sampler.discarded"));
+    this.receivedSpansTotal =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "received.total"));
+    this.discoveredHeartbeatMetrics = Sets.newConcurrentHashSet();
+    this.scheduledExecutorService =
+        Executors.newScheduledThreadPool(1, new NamedThreadFactory("jaeger-heart-beater"));
+    scheduledExecutorService.scheduleAtFixedRate(this, 1, 1, TimeUnit.MINUTES);
+
+    if (wfSender != null) {
+      wfInternalReporter =
+          new WavefrontInternalReporter.Builder()
+              .prefixedWith(TRACING_DERIVED_PREFIX)
+              .withSource(DEFAULT_SOURCE)
+              .reportMinuteDistribution()
+              .build(wfSender);
+      // Start the reporter
+      wfInternalReporter.start(1, TimeUnit.MINUTES);
+    } else {
+      wfInternalReporter = null;
+    }
+  }
+
+  @Override
+  protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request)
+      throws URISyntaxException {
+    URI uri = new URI(request.uri());
+    String path = uri.getPath().endsWith("/") ? uri.getPath() : uri.getPath() + "/";
+
+    // Validate Uri Path and HTTP method of incoming Jaeger spans.
+    if (!path.equals(JAEGER_VALID_PATH)) {
+      writeHttpResponse(ctx, HttpResponseStatus.BAD_REQUEST, "Unsupported URL path.", request);
+      logWarning("Requested URI path '" + path + "' is not supported.", null, ctx);
+      return;
+    }
+    if (!request.method().toString().equalsIgnoreCase(JAEGER_VALID_HTTP_METHOD)) {
+      writeHttpResponse(ctx, HttpResponseStatus.BAD_REQUEST, "Unsupported Http method.", request);
+      logWarning(
+          "Requested http method '" + request.method().toString() + "' is not supported.",
+          null,
+          ctx);
+      return;
+    }
+
+    HttpResponseStatus status;
+    StringBuilder output = new StringBuilder();
+
+    try {
+      byte[] bytesArray = new byte[request.content().nioBuffer().remaining()];
+      request.content().nioBuffer().get(bytesArray, 0, bytesArray.length);
+      Batch batch = new Batch();
+      new TDeserializer().deserialize(batch, bytesArray);
+
+      processBatch(
+          batch,
+          output,
+          DEFAULT_SOURCE,
+          proxyLevelApplicationName,
+          spanHandler,
+          spanLogsHandler,
+          wfInternalReporter,
+          traceDisabled,
+          spanLogsDisabled,
+          preprocessorSupplier,
+          sampler,
+          traceDerivedCustomTagKeys,
+          discardedTraces,
+          discardedBatches,
+          discardedSpansBySampler,
+          discoveredHeartbeatMetrics,
+          receivedSpansTotal);
+      status = HttpResponseStatus.ACCEPTED;
+      processedBatches.inc();
+    } catch (Exception e) {
+      failedBatches.inc();
+      output.append(errorMessageWithRootCause(e));
+      status = HttpResponseStatus.BAD_REQUEST;
+      logger.log(Level.WARNING, "Jaeger HTTP batch processing failed", Throwables.getRootCause(e));
+    }
+    writeHttpResponse(ctx, status, output, request);
+  }
+
+  @Override
+  public void run() {
+    try {
+      reportHeartbeats(wfSender, discoveredHeartbeatMetrics, JAEGER_COMPONENT);
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Cannot report heartbeat metric to wavefront");
+    }
+  }
+
+  @Override
+  public void close() {
+    scheduledExecutorService.shutdownNow();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerProtobufUtils.java b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerProtobufUtils.java
new file mode 100644
index 000000000..6f44c2596
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerProtobufUtils.java
@@ -0,0 +1,392 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.google.protobuf.util.Durations.toMicros;
+import static com.google.protobuf.util.Durations.toMillis;
+import static com.google.protobuf.util.Timestamps.toMicros;
+import static com.google.protobuf.util.Timestamps.toMillis;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPANLOGS_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.ERROR_SPAN_TAG_VAL;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportWavefrontGeneratedData;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.ERROR_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.NULL_TAG_VAL;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SOURCE_KEY;
+
+import com.google.common.collect.ImmutableSet;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.common.TraceConstants;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.yammer.metrics.core.Counter;
+import io.opentelemetry.exporter.jaeger.proto.api_v2.Model;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
+
+/**
+ * Utility methods for processing Jaeger Protobuf trace data.
+ *
+ * @author Hao Song (songhao@vmware.com)
+ */
+public abstract class JaegerProtobufUtils {
+  protected static final Logger logger =
+      Logger.getLogger(JaegerProtobufUtils.class.getCanonicalName());
+
+  // TODO: support sampling
+  private static final Set IGNORE_TAGS = ImmutableSet.of("sampler.type", "sampler.param");
+  private static final Logger JAEGER_DATA_LOGGER = Logger.getLogger("JaegerDataLogger");
+
+  private JaegerProtobufUtils() {}
+
+  public static void processBatch(
+      Model.Batch batch,
+      @Nullable StringBuilder output,
+      String sourceName,
+      String applicationName,
+      ReportableEntityHandler spanHandler,
+      ReportableEntityHandler spanLogsHandler,
+      @Nullable WavefrontInternalReporter wfInternalReporter,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      Supplier preprocessorSupplier,
+      SpanSampler sampler,
+      Set traceDerivedCustomTagKeys,
+      Counter discardedTraces,
+      Counter discardedBatches,
+      Counter discardedSpansBySampler,
+      Set, String>> discoveredHeartbeatMetrics,
+      Counter receivedSpansTotal) {
+    String serviceName = batch.getProcess().getServiceName();
+    List processAnnotations = new ArrayList<>();
+    boolean isSourceProcessTagPresent = false;
+    String cluster = NULL_TAG_VAL;
+    String shard = NULL_TAG_VAL;
+
+    if (batch.getProcess().getTagsList() != null) {
+      for (Model.KeyValue tag : batch.getProcess().getTagsList()) {
+        if (tag.getKey().equals(APPLICATION_TAG_KEY) && tag.getVType() == Model.ValueType.STRING) {
+          applicationName = tag.getVStr();
+          continue;
+        }
+
+        if (tag.getKey().equals(CLUSTER_TAG_KEY) && tag.getVType() == Model.ValueType.STRING) {
+          cluster = tag.getVStr();
+          continue;
+        }
+
+        if (tag.getKey().equals(SHARD_TAG_KEY) && tag.getVType() == Model.ValueType.STRING) {
+          shard = tag.getVStr();
+          continue;
+        }
+
+        // source tag precedence :
+        // "source" in span tag > "source" in process tag > "hostname" in process tag >
+        // DEFAULT
+        if (tag.getKey().equals("hostname") && tag.getVType() == Model.ValueType.STRING) {
+          if (!isSourceProcessTagPresent) {
+            sourceName = tag.getVStr();
+          }
+          continue;
+        }
+
+        if (tag.getKey().equals(SOURCE_KEY) && tag.getVType() == Model.ValueType.STRING) {
+          sourceName = tag.getVStr();
+          isSourceProcessTagPresent = true;
+          continue;
+        }
+
+        if (tag.getKey().equals(SERVICE_TAG_KEY) && tag.getVType() == Model.ValueType.STRING) {
+          // ignore "service" tags, since service is a field on the span
+          continue;
+        }
+
+        Annotation annotation = tagToAnnotation(tag);
+        processAnnotations.add(annotation);
+      }
+    }
+
+    if (isFeatureDisabled(traceDisabled, SPAN_DISABLED, discardedBatches, output)) {
+      discardedTraces.inc(batch.getSpansCount());
+      receivedSpansTotal.inc(batch.getSpansCount());
+      return;
+    }
+    receivedSpansTotal.inc(batch.getSpansCount());
+    for (Model.Span span : batch.getSpansList()) {
+      processSpan(
+          span,
+          serviceName,
+          sourceName,
+          applicationName,
+          cluster,
+          shard,
+          processAnnotations,
+          spanHandler,
+          spanLogsHandler,
+          wfInternalReporter,
+          spanLogsDisabled,
+          preprocessorSupplier,
+          sampler,
+          traceDerivedCustomTagKeys,
+          discardedSpansBySampler,
+          discoveredHeartbeatMetrics);
+    }
+  }
+
+  private static void processSpan(
+      Model.Span span,
+      String serviceName,
+      String sourceName,
+      String applicationName,
+      String cluster,
+      String shard,
+      List processAnnotations,
+      ReportableEntityHandler spanHandler,
+      ReportableEntityHandler spanLogsHandler,
+      @Nullable WavefrontInternalReporter wfInternalReporter,
+      Supplier spanLogsDisabled,
+      Supplier preprocessorSupplier,
+      SpanSampler sampler,
+      Set traceDerivedCustomTagKeys,
+      Counter discardedSpansBySampler,
+      Set, String>> discoveredHeartbeatMetrics) {
+    List annotations = new ArrayList<>(processAnnotations);
+    // serviceName is mandatory in Jaeger
+    annotations.add(new Annotation(SERVICE_TAG_KEY, serviceName));
+
+    String componentTagValue = NULL_TAG_VAL;
+    boolean isError = false;
+
+    if (span.getTagsList() != null) {
+      for (Model.KeyValue tag : span.getTagsList()) {
+        if (IGNORE_TAGS.contains(tag.getKey())
+            || (tag.getVType() == Model.ValueType.STRING && StringUtils.isBlank(tag.getVStr()))) {
+          continue;
+        }
+
+        Annotation annotation = tagToAnnotation(tag);
+        if (annotation != null) {
+          switch (annotation.getKey()) {
+            case APPLICATION_TAG_KEY:
+              applicationName = annotation.getValue();
+              continue;
+            case CLUSTER_TAG_KEY:
+              cluster = annotation.getValue();
+              continue;
+            case SHARD_TAG_KEY:
+              shard = annotation.getValue();
+              continue;
+            case SOURCE_KEY:
+              // Do not add source to annotation span tag list.
+              sourceName = annotation.getValue();
+              continue;
+            case SERVICE_TAG_KEY:
+              // Do not use service tag from annotations, use field instead
+              continue;
+            case COMPONENT_TAG_KEY:
+              componentTagValue = annotation.getValue();
+              break;
+            case ERROR_TAG_KEY:
+              // only error=true is supported
+              isError = annotation.getValue().equals(ERROR_SPAN_TAG_VAL);
+              break;
+          }
+          annotations.add(annotation);
+        }
+      }
+    }
+
+    // Add all wavefront indexed tags. These are set based on below hierarchy.
+    // Span Level > Process Level > Proxy Level > Default
+    annotations.add(new Annotation(APPLICATION_TAG_KEY, applicationName));
+    annotations.add(new Annotation(CLUSTER_TAG_KEY, cluster));
+    annotations.add(new Annotation(SHARD_TAG_KEY, shard));
+
+    if (span.getReferencesList() != null) {
+      for (Model.SpanRef reference : span.getReferencesList()) {
+        switch (reference.getRefType()) {
+          case CHILD_OF:
+            if (!reference.getSpanId().isEmpty()) {
+              annotations.add(
+                  new Annotation(
+                      TraceConstants.PARENT_KEY, SpanUtils.toStringId(reference.getSpanId())));
+            }
+            break;
+          case FOLLOWS_FROM:
+            if (!reference.getSpanId().isEmpty()) {
+              annotations.add(
+                  new Annotation(
+                      TraceConstants.FOLLOWS_FROM_KEY,
+                      SpanUtils.toStringId(reference.getSpanId())));
+            }
+          default:
+        }
+      }
+    }
+
+    if (!spanLogsDisabled.get() && span.getLogsCount() > 0) {
+      annotations.add(new Annotation("_spanLogs", "true"));
+    }
+
+    Span wavefrontSpan =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setName(span.getOperationName())
+            .setSource(sourceName)
+            .setSpanId(SpanUtils.toStringId(span.getSpanId()))
+            .setTraceId(SpanUtils.toStringId(span.getTraceId()))
+            .setStartMillis(toMillis(span.getStartTime()))
+            .setDuration(toMillis(span.getDuration()))
+            .setAnnotations(annotations)
+            .build();
+
+    // Log Jaeger spans as well as Wavefront spans for debugging purposes.
+    if (JAEGER_DATA_LOGGER.isLoggable(Level.FINEST)) {
+      JAEGER_DATA_LOGGER.info("Inbound Jaeger span: " + span.toString());
+      JAEGER_DATA_LOGGER.info("Converted Wavefront span: " + wavefrontSpan.toString());
+    }
+
+    if (preprocessorSupplier != null) {
+      ReportableEntityPreprocessor preprocessor = preprocessorSupplier.get();
+      String[] messageHolder = new String[1];
+      preprocessor.forSpan().transform(wavefrontSpan);
+      if (!preprocessor.forSpan().filter(wavefrontSpan, messageHolder)) {
+        if (messageHolder[0] != null) {
+          spanHandler.reject(wavefrontSpan, messageHolder[0]);
+        } else {
+          spanHandler.block(wavefrontSpan);
+        }
+        return;
+      }
+    }
+    if (sampler.sample(wavefrontSpan, discardedSpansBySampler)) {
+      spanHandler.report(wavefrontSpan);
+      if (span.getLogsCount() > 0
+          && !isFeatureDisabled(spanLogsDisabled, SPANLOGS_DISABLED, null)) {
+        SpanLogs spanLogs =
+            SpanLogs.newBuilder()
+                .setCustomer("default")
+                .setTraceId(wavefrontSpan.getTraceId())
+                .setSpanId(wavefrontSpan.getSpanId())
+                .setLogs(
+                    span.getLogsList().stream()
+                        .map(
+                            x -> {
+                              Map fields = new HashMap<>(x.getFieldsCount());
+                              x.getFieldsList()
+                                  .forEach(
+                                      t -> {
+                                        switch (t.getVType()) {
+                                          case STRING:
+                                            fields.put(t.getKey(), t.getVStr());
+                                            break;
+                                          case BOOL:
+                                            fields.put(t.getKey(), String.valueOf(t.getVBool()));
+                                            break;
+                                          case INT64:
+                                            fields.put(t.getKey(), String.valueOf(t.getVInt64()));
+                                            break;
+                                          case FLOAT64:
+                                            fields.put(t.getKey(), String.valueOf(t.getVFloat64()));
+                                            break;
+                                          case BINARY:
+                                            // ignore
+                                          default:
+                                        }
+                                      });
+                              return SpanLog.newBuilder()
+                                  .setTimestamp(toMicros(x.getTimestamp()))
+                                  .setFields(fields)
+                                  .build();
+                            })
+                        .collect(Collectors.toList()))
+                .build();
+        spanLogsHandler.report(spanLogs);
+      }
+    }
+
+    // report stats irrespective of span sampling.
+    if (wfInternalReporter != null) {
+      // Set post preprocessor rule values and report converted metrics/histograms from the
+      // span
+      List processedAnnotations = wavefrontSpan.getAnnotations();
+      for (Annotation processedAnnotation : processedAnnotations) {
+        switch (processedAnnotation.getKey()) {
+          case APPLICATION_TAG_KEY:
+            applicationName = processedAnnotation.getValue();
+            continue;
+          case SERVICE_TAG_KEY:
+            serviceName = processedAnnotation.getValue();
+            continue;
+          case CLUSTER_TAG_KEY:
+            cluster = processedAnnotation.getValue();
+            continue;
+          case SHARD_TAG_KEY:
+            shard = processedAnnotation.getValue();
+            continue;
+          case COMPONENT_TAG_KEY:
+            componentTagValue = processedAnnotation.getValue();
+            continue;
+          case ERROR_TAG_KEY:
+            isError = processedAnnotation.getValue().equals(ERROR_SPAN_TAG_VAL);
+            continue;
+        }
+      }
+      List> spanTags =
+          processedAnnotations.stream()
+              .map(a -> new Pair<>(a.getKey(), a.getValue()))
+              .collect(Collectors.toList());
+      discoveredHeartbeatMetrics.add(
+          reportWavefrontGeneratedData(
+              wfInternalReporter,
+              wavefrontSpan.getName(),
+              applicationName,
+              serviceName,
+              cluster,
+              shard,
+              wavefrontSpan.getSource(),
+              componentTagValue,
+              isError,
+              toMicros(span.getDuration()),
+              traceDerivedCustomTagKeys,
+              spanTags,
+              true));
+    }
+  }
+
+  @Nullable
+  private static Annotation tagToAnnotation(Model.KeyValue tag) {
+    switch (tag.getVType()) {
+      case BOOL:
+        return new Annotation(tag.getKey(), String.valueOf(tag.getVBool()));
+      case INT64:
+        return new Annotation(tag.getKey(), String.valueOf(tag.getVInt64()));
+      case FLOAT64:
+        return new Annotation(tag.getKey(), String.valueOf(tag.getVFloat64()));
+      case STRING:
+        return new Annotation(tag.getKey(), tag.getVStr());
+      case BINARY:
+      default:
+        return null;
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerTChannelCollectorHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerTChannelCollectorHandler.java
new file mode 100644
index 000000000..689d57388
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerTChannelCollectorHandler.java
@@ -0,0 +1,201 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.agent.listeners.tracing.JaegerThriftUtils.processBatch;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.TRACING_DERIVED_PREFIX;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportHeartbeats;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.Sets;
+import com.uber.tchannel.api.handlers.ThriftRequestHandler;
+import com.uber.tchannel.messages.ThriftRequest;
+import com.uber.tchannel.messages.ThriftResponse;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.common.NamedThreadFactory;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import io.jaegertracing.thriftjava.Batch;
+import io.jaegertracing.thriftjava.Collector;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
+
+/**
+ * Handler that processes trace data in Jaeger Thrift compact format and converts them to Wavefront
+ * format
+ *
+ * @author vasily@wavefront.com
+ */
+public class JaegerTChannelCollectorHandler
+    extends ThriftRequestHandler
+    implements Runnable, Closeable {
+  protected static final Logger logger =
+      Logger.getLogger(JaegerTChannelCollectorHandler.class.getCanonicalName());
+
+  private static final String JAEGER_COMPONENT = "jaeger";
+  private static final String DEFAULT_SOURCE = "jaeger";
+
+  private final ReportableEntityHandler spanHandler;
+  private final ReportableEntityHandler spanLogsHandler;
+  @Nullable private final WavefrontSender wfSender;
+  @Nullable private final WavefrontInternalReporter wfInternalReporter;
+  private final Supplier traceDisabled;
+  private final Supplier spanLogsDisabled;
+  private final Supplier preprocessorSupplier;
+  private final SpanSampler sampler;
+  private final String proxyLevelApplicationName;
+  private final Set traceDerivedCustomTagKeys;
+
+  private final Counter receivedSpansTotal;
+  private final Counter discardedTraces;
+  private final Counter discardedBatches;
+  private final Counter processedBatches;
+  private final Counter failedBatches;
+  private final Counter discardedSpansBySampler;
+  private final Set, String>> discoveredHeartbeatMetrics;
+  private final ScheduledExecutorService scheduledExecutorService;
+
+  public JaegerTChannelCollectorHandler(
+      String handle,
+      ReportableEntityHandlerFactory handlerFactory,
+      @Nullable WavefrontSender wfSender,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      @Nullable Supplier preprocessor,
+      SpanSampler sampler,
+      @Nullable String traceJaegerApplicationName,
+      Set traceDerivedCustomTagKeys) {
+    this(
+        handle,
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)),
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, handle)),
+        wfSender,
+        traceDisabled,
+        spanLogsDisabled,
+        preprocessor,
+        sampler,
+        traceJaegerApplicationName,
+        traceDerivedCustomTagKeys);
+  }
+
+  public JaegerTChannelCollectorHandler(
+      String handle,
+      ReportableEntityHandler spanHandler,
+      ReportableEntityHandler spanLogsHandler,
+      @Nullable WavefrontSender wfSender,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      @Nullable Supplier preprocessor,
+      SpanSampler sampler,
+      @Nullable String traceJaegerApplicationName,
+      Set traceDerivedCustomTagKeys) {
+    this.spanHandler = spanHandler;
+    this.spanLogsHandler = spanLogsHandler;
+    this.wfSender = wfSender;
+    this.traceDisabled = traceDisabled;
+    this.spanLogsDisabled = spanLogsDisabled;
+    this.preprocessorSupplier = preprocessor;
+    this.sampler = sampler;
+    this.proxyLevelApplicationName =
+        StringUtils.isBlank(traceJaegerApplicationName)
+            ? "Jaeger"
+            : traceJaegerApplicationName.trim();
+    this.traceDerivedCustomTagKeys = traceDerivedCustomTagKeys;
+    this.discardedTraces = Metrics.newCounter(new MetricName("spans." + handle, "", "discarded"));
+    this.discardedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "discarded"));
+    this.processedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "processed"));
+    this.failedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "failed"));
+    this.discardedSpansBySampler =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "sampler.discarded"));
+    this.receivedSpansTotal =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "received.total"));
+    this.discoveredHeartbeatMetrics = Sets.newConcurrentHashSet();
+    this.scheduledExecutorService =
+        Executors.newScheduledThreadPool(1, new NamedThreadFactory("jaeger-heart-beater"));
+    scheduledExecutorService.scheduleAtFixedRate(this, 1, 1, TimeUnit.MINUTES);
+
+    if (wfSender != null) {
+      wfInternalReporter =
+          new WavefrontInternalReporter.Builder()
+              .prefixedWith(TRACING_DERIVED_PREFIX)
+              .withSource(DEFAULT_SOURCE)
+              .reportMinuteDistribution()
+              .build(wfSender);
+      // Start the reporter
+      wfInternalReporter.start(1, TimeUnit.MINUTES);
+    } else {
+      wfInternalReporter = null;
+    }
+  }
+
+  @Override
+  public ThriftResponse handleImpl(
+      ThriftRequest request) {
+    for (Batch batch : request.getBody(Collector.submitBatches_args.class).getBatches()) {
+      try {
+        processBatch(
+            batch,
+            null,
+            DEFAULT_SOURCE,
+            proxyLevelApplicationName,
+            spanHandler,
+            spanLogsHandler,
+            wfInternalReporter,
+            traceDisabled,
+            spanLogsDisabled,
+            preprocessorSupplier,
+            sampler,
+            traceDerivedCustomTagKeys,
+            discardedTraces,
+            discardedBatches,
+            discardedSpansBySampler,
+            discoveredHeartbeatMetrics,
+            receivedSpansTotal);
+        processedBatches.inc();
+      } catch (Exception e) {
+        failedBatches.inc();
+        logger.log(
+            Level.WARNING, "Jaeger Thrift batch processing failed", Throwables.getRootCause(e));
+      }
+    }
+    return new ThriftResponse.Builder(request)
+        .setBody(new Collector.submitBatches_result())
+        .build();
+  }
+
+  @Override
+  public void run() {
+    try {
+      reportHeartbeats(wfSender, discoveredHeartbeatMetrics, JAEGER_COMPONENT);
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Cannot report heartbeat metric to wavefront");
+    }
+  }
+
+  @Override
+  public void close() {
+    scheduledExecutorService.shutdownNow();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerThriftUtils.java b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerThriftUtils.java
new file mode 100644
index 000000000..2849874c8
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/JaegerThriftUtils.java
@@ -0,0 +1,403 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPANLOGS_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.ERROR_SPAN_TAG_VAL;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportWavefrontGeneratedData;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.ERROR_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.NULL_TAG_VAL;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SOURCE_KEY;
+
+import com.google.common.collect.ImmutableSet;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.common.TraceConstants;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.yammer.metrics.core.Counter;
+import io.jaegertracing.thriftjava.Batch;
+import io.jaegertracing.thriftjava.SpanRef;
+import io.jaegertracing.thriftjava.Tag;
+import io.jaegertracing.thriftjava.TagType;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
+
+/**
+ * Utility methods for processing Jaeger Thrift trace data.
+ *
+ * @author Han Zhang (zhanghan@vmware.com)
+ */
+public abstract class JaegerThriftUtils {
+  protected static final Logger logger =
+      Logger.getLogger(JaegerThriftUtils.class.getCanonicalName());
+
+  // TODO: support sampling
+  private static final Set IGNORE_TAGS = ImmutableSet.of("sampler.type", "sampler.param");
+  private static final Logger JAEGER_DATA_LOGGER = Logger.getLogger("JaegerDataLogger");
+
+  private JaegerThriftUtils() {}
+
+  public static void processBatch(
+      Batch batch,
+      @Nullable StringBuilder output,
+      String sourceName,
+      String applicationName,
+      ReportableEntityHandler spanHandler,
+      ReportableEntityHandler spanLogsHandler,
+      @Nullable WavefrontInternalReporter wfInternalReporter,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      Supplier preprocessorSupplier,
+      SpanSampler sampler,
+      Set traceDerivedCustomTagKeys,
+      Counter discardedTraces,
+      Counter discardedBatches,
+      Counter discardedSpansBySampler,
+      Set, String>> discoveredHeartbeatMetrics,
+      Counter receivedSpansTotal) {
+    String serviceName = batch.getProcess().getServiceName();
+    List processAnnotations = new ArrayList<>();
+    boolean isSourceProcessTagPresent = false;
+    String cluster = NULL_TAG_VAL;
+    String shard = NULL_TAG_VAL;
+
+    if (batch.getProcess().getTags() != null) {
+      for (Tag tag : batch.getProcess().getTags()) {
+        if (tag.getKey().equals(APPLICATION_TAG_KEY) && tag.getVType() == TagType.STRING) {
+          applicationName = tag.getVStr();
+          continue;
+        }
+
+        if (tag.getKey().equals(CLUSTER_TAG_KEY) && tag.getVType() == TagType.STRING) {
+          cluster = tag.getVStr();
+          continue;
+        }
+
+        if (tag.getKey().equals(SHARD_TAG_KEY) && tag.getVType() == TagType.STRING) {
+          shard = tag.getVStr();
+          continue;
+        }
+
+        // source tag precedence :
+        // "source" in span tag > "source" in process tag > "hostname" in process tag >
+        // DEFAULT
+        if (tag.getKey().equals("hostname") && tag.getVType() == TagType.STRING) {
+          if (!isSourceProcessTagPresent) {
+            sourceName = tag.getVStr();
+          }
+          continue;
+        }
+
+        if (tag.getKey().equals(SOURCE_KEY) && tag.getVType() == TagType.STRING) {
+          sourceName = tag.getVStr();
+          isSourceProcessTagPresent = true;
+          continue;
+        }
+
+        if (tag.getKey().equals(SERVICE_TAG_KEY) && tag.getVType() == TagType.STRING) {
+          // ignore "service" tags, since service is a field on the span
+          continue;
+        }
+
+        Annotation annotation = tagToAnnotation(tag);
+        processAnnotations.add(annotation);
+      }
+    }
+    if (isFeatureDisabled(traceDisabled, SPAN_DISABLED, discardedBatches, output)) {
+      discardedTraces.inc(batch.getSpansSize());
+      receivedSpansTotal.inc(batch.getSpansSize());
+      return;
+    }
+    receivedSpansTotal.inc(batch.getSpansSize());
+    for (io.jaegertracing.thriftjava.Span span : batch.getSpans()) {
+      processSpan(
+          span,
+          serviceName,
+          sourceName,
+          applicationName,
+          cluster,
+          shard,
+          processAnnotations,
+          spanHandler,
+          spanLogsHandler,
+          wfInternalReporter,
+          spanLogsDisabled,
+          preprocessorSupplier,
+          sampler,
+          traceDerivedCustomTagKeys,
+          discardedSpansBySampler,
+          discoveredHeartbeatMetrics);
+    }
+  }
+
+  private static void processSpan(
+      io.jaegertracing.thriftjava.Span span,
+      String serviceName,
+      String sourceName,
+      String applicationName,
+      String cluster,
+      String shard,
+      List processAnnotations,
+      ReportableEntityHandler spanHandler,
+      ReportableEntityHandler spanLogsHandler,
+      @Nullable WavefrontInternalReporter wfInternalReporter,
+      Supplier spanLogsDisabled,
+      Supplier preprocessorSupplier,
+      SpanSampler sampler,
+      Set traceDerivedCustomTagKeys,
+      Counter discardedSpansBySampler,
+      Set, String>> discoveredHeartbeatMetrics) {
+    List annotations = new ArrayList<>(processAnnotations);
+
+    String traceId = new UUID(span.getTraceIdHigh(), span.getTraceIdLow()).toString();
+    String strippedTraceId = StringUtils.stripStart(traceId.replace("-", ""), "0");
+    strippedTraceId = strippedTraceId.length() > 0 ? strippedTraceId : "0";
+    annotations.add(new Annotation("jaegerSpanId", Long.toHexString(span.getSpanId())));
+    annotations.add(new Annotation("jaegerTraceId", strippedTraceId));
+
+    // serviceName is mandatory in Jaeger
+    annotations.add(new Annotation(SERVICE_TAG_KEY, serviceName));
+    long parentSpanId = span.getParentSpanId();
+    if (parentSpanId != 0) {
+      annotations.add(new Annotation("parent", new UUID(0, parentSpanId).toString()));
+    }
+
+    String componentTagValue = NULL_TAG_VAL;
+    boolean isError = false;
+
+    if (span.getTags() != null) {
+      for (Tag tag : span.getTags()) {
+        if (IGNORE_TAGS.contains(tag.getKey())
+            || (tag.vType == TagType.STRING && StringUtils.isBlank(tag.getVStr()))) {
+          continue;
+        }
+
+        Annotation annotation = tagToAnnotation(tag);
+        if (annotation != null) {
+          switch (annotation.getKey()) {
+            case APPLICATION_TAG_KEY:
+              applicationName = annotation.getValue();
+              continue;
+            case CLUSTER_TAG_KEY:
+              cluster = annotation.getValue();
+              continue;
+            case SHARD_TAG_KEY:
+              shard = annotation.getValue();
+              continue;
+            case SOURCE_KEY:
+              // Do not add source to annotation span tag list.
+              sourceName = annotation.getValue();
+              continue;
+            case SERVICE_TAG_KEY:
+              // Do not use service tag from annotations, use field instead
+              continue;
+            case COMPONENT_TAG_KEY:
+              componentTagValue = annotation.getValue();
+              break;
+            case ERROR_TAG_KEY:
+              // only error=true is supported
+              isError = annotation.getValue().equals(ERROR_SPAN_TAG_VAL);
+              break;
+          }
+          annotations.add(annotation);
+        }
+      }
+    }
+
+    // Add all wavefront indexed tags. These are set based on below hierarchy.
+    // Span Level > Process Level > Proxy Level > Default
+    annotations.add(new Annotation(APPLICATION_TAG_KEY, applicationName));
+    annotations.add(new Annotation(CLUSTER_TAG_KEY, cluster));
+    annotations.add(new Annotation(SHARD_TAG_KEY, shard));
+
+    if (span.getReferences() != null) {
+      for (SpanRef reference : span.getReferences()) {
+        switch (reference.refType) {
+          case CHILD_OF:
+            if (reference.getSpanId() != 0 && reference.getSpanId() != parentSpanId) {
+              annotations.add(
+                  new Annotation(
+                      TraceConstants.PARENT_KEY, new UUID(0, reference.getSpanId()).toString()));
+            }
+          case FOLLOWS_FROM:
+            if (reference.getSpanId() != 0) {
+              annotations.add(
+                  new Annotation(
+                      TraceConstants.FOLLOWS_FROM_KEY,
+                      new UUID(0, reference.getSpanId()).toString()));
+            }
+          default:
+        }
+      }
+    }
+
+    if (!spanLogsDisabled.get() && span.getLogs() != null && !span.getLogs().isEmpty()) {
+      annotations.add(new Annotation("_spanLogs", "true"));
+    }
+
+    Span wavefrontSpan =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setName(span.getOperationName())
+            .setSource(sourceName)
+            .setSpanId(new UUID(0, span.getSpanId()).toString())
+            .setTraceId(traceId)
+            .setStartMillis(span.getStartTime() / 1000)
+            .setDuration(span.getDuration() / 1000)
+            .setAnnotations(annotations)
+            .build();
+
+    // Log Jaeger spans as well as Wavefront spans for debugging purposes.
+    if (JAEGER_DATA_LOGGER.isLoggable(Level.FINEST)) {
+      JAEGER_DATA_LOGGER.info("Inbound Jaeger span: " + span.toString());
+      JAEGER_DATA_LOGGER.info("Converted Wavefront span: " + wavefrontSpan.toString());
+    }
+
+    if (preprocessorSupplier != null) {
+      ReportableEntityPreprocessor preprocessor = preprocessorSupplier.get();
+      String[] messageHolder = new String[1];
+      preprocessor.forSpan().transform(wavefrontSpan);
+      if (!preprocessor.forSpan().filter(wavefrontSpan, messageHolder)) {
+        if (messageHolder[0] != null) {
+          spanHandler.reject(wavefrontSpan, messageHolder[0]);
+        } else {
+          spanHandler.block(wavefrontSpan);
+        }
+        return;
+      }
+    }
+    if (sampler.sample(wavefrontSpan, discardedSpansBySampler)) {
+      spanHandler.report(wavefrontSpan);
+      if (span.getLogs() != null
+          && !span.getLogs().isEmpty()
+          && !isFeatureDisabled(spanLogsDisabled, SPANLOGS_DISABLED, null)) {
+        SpanLogs spanLogs =
+            SpanLogs.newBuilder()
+                .setCustomer("default")
+                .setTraceId(wavefrontSpan.getTraceId())
+                .setSpanId(wavefrontSpan.getSpanId())
+                .setLogs(
+                    span.getLogs().stream()
+                        .map(
+                            x -> {
+                              Map fields = new HashMap<>(x.fields.size());
+                              x.fields.forEach(
+                                  t -> {
+                                    switch (t.vType) {
+                                      case STRING:
+                                        fields.put(t.getKey(), t.getVStr());
+                                        break;
+                                      case BOOL:
+                                        fields.put(t.getKey(), String.valueOf(t.isVBool()));
+                                        break;
+                                      case LONG:
+                                        fields.put(t.getKey(), String.valueOf(t.getVLong()));
+                                        break;
+                                      case DOUBLE:
+                                        fields.put(t.getKey(), String.valueOf(t.getVDouble()));
+                                        break;
+                                      case BINARY:
+                                        // ignore
+                                      default:
+                                    }
+                                  });
+                              return SpanLog.newBuilder()
+                                  .setTimestamp(x.timestamp)
+                                  .setFields(fields)
+                                  .build();
+                            })
+                        .collect(Collectors.toList()))
+                .build();
+        SpanUtils.addSpanLine(wavefrontSpan, spanLogs);
+        spanLogsHandler.report(spanLogs);
+      }
+    }
+
+    // report stats irrespective of span sampling.
+    if (wfInternalReporter != null) {
+      // Set post preprocessor rule values and report converted metrics/histograms from the
+      // span
+      List processedAnnotations = wavefrontSpan.getAnnotations();
+      for (Annotation processedAnnotation : processedAnnotations) {
+        switch (processedAnnotation.getKey()) {
+          case APPLICATION_TAG_KEY:
+            applicationName = processedAnnotation.getValue();
+            continue;
+          case SERVICE_TAG_KEY:
+            serviceName = processedAnnotation.getValue();
+            continue;
+          case CLUSTER_TAG_KEY:
+            cluster = processedAnnotation.getValue();
+            continue;
+          case SHARD_TAG_KEY:
+            shard = processedAnnotation.getValue();
+            continue;
+          case COMPONENT_TAG_KEY:
+            componentTagValue = processedAnnotation.getValue();
+            continue;
+          case ERROR_TAG_KEY:
+            isError = processedAnnotation.getValue().equals(ERROR_SPAN_TAG_VAL);
+            continue;
+        }
+      }
+      List> spanTags =
+          processedAnnotations.stream()
+              .map(a -> new Pair<>(a.getKey(), a.getValue()))
+              .collect(Collectors.toList());
+      // TODO: Modify to use new method from wavefront internal reporter.
+      discoveredHeartbeatMetrics.add(
+          reportWavefrontGeneratedData(
+              wfInternalReporter,
+              wavefrontSpan.getName(),
+              applicationName,
+              serviceName,
+              cluster,
+              shard,
+              wavefrontSpan.getSource(),
+              componentTagValue,
+              isError,
+              span.getDuration(),
+              traceDerivedCustomTagKeys,
+              spanTags,
+              true));
+    }
+  }
+
+  @Nullable
+  private static Annotation tagToAnnotation(Tag tag) {
+    switch (tag.vType) {
+      case BOOL:
+        return new Annotation(tag.getKey(), String.valueOf(tag.isVBool()));
+      case LONG:
+        return new Annotation(tag.getKey(), String.valueOf(tag.getVLong()));
+      case DOUBLE:
+        return new Annotation(tag.getKey(), String.valueOf(tag.getVDouble()));
+      case STRING:
+        return new Annotation(tag.getKey(), tag.getVStr());
+      case BINARY:
+      default:
+        return null;
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/tracing/SpanUtils.java b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/SpanUtils.java
new file mode 100644
index 000000000..cd337f247
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/SpanUtils.java
@@ -0,0 +1,198 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.agent.channel.ChannelUtils.formatErrorMessage;
+import static com.wavefront.agent.sampler.SpanSampler.SPAN_SAMPLING_POLICY_TAG;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.protobuf.ByteString;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.data.AnnotationUtils;
+import com.wavefront.ingester.ReportableEntityDecoder;
+import io.netty.channel.ChannelHandlerContext;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
+
+/**
+ * Utility methods for handling Span and SpanLogs.
+ *
+ * @author Shipeng Xie (xshipeng@vmware.com)
+ */
+public final class SpanUtils {
+  private static final Logger logger = Logger.getLogger(SpanUtils.class.getCanonicalName());
+  private static final ObjectMapper JSON_PARSER = new ObjectMapper();
+
+  private SpanUtils() {}
+
+  /**
+   * Preprocess and handle span.
+   *
+   * @param message encoded span data.
+   * @param decoder span decoder.
+   * @param handler span handler.
+   * @param spanReporter span reporter.
+   * @param preprocessorSupplier span preprocessor.
+   * @param ctx channel handler context.
+   * @param samplerFunc span sampler.
+   */
+  public static void preprocessAndHandleSpan(
+      String message,
+      ReportableEntityDecoder decoder,
+      ReportableEntityHandler handler,
+      Consumer spanReporter,
+      @Nullable Supplier preprocessorSupplier,
+      @Nullable ChannelHandlerContext ctx,
+      Function samplerFunc) {
+    ReportableEntityPreprocessor preprocessor =
+        preprocessorSupplier == null ? null : preprocessorSupplier.get();
+    String[] messageHolder = new String[1];
+
+    // transform the line if needed
+    if (preprocessor != null) {
+      message = preprocessor.forPointLine().transform(message);
+
+      if (!preprocessor.forPointLine().filter(message, messageHolder)) {
+        if (messageHolder[0] != null) {
+          handler.reject((Span) null, messageHolder[0]);
+        } else {
+          handler.block(null, message);
+        }
+        return;
+      }
+    }
+    List output = new ArrayList<>(1);
+    try {
+      decoder.decode(message, output, "dummy");
+    } catch (Exception e) {
+      handler.reject(message, formatErrorMessage(message, e, ctx));
+      return;
+    }
+
+    for (Span object : output) {
+      if (preprocessor != null) {
+        preprocessor.forSpan().transform(object);
+        if (!preprocessor.forSpan().filter(object, messageHolder)) {
+          if (messageHolder[0] != null) {
+            handler.reject(object, messageHolder[0]);
+          } else {
+            handler.block(object);
+          }
+          return;
+        }
+      }
+      if (samplerFunc.apply(object)) {
+        spanReporter.accept(object);
+      }
+    }
+  }
+
+  /**
+   * Handle spanLogs.
+   *
+   * @param message encoded spanLogs data.
+   * @param spanLogsDecoder spanLogs decoder.
+   * @param spanDecoder span decoder.
+   * @param handler spanLogs handler.
+   * @param preprocessorSupplier spanLogs preprocessor.
+   * @param ctx channel handler context.
+   * @param samplerFunc span sampler.
+   */
+  public static void handleSpanLogs(
+      String message,
+      ReportableEntityDecoder spanLogsDecoder,
+      ReportableEntityDecoder spanDecoder,
+      ReportableEntityHandler handler,
+      @Nullable Supplier preprocessorSupplier,
+      @Nullable ChannelHandlerContext ctx,
+      Function samplerFunc) {
+    List spanLogsOutput = new ArrayList<>(1);
+    try {
+      spanLogsDecoder.decode(JSON_PARSER.readTree(message), spanLogsOutput, "dummy");
+    } catch (Exception e) {
+      handler.reject(message, formatErrorMessage(message, e, ctx));
+      return;
+    }
+
+    for (SpanLogs spanLogs : spanLogsOutput) {
+      String spanMessage = spanLogs.getSpan();
+      if (spanMessage == null) {
+        // For backwards compatibility, report the span logs if span line data is not
+        // included
+        addSpanLine(null, spanLogs);
+        handler.report(spanLogs);
+      } else {
+        ReportableEntityPreprocessor preprocessor =
+            preprocessorSupplier == null ? null : preprocessorSupplier.get();
+        String[] spanMessageHolder = new String[1];
+
+        // transform the line if needed
+        if (preprocessor != null) {
+          spanMessage = preprocessor.forPointLine().transform(spanMessage);
+
+          if (!preprocessor.forPointLine().filter(message, spanMessageHolder)) {
+            if (spanMessageHolder[0] != null) {
+              handler.reject(spanLogs, spanMessageHolder[0]);
+            } else {
+              handler.block(spanLogs);
+            }
+            return;
+          }
+        }
+        List spanOutput = new ArrayList<>(1);
+        try {
+          spanDecoder.decode(spanMessage, spanOutput, "dummy");
+        } catch (Exception e) {
+          handler.reject(spanLogs, formatErrorMessage(message, e, ctx));
+          return;
+        }
+
+        if (!spanOutput.isEmpty()) {
+          Span span = spanOutput.get(0);
+          if (preprocessor != null) {
+            preprocessor.forSpan().transform(span);
+            if (!preprocessor.forSpan().filter(span, spanMessageHolder)) {
+              if (spanMessageHolder[0] != null) {
+                handler.reject(spanLogs, spanMessageHolder[0]);
+              } else {
+                handler.block(spanLogs);
+              }
+              return;
+            }
+          }
+          if (samplerFunc.apply(span)) {
+            // override spanLine to indicate it is already sampled
+            addSpanLine(span, spanLogs);
+            handler.report(spanLogs);
+          }
+        }
+      }
+    }
+  }
+
+  public static String toStringId(ByteString id) {
+    ByteBuffer byteBuffer = ByteBuffer.wrap(id.toByteArray());
+    long mostSigBits = id.toByteArray().length > 8 ? byteBuffer.getLong() : 0L;
+    long leastSigBits = new BigInteger(1, byteBuffer.array()).longValue();
+    UUID uuid = new UUID(mostSigBits, leastSigBits);
+    return uuid.toString();
+  }
+
+  public static void addSpanLine(Span span, SpanLogs spanLogs) {
+    String policyId = null;
+    if (span != null && span.getAnnotations() != null) {
+      policyId = AnnotationUtils.getValue(span.getAnnotations(), SPAN_SAMPLING_POLICY_TAG);
+    }
+    spanLogs.setSpan(SPAN_SAMPLING_POLICY_TAG + "=" + (policyId == null ? "NONE" : policyId));
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/tracing/TracePortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/TracePortUnificationHandler.java
new file mode 100644
index 000000000..2cec5b612
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/TracePortUnificationHandler.java
@@ -0,0 +1,170 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPANLOGS_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled;
+import static com.wavefront.agent.listeners.tracing.SpanUtils.handleSpanLogs;
+import static com.wavefront.agent.listeners.tracing.SpanUtils.preprocessAndHandleSpan;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.annotations.VisibleForTesting;
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.formatter.DataFormat;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.agent.listeners.AbstractLineDelimitedHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.ingester.ReportableEntityDecoder;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.util.CharsetUtil;
+import java.net.URI;
+import java.util.function.Supplier;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
+
+/**
+ * Process incoming trace-formatted data.
+ *
+ * 
Accepts incoming messages of either String or FullHttpRequest type: single Span in a string,
+ * or multiple points in the HTTP post body, newline-delimited.
+ *
+ * @author vasily@wavefront.com
+ */
+@ChannelHandler.Sharable
+public class TracePortUnificationHandler extends AbstractLineDelimitedHandler {
+
+  protected final ReportableEntityHandler handler;
+  private final ReportableEntityHandler spanLogsHandler;
+  private final ReportableEntityDecoder decoder;
+  private final ReportableEntityDecoder spanLogsDecoder;
+  private final Supplier preprocessorSupplier;
+  private final SpanSampler sampler;
+  private final Supplier traceDisabled;
+  private final Supplier spanLogsDisabled;
+
+  protected final Counter discardedSpans;
+  protected final Counter discardedSpanLogs;
+  private final Counter discardedSpansBySampler;
+  private final Counter discardedSpanLogsBySampler;
+  private final Counter receivedSpansTotal;
+
+  public TracePortUnificationHandler(
+      final String handle,
+      final TokenAuthenticator tokenAuthenticator,
+      final HealthCheckManager healthCheckManager,
+      final ReportableEntityDecoder traceDecoder,
+      final ReportableEntityDecoder spanLogsDecoder,
+      @Nullable final Supplier preprocessor,
+      final ReportableEntityHandlerFactory handlerFactory,
+      final SpanSampler sampler,
+      final Supplier traceDisabled,
+      final Supplier spanLogsDisabled) {
+    this(
+        handle,
+        tokenAuthenticator,
+        healthCheckManager,
+        traceDecoder,
+        spanLogsDecoder,
+        preprocessor,
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)),
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, handle)),
+        sampler,
+        traceDisabled,
+        spanLogsDisabled);
+  }
+
+  @VisibleForTesting
+  public TracePortUnificationHandler(
+      final String handle,
+      final TokenAuthenticator tokenAuthenticator,
+      final HealthCheckManager healthCheckManager,
+      final ReportableEntityDecoder traceDecoder,
+      final ReportableEntityDecoder spanLogsDecoder,
+      @Nullable final Supplier preprocessor,
+      final ReportableEntityHandler handler,
+      final ReportableEntityHandler spanLogsHandler,
+      final SpanSampler sampler,
+      final Supplier traceDisabled,
+      final Supplier spanLogsDisabled) {
+    super(tokenAuthenticator, healthCheckManager, handle);
+    this.decoder = traceDecoder;
+    this.spanLogsDecoder = spanLogsDecoder;
+    this.handler = handler;
+    this.spanLogsHandler = spanLogsHandler;
+    this.preprocessorSupplier = preprocessor;
+    this.sampler = sampler;
+    this.traceDisabled = traceDisabled;
+    this.spanLogsDisabled = spanLogsDisabled;
+    this.discardedSpans = Metrics.newCounter(new MetricName("spans." + handle, "", "discarded"));
+    this.discardedSpanLogs =
+        Metrics.newCounter(new MetricName("spanLogs." + handle, "", "discarded"));
+    this.discardedSpansBySampler =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "sampler.discarded"));
+    this.discardedSpanLogsBySampler =
+        Metrics.newCounter(new MetricName("spanLogs." + handle, "", "sampler.discarded"));
+    this.receivedSpansTotal =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "received.total"));
+  }
+
+  @Nullable
+  @Override
+  protected DataFormat getFormat(FullHttpRequest httpRequest) {
+    return DataFormat.parse(
+        URLEncodedUtils.parse(URI.create(httpRequest.uri()), CharsetUtil.UTF_8).stream()
+            .filter(x -> x.getName().equals("format") || x.getName().equals("f"))
+            .map(NameValuePair::getValue)
+            .findFirst()
+            .orElse(null));
+  }
+
+  @Override
+  protected void processLine(
+      final ChannelHandlerContext ctx, @Nonnull String message, @Nullable DataFormat format) {
+    if (format == DataFormat.SPAN_LOG || (message.startsWith("{") && message.endsWith("}"))) {
+      if (isFeatureDisabled(spanLogsDisabled, SPANLOGS_DISABLED, discardedSpanLogs)) return;
+      handleSpanLogs(
+          message,
+          spanLogsDecoder,
+          decoder,
+          spanLogsHandler,
+          preprocessorSupplier,
+          ctx,
+          span -> sampler.sample(span, discardedSpanLogsBySampler));
+      return;
+    }
+
+    // Payload is a span.
+    receivedSpansTotal.inc();
+    if (isFeatureDisabled(traceDisabled, SPAN_DISABLED, discardedSpans)) return;
+    preprocessAndHandleSpan(
+        message,
+        decoder,
+        handler,
+        this::report,
+        preprocessorSupplier,
+        ctx,
+        span -> sampler.sample(span, discardedSpansBySampler));
+  }
+
+  /**
+   * Report span and derived metrics if needed.
+   *
+   * @param object span.
+   */
+  protected void report(Span object) {
+    handler.report(object);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java
new file mode 100644
index 000000000..535c82664
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandler.java
@@ -0,0 +1,502 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.agent.channel.ChannelUtils.errorMessageWithRootCause;
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPANLOGS_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED;
+import static com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.DEBUG_SPAN_TAG_KEY;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.DEBUG_SPAN_TAG_VAL;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.ERROR_SPAN_TAG_KEY;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.ERROR_SPAN_TAG_VAL;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportHeartbeats;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.reportWavefrontGeneratedData;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.DEBUG_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.ERROR_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.NULL_TAG_VAL;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SOURCE_KEY;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.wavefront.agent.auth.TokenAuthenticatorBuilder;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.agent.listeners.AbstractHttpOnlyHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.common.NamedThreadFactory;
+import com.wavefront.common.TraceConstants;
+import com.wavefront.common.Utils;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
+import zipkin2.SpanBytesDecoderDetector;
+import zipkin2.codec.BytesDecoder;
+
+/**
+ * Handler that processes Zipkin trace data over HTTP and converts them to Wavefront format.
+ *
+ * @author Anil Kodali (akodali@vmware.com)
+ */
+@ChannelHandler.Sharable
+public class ZipkinPortUnificationHandler extends AbstractHttpOnlyHandler
+    implements Runnable, Closeable {
+  private static final Logger logger =
+      Logger.getLogger(ZipkinPortUnificationHandler.class.getCanonicalName());
+
+  private final ReportableEntityHandler spanHandler;
+  private final ReportableEntityHandler spanLogsHandler;
+  @Nullable private final WavefrontSender wfSender;
+  @Nullable private final WavefrontInternalReporter wfInternalReporter;
+  private final Supplier traceDisabled;
+  private final Supplier spanLogsDisabled;
+  private final Supplier preprocessorSupplier;
+  private final SpanSampler sampler;
+  private final Counter discardedBatches;
+  private final Counter processedBatches;
+  private final Counter failedBatches;
+  private final Counter discardedSpansBySampler;
+  private final Counter receivedSpansTotal;
+  private final Counter discardedTraces;
+  private final Set, String>> discoveredHeartbeatMetrics;
+  private final ScheduledExecutorService scheduledExecutorService;
+
+  private static final Set ZIPKIN_VALID_PATHS =
+      ImmutableSet.of("/api/v1/spans/", "/api/v2/spans/");
+  private static final String ZIPKIN_VALID_HTTP_METHOD = "POST";
+  private static final String ZIPKIN_COMPONENT = "zipkin";
+  private static final String DEFAULT_SOURCE = "zipkin";
+  private static final String DEFAULT_SERVICE = "defaultService";
+  private static final String DEFAULT_SPAN_NAME = "defaultOperation";
+  private static final String SPAN_TAG_ERROR = "error";
+  private final String proxyLevelApplicationName;
+  private final Set traceDerivedCustomTagKeys;
+
+  private static final Logger ZIPKIN_DATA_LOGGER = Logger.getLogger("ZipkinDataLogger");
+
+  public ZipkinPortUnificationHandler(
+      String handle,
+      final HealthCheckManager healthCheckManager,
+      ReportableEntityHandlerFactory handlerFactory,
+      @Nullable WavefrontSender wfSender,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      @Nullable Supplier preprocessor,
+      SpanSampler sampler,
+      @Nullable String traceZipkinApplicationName,
+      Set traceDerivedCustomTagKeys) {
+    this(
+        handle,
+        healthCheckManager,
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE, handle)),
+        handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, handle)),
+        wfSender,
+        traceDisabled,
+        spanLogsDisabled,
+        preprocessor,
+        sampler,
+        traceZipkinApplicationName,
+        traceDerivedCustomTagKeys);
+  }
+
+  @VisibleForTesting
+  ZipkinPortUnificationHandler(
+      final String handle,
+      final HealthCheckManager healthCheckManager,
+      ReportableEntityHandler spanHandler,
+      ReportableEntityHandler spanLogsHandler,
+      @Nullable WavefrontSender wfSender,
+      Supplier traceDisabled,
+      Supplier spanLogsDisabled,
+      @Nullable Supplier preprocessor,
+      SpanSampler sampler,
+      @Nullable String traceZipkinApplicationName,
+      Set traceDerivedCustomTagKeys) {
+    super(TokenAuthenticatorBuilder.create().build(), healthCheckManager, handle);
+    this.spanHandler = spanHandler;
+    this.spanLogsHandler = spanLogsHandler;
+    this.wfSender = wfSender;
+    this.traceDisabled = traceDisabled;
+    this.spanLogsDisabled = spanLogsDisabled;
+    this.preprocessorSupplier = preprocessor;
+    this.sampler = sampler;
+    this.proxyLevelApplicationName =
+        StringUtils.isBlank(traceZipkinApplicationName)
+            ? "Zipkin"
+            : traceZipkinApplicationName.trim();
+    this.traceDerivedCustomTagKeys = traceDerivedCustomTagKeys;
+    this.discardedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "discarded"));
+    this.processedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "processed"));
+    this.failedBatches =
+        Metrics.newCounter(new MetricName("spans." + handle + ".batches", "", "failed"));
+    this.discardedSpansBySampler =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "sampler.discarded"));
+    this.receivedSpansTotal =
+        Metrics.newCounter(new MetricName("spans." + handle, "", "received.total"));
+    this.discardedTraces = Metrics.newCounter(new MetricName("spans." + handle, "", "discarded"));
+    this.discoveredHeartbeatMetrics = Sets.newConcurrentHashSet();
+    this.scheduledExecutorService =
+        Executors.newScheduledThreadPool(1, new NamedThreadFactory("zipkin-heart-beater"));
+    scheduledExecutorService.scheduleAtFixedRate(this, 1, 1, TimeUnit.MINUTES);
+
+    if (wfSender != null) {
+      wfInternalReporter =
+          new WavefrontInternalReporter.Builder()
+              .prefixedWith("tracing.derived")
+              .withSource(DEFAULT_SOURCE)
+              .reportMinuteDistribution()
+              .build(wfSender);
+      // Start the reporter
+      wfInternalReporter.start(1, TimeUnit.MINUTES);
+    } else {
+      wfInternalReporter = null;
+    }
+  }
+
+  @Override
+  protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request)
+      throws URISyntaxException {
+    URI uri = new URI(request.uri());
+    String path = uri.getPath().endsWith("/") ? uri.getPath() : uri.getPath() + "/";
+
+    // Validate Uri Path and HTTP method of incoming Zipkin spans.
+    if (!ZIPKIN_VALID_PATHS.contains(path)) {
+      writeHttpResponse(ctx, HttpResponseStatus.BAD_REQUEST, "Unsupported URL path.", request);
+      logWarning("Requested URI path '" + path + "' is not supported.", null, ctx);
+      return;
+    }
+    if (!request.method().toString().equalsIgnoreCase(ZIPKIN_VALID_HTTP_METHOD)) {
+      writeHttpResponse(ctx, HttpResponseStatus.BAD_REQUEST, "Unsupported Http method.", request);
+      logWarning(
+          "Requested http method '" + request.method().toString() + "' is not supported.",
+          null,
+          ctx);
+      return;
+    }
+
+    HttpResponseStatus status;
+    StringBuilder output = new StringBuilder();
+
+    try {
+      byte[] bytesArray = new byte[request.content().nioBuffer().remaining()];
+      request.content().nioBuffer().get(bytesArray, 0, bytesArray.length);
+      BytesDecoder decoder =
+          SpanBytesDecoderDetector.decoderForListMessage(bytesArray);
+      List zipkinSpanSink = new ArrayList<>();
+      decoder.decodeList(bytesArray, zipkinSpanSink);
+
+      // Handle case when tracing is disabled, ignore reported spans.
+      if (isFeatureDisabled(traceDisabled, SPAN_DISABLED, discardedBatches, output)) {
+        status = HttpResponseStatus.ACCEPTED;
+        writeHttpResponse(ctx, status, output, request);
+        discardedTraces.inc(zipkinSpanSink.size());
+        receivedSpansTotal.inc(zipkinSpanSink.size());
+        processedBatches.inc();
+        return;
+      }
+      receivedSpansTotal.inc(zipkinSpanSink.size());
+      processZipkinSpans(zipkinSpanSink);
+      status = HttpResponseStatus.ACCEPTED;
+      processedBatches.inc();
+    } catch (Exception e) {
+      failedBatches.inc();
+      output.append(errorMessageWithRootCause(e));
+      status = HttpResponseStatus.BAD_REQUEST;
+      logger.log(Level.WARNING, "Zipkin batch processing failed", Throwables.getRootCause(e));
+    }
+    writeHttpResponse(ctx, status, output, request);
+  }
+
+  private void processZipkinSpans(List zipkinSpans) {
+    for (zipkin2.Span zipkinSpan : zipkinSpans) {
+      processZipkinSpan(zipkinSpan);
+    }
+  }
+
+  private void processZipkinSpan(zipkin2.Span zipkinSpan) {
+    if (ZIPKIN_DATA_LOGGER.isLoggable(Level.FINEST)) {
+      ZIPKIN_DATA_LOGGER.info("Inbound Zipkin span: " + zipkinSpan.toString());
+    }
+    // Add application tags, span references, span kind and http uri, responses etc.
+    List annotations = new ArrayList<>();
+
+    // Add original Zipkin trace and span ids as tags to make finding them easier
+    annotations.add(new Annotation("zipkinSpanId", zipkinSpan.id()));
+    annotations.add(new Annotation("zipkinTraceId", zipkinSpan.traceId()));
+
+    // Set Span's References.
+    if (zipkinSpan.parentId() != null) {
+      annotations.add(
+          new Annotation(
+              TraceConstants.PARENT_KEY, Utils.convertToUuidString(zipkinSpan.parentId())));
+    }
+
+    // Set Span Kind.
+    if (zipkinSpan.kind() != null) {
+      String kind = zipkinSpan.kind().toString().toLowerCase();
+      annotations.add(new Annotation("span.kind", kind));
+      if (zipkinSpan.annotations() != null && !zipkinSpan.annotations().isEmpty()) {
+        annotations.add(new Annotation("_spanSecondaryId", kind));
+      }
+    }
+
+    // Set Span's service name.
+    String serviceName =
+        zipkinSpan.localServiceName() == null ? DEFAULT_SERVICE : zipkinSpan.localServiceName();
+    annotations.add(new Annotation(SERVICE_TAG_KEY, serviceName));
+
+    String applicationName = this.proxyLevelApplicationName;
+    String cluster = NULL_TAG_VAL;
+    String shard = NULL_TAG_VAL;
+    String componentTagValue = NULL_TAG_VAL;
+    boolean isError = false;
+    boolean isDebugSpanTag = false;
+
+    // Set all other Span Tags.
+    Set ignoreKeys = new HashSet<>(ImmutableSet.of(SOURCE_KEY));
+    if (zipkinSpan.tags() != null && zipkinSpan.tags().size() > 0) {
+      for (Map.Entry tag : zipkinSpan.tags().entrySet()) {
+        if (!ignoreKeys.contains(tag.getKey().toLowerCase())
+            && !StringUtils.isBlank(tag.getValue())) {
+          Annotation annotation = new Annotation(tag.getKey(), tag.getValue());
+          switch (annotation.getKey()) {
+            case APPLICATION_TAG_KEY:
+              applicationName = annotation.getValue();
+              continue;
+            case CLUSTER_TAG_KEY:
+              cluster = annotation.getValue();
+              continue;
+            case SHARD_TAG_KEY:
+              shard = annotation.getValue();
+              continue;
+            case COMPONENT_TAG_KEY:
+              componentTagValue = annotation.getValue();
+              break;
+            case ERROR_SPAN_TAG_KEY:
+              isError = true;
+              // Ignore the original error value
+              annotation.setValue(ERROR_SPAN_TAG_VAL);
+              break;
+            case DEBUG_TAG_KEY:
+              isDebugSpanTag = annotation.getValue().equals(DEBUG_SPAN_TAG_VAL);
+              break;
+          }
+          annotations.add(annotation);
+        }
+      }
+    }
+
+    // Add all wavefront indexed tags. These are set based on below hierarchy.
+    // Span Level > Proxy Level > Default
+    annotations.add(new Annotation(APPLICATION_TAG_KEY, applicationName));
+    annotations.add(new Annotation(CLUSTER_TAG_KEY, cluster));
+    annotations.add(new Annotation(SHARD_TAG_KEY, shard));
+
+    // Add Sampling related annotations.
+    // Add a debug span tag as needed to enable sampling of this span with intelligent sampling.
+    boolean isDebug = zipkinSpan.debug() != null ? zipkinSpan.debug() : false;
+    if (!isDebugSpanTag && isDebug) {
+      annotations.add(new Annotation(DEBUG_SPAN_TAG_KEY, DEBUG_SPAN_TAG_VAL));
+    }
+
+    // Add additional annotations.
+    if (zipkinSpan.localEndpoint() != null && zipkinSpan.localEndpoint().ipv4() != null) {
+      annotations.add(new Annotation("ipv4", zipkinSpan.localEndpoint().ipv4()));
+    }
+
+    if (!spanLogsDisabled.get()
+        && zipkinSpan.annotations() != null
+        && !zipkinSpan.annotations().isEmpty()) {
+      annotations.add(new Annotation("_spanLogs", "true"));
+    }
+
+    /* Add source of the span following the below:
+     *    1. If "source" is provided by span tags , use it else
+     *    2. Default "source" to "zipkin".
+     */
+    String sourceName = DEFAULT_SOURCE;
+    if (zipkinSpan.tags() != null && zipkinSpan.tags().size() > 0) {
+      if (zipkinSpan.tags().get(SOURCE_KEY) != null) {
+        sourceName = zipkinSpan.tags().get(SOURCE_KEY);
+      }
+    }
+    // Set spanName.
+    String spanName = zipkinSpan.name() == null ? DEFAULT_SPAN_NAME : zipkinSpan.name();
+
+    String spanId = Utils.convertToUuidString(zipkinSpan.id());
+    String traceId = Utils.convertToUuidString(zipkinSpan.traceId());
+    // Build wavefront span
+    Span wavefrontSpan =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setName(spanName)
+            .setSource(sourceName)
+            .setSpanId(spanId)
+            .setTraceId(traceId)
+            .setStartMillis(zipkinSpan.timestampAsLong() / 1000)
+            .setDuration(zipkinSpan.durationAsLong() / 1000)
+            .setAnnotations(annotations)
+            .build();
+
+    if (zipkinSpan.tags().containsKey(SPAN_TAG_ERROR)) {
+      if (ZIPKIN_DATA_LOGGER.isLoggable(Level.FINER)) {
+        ZIPKIN_DATA_LOGGER.info(
+            "Span id :: "
+                + spanId
+                + " with trace id :: "
+                + traceId
+                + " , includes error tag :: "
+                + zipkinSpan.tags().get(SPAN_TAG_ERROR));
+      }
+    }
+    // Log Zipkin spans as well as Wavefront spans for debugging purposes.
+    if (ZIPKIN_DATA_LOGGER.isLoggable(Level.FINEST)) {
+      ZIPKIN_DATA_LOGGER.info("Converted Wavefront span: " + wavefrontSpan.toString());
+    }
+
+    if (preprocessorSupplier != null) {
+      ReportableEntityPreprocessor preprocessor = preprocessorSupplier.get();
+      String[] messageHolder = new String[1];
+      preprocessor.forSpan().transform(wavefrontSpan);
+      if (!preprocessor.forSpan().filter(wavefrontSpan, messageHolder)) {
+        if (messageHolder[0] != null) {
+          spanHandler.reject(wavefrontSpan, messageHolder[0]);
+        } else {
+          spanHandler.block(wavefrontSpan);
+        }
+        return;
+      }
+    }
+
+    if (sampler.sample(wavefrontSpan, discardedSpansBySampler)) {
+      spanHandler.report(wavefrontSpan);
+
+      if (zipkinSpan.annotations() != null
+          && !zipkinSpan.annotations().isEmpty()
+          && !isFeatureDisabled(spanLogsDisabled, SPANLOGS_DISABLED, null)) {
+        SpanLogs spanLogs =
+            SpanLogs.newBuilder()
+                .setCustomer("default")
+                .setTraceId(wavefrontSpan.getTraceId())
+                .setSpanId(wavefrontSpan.getSpanId())
+                .setSpanSecondaryId(
+                    zipkinSpan.kind() != null ? zipkinSpan.kind().toString().toLowerCase() : null)
+                .setLogs(
+                    zipkinSpan.annotations().stream()
+                        .map(
+                            x ->
+                                SpanLog.newBuilder()
+                                    .setTimestamp(x.timestamp())
+                                    .setFields(ImmutableMap.of("annotation", x.value()))
+                                    .build())
+                        .collect(Collectors.toList()))
+                .build();
+        SpanUtils.addSpanLine(wavefrontSpan, spanLogs);
+        spanLogsHandler.report(spanLogs);
+      }
+    }
+    // report stats irrespective of span sampling.
+    if (wfInternalReporter != null) {
+      // Set post preprocessor rule values and report converted metrics/histograms from the
+      // span
+      List processedAnnotations = wavefrontSpan.getAnnotations();
+      for (Annotation processedAnnotation : processedAnnotations) {
+        switch (processedAnnotation.getKey()) {
+          case APPLICATION_TAG_KEY:
+            applicationName = processedAnnotation.getValue();
+            continue;
+          case SERVICE_TAG_KEY:
+            serviceName = processedAnnotation.getValue();
+            continue;
+          case CLUSTER_TAG_KEY:
+            cluster = processedAnnotation.getValue();
+            continue;
+          case SHARD_TAG_KEY:
+            shard = processedAnnotation.getValue();
+            continue;
+          case COMPONENT_TAG_KEY:
+            componentTagValue = processedAnnotation.getValue();
+            continue;
+          case ERROR_TAG_KEY:
+            isError = true;
+            continue;
+        }
+      }
+      List> spanTags =
+          processedAnnotations.stream()
+              .map(a -> new Pair<>(a.getKey(), a.getValue()))
+              .collect(Collectors.toList());
+      discoveredHeartbeatMetrics.add(
+          reportWavefrontGeneratedData(
+              wfInternalReporter,
+              wavefrontSpan.getName(),
+              applicationName,
+              serviceName,
+              cluster,
+              shard,
+              wavefrontSpan.getSource(),
+              componentTagValue,
+              isError,
+              zipkinSpan.durationAsLong(),
+              traceDerivedCustomTagKeys,
+              spanTags,
+              true));
+    }
+  }
+
+  @Override
+  public void run() {
+    try {
+      reportHeartbeats(wfSender, discoveredHeartbeatMetrics, ZIPKIN_COMPONENT);
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Cannot report heartbeat metric to wavefront");
+    }
+  }
+
+  @Override
+  public void close() {
+    scheduledExecutorService.shutdownNow();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/ChangeableGauge.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/ChangeableGauge.java
index d2edd003d..9b3741fa3 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/ChangeableGauge.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/ChangeableGauge.java
@@ -2,9 +2,7 @@
 
 import com.yammer.metrics.core.Gauge;
 
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public class ChangeableGauge extends Gauge {
   private T value;
 
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/EvictingMetricsRegistry.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/EvictingMetricsRegistry.java
index 691696a02..7b6e09084 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/EvictingMetricsRegistry.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/EvictingMetricsRegistry.java
@@ -1,93 +1,140 @@
 package com.wavefront.agent.logsharvesting;
 
-import com.google.common.collect.Sets;
-
 import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.CacheWriter;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.github.benmanes.caffeine.cache.RemovalCause;
+import com.github.benmanes.caffeine.cache.Ticker;
+import com.google.common.collect.Sets;
 import com.wavefront.agent.config.MetricMatcher;
 import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.Metric;
-import com.yammer.metrics.core.MetricName;
-import com.yammer.metrics.core.MetricsRegistry;
 import com.yammer.metrics.core.DeltaCounter;
 import com.yammer.metrics.core.Gauge;
 import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.Metric;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricsRegistry;
 import com.yammer.metrics.core.WavefrontHistogram;
-
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.function.Supplier;
-import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 /**
  * Wrapper for a Yammer {@link com.yammer.metrics.core.MetricsRegistry}, but has extra features
  * regarding automatic removal of metrics.
  *
- * With the introduction of Delta Counter for Yammer metrics, this class now treats Counters as
- * Delta Counters. So anybody using this {@link #getCounter(MetricName, MetricMatcher)} method
- * will get an instance of Delta counter.
+ * 
With the introduction of Delta Counter for Yammer metrics, this class now treats Counters as
+ * Delta Counters. So anybody using this {@link #getCounter(MetricName, MetricMatcher)} method will
+ * get an instance of Delta counter.
  *
  * @author Mori Bellamy (mori@wavefront.com)
  */
 public class EvictingMetricsRegistry {
-  protected static final Logger logger = Logger.getLogger(EvictingMetricsRegistry.class.getCanonicalName());
   private final MetricsRegistry metricsRegistry;
   private final Cache metricCache;
   private final LoadingCache> metricNamesForMetricMatchers;
   private final boolean wavefrontHistograms;
+  private final boolean useDeltaCounters;
   private final Supplier nowMillis;
 
-  EvictingMetricsRegistry(long expiryMillis, boolean wavefrontHistograms, Supplier nowMillis) {
-    this.metricsRegistry = new MetricsRegistry();
+  EvictingMetricsRegistry(
+      MetricsRegistry metricRegistry,
+      long expiryMillis,
+      boolean wavefrontHistograms,
+      boolean useDeltaCounters,
+      Supplier nowMillis,
+      Ticker ticker) {
+    this.metricsRegistry = metricRegistry;
     this.nowMillis = nowMillis;
     this.wavefrontHistograms = wavefrontHistograms;
-    this.metricCache = Caffeine.newBuilder()
-        .expireAfterAccess(expiryMillis, TimeUnit.MILLISECONDS)
-        .removalListener((metricName, metric, reason) -> {
-          if (metricName == null || metric == null) {
-            logger.severe("Application error, pulled null key or value from metricCache.");
-            return;
-          }
-          metricsRegistry.removeMetric(metricName);
-        }).build();
-    this.metricNamesForMetricMatchers = Caffeine.>newBuilder()
-        .build((metricMatcher) -> Sets.newHashSet());
+    this.useDeltaCounters = useDeltaCounters;
+    this.metricCache =
+        Caffeine.newBuilder()
+            .expireAfterAccess(expiryMillis, TimeUnit.MILLISECONDS)
+            .ticker(ticker)
+            .writer(
+                new CacheWriter() {
+                  @Override
+                  public void write(@Nonnull MetricName key, @Nonnull Metric value) {}
+
+                  @Override
+                  public void delete(
+                      @Nonnull MetricName key,
+                      @Nullable Metric value,
+                      @Nonnull RemovalCause cause) {
+                    if ((cause == RemovalCause.EXPIRED || cause == RemovalCause.EXPLICIT)
+                        && metricsRegistry.allMetrics().get(key) == value) {
+                      metricsRegistry.removeMetric(key);
+                    }
+                  }
+                })
+            .build();
+    this.metricNamesForMetricMatchers =
+        Caffeine.newBuilder().build((metricMatcher) -> Sets.newHashSet());
   }
 
   public Counter getCounter(MetricName metricName, MetricMatcher metricMatcher) {
-    // use delta counters instead of regular counters. It helps with load balancers present in
-    // front of proxy (PUB-125)
-    MetricName newMetricName = DeltaCounter.getDeltaCounterMetricName(metricName);
-    metricNamesForMetricMatchers.get(metricMatcher).add(newMetricName);
-    return (Counter) metricCache.get(newMetricName, key -> DeltaCounter.get(metricsRegistry,
-            newMetricName));
+    if (useDeltaCounters) {
+      // use delta counters instead of regular counters. It helps with load balancers present
+      // in
+      // front of proxy (PUB-125)
+      MetricName newMetricName = DeltaCounter.getDeltaCounterMetricName(metricName);
+      return put(
+          newMetricName, metricMatcher, key -> DeltaCounter.get(metricsRegistry, newMetricName));
+    } else {
+      return put(metricName, metricMatcher, metricsRegistry::newCounter);
+    }
   }
 
   public Gauge getGauge(MetricName metricName, MetricMatcher metricMatcher) {
-    metricNamesForMetricMatchers.get(metricMatcher).add(metricName);
-    return (Gauge) metricCache.get(
-        metricName, (key) -> metricsRegistry.newGauge(key, new ChangeableGauge()));
+    return put(
+        metricName,
+        metricMatcher,
+        key -> metricsRegistry.newGauge(key, new ChangeableGauge()));
   }
 
   public Histogram getHistogram(MetricName metricName, MetricMatcher metricMatcher) {
-    metricNamesForMetricMatchers.get(metricMatcher).add(metricName);
-    return (Histogram) metricCache.get(
+    return put(
         metricName,
-        (key) -> wavefrontHistograms
-            ? WavefrontHistogram.get(metricsRegistry, key, this.nowMillis)
-            : metricsRegistry.newHistogram(metricName, false));
+        metricMatcher,
+        key ->
+            wavefrontHistograms
+                ? WavefrontHistogram.get(metricsRegistry, key, this.nowMillis)
+                : metricsRegistry.newHistogram(metricName, false));
   }
 
   public synchronized void evict(MetricMatcher evicted) {
-    for (MetricName toRemove : metricNamesForMetricMatchers.get(evicted)) {
+    for (MetricName toRemove : Objects.requireNonNull(metricNamesForMetricMatchers.get(evicted))) {
       metricCache.invalidate(toRemove);
     }
     metricNamesForMetricMatchers.invalidate(evicted);
   }
 
-  public MetricsRegistry metricsRegistry() {
-    return metricsRegistry;
+  public void cleanUp() {
+    metricCache.cleanUp();
   }
 
+  @SuppressWarnings("unchecked")
+  private  M put(
+      MetricName metricName, MetricMatcher metricMatcher, Function getter) {
+    @Nullable Metric cached = metricCache.getIfPresent(metricName);
+    Objects.requireNonNull(metricNamesForMetricMatchers.get(metricMatcher)).add(metricName);
+    if (cached != null && cached == metricsRegistry.allMetrics().get(metricName)) {
+      return (M) cached;
+    }
+    return (M)
+        metricCache
+            .asMap()
+            .compute(
+                metricName,
+                (name, existing) -> {
+                  @Nullable Metric expected = metricsRegistry.allMetrics().get(name);
+                  return expected == null ? getter.apply(name) : expected;
+                });
+  }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/FilebeatIngester.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/FilebeatIngester.java
index 71faaac3d..09e5ad2bf 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/FilebeatIngester.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/FilebeatIngester.java
@@ -4,25 +4,21 @@
 import com.yammer.metrics.core.Counter;
 import com.yammer.metrics.core.Histogram;
 import com.yammer.metrics.core.MetricName;
-
-import org.logstash.beats.IMessageListener;
-import org.logstash.beats.Message;
-
+import io.netty.channel.ChannelHandlerContext;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.logstash.beats.IMessageListener;
+import org.logstash.beats.Message;
 
-import io.netty.channel.ChannelHandlerContext;
-
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public class FilebeatIngester implements IMessageListener {
   protected static final Logger logger = Logger.getLogger(LogsIngester.class.getCanonicalName());
-  private LogsIngester logsIngester;
-  private Counter received, malformed;
-  private Histogram drift;
-  private Supplier currentMillis;
+  private final LogsIngester logsIngester;
+  private final Counter received;
+  private final Counter malformed;
+  private final Histogram drift;
+  private final Supplier currentMillis;
 
   public FilebeatIngester(LogsIngester logsIngester, Supplier currentMillis) {
     this.logsIngester = logsIngester;
@@ -39,7 +35,8 @@ public void onNewMessage(ChannelHandlerContext ctx, Message message) {
     try {
       filebeatMessage = new FilebeatMessage(message);
     } catch (MalformedMessageException exn) {
-      logger.severe("Malformed message received from filebeat, dropping.");
+      logger.severe(
+          "Malformed message received from filebeat, dropping (" + exn.getMessage() + ")");
       malformed.inc();
       return;
     }
@@ -49,7 +46,6 @@ public void onNewMessage(ChannelHandlerContext ctx, Message message) {
     }
 
     logsIngester.ingestLog(filebeatMessage);
-
   }
 
   @Override
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/FilebeatMessage.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/FilebeatMessage.java
index f3e2340b7..6d0d6dedb 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/FilebeatMessage.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/FilebeatMessage.java
@@ -1,15 +1,14 @@
 package com.wavefront.agent.logsharvesting;
 
-import org.logstash.beats.Message;
-
+import com.google.common.collect.ImmutableMap;
 import java.time.Instant;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 import java.time.temporal.TemporalAccessor;
 import java.util.Date;
 import java.util.Map;
-
 import javax.annotation.Nullable;
+import org.logstash.beats.Message;
 
 /**
  * Abstraction for {@link org.logstash.beats.Message}
@@ -18,17 +17,19 @@
  */
 public class FilebeatMessage implements LogsMessage {
   private final Message wrapped;
-  private final Map messageData;
-  private final Map beatData;
+  private final Map messageData;
+  private final Map beatData;
   private final String logLine;
   private Long timestampMillis = null;
 
+  @SuppressWarnings("unchecked")
   public FilebeatMessage(Message wrapped) throws MalformedMessageException {
     this.wrapped = wrapped;
     this.messageData = this.wrapped.getData();
-    if (!this.messageData.containsKey("beat")) throw new MalformedMessageException("No beat metadata.");
-    this.beatData = (Map) this.messageData.get("beat");
-    if (!this.messageData.containsKey("message")) throw new MalformedMessageException("No log line in message.");
+    this.beatData = (Map) this.messageData.getOrDefault("beat", ImmutableMap.of());
+    if (!this.messageData.containsKey("message")) {
+      throw new MalformedMessageException("No log line in message.");
+    }
     this.logLine = (String) this.messageData.get("message");
     if (getTimestampMillis() == null) throw new MalformedMessageException("No timestamp metadata.");
   }
@@ -55,9 +56,23 @@ public String getLogLine() {
 
   @Override
   public String hostOrDefault(String fallbackHost) {
+    // < 7.0: return beat.hostname
     if (this.beatData.containsKey("hostname")) {
       return (String) this.beatData.get("hostname");
     }
+    // 7.0+: return host.name or agent.hostname
+    if (this.messageData.containsKey("host")) {
+      Map hostData = (Map) this.messageData.get("host");
+      if (hostData.containsKey("name")) {
+        return (String) hostData.get("name");
+      }
+    }
+    if (this.messageData.containsKey("agent")) {
+      Map agentData = (Map) this.messageData.get("agent");
+      if (agentData.containsKey("hostname")) {
+        return (String) agentData.get("hostname");
+      }
+    }
     return fallbackHost;
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/FlushProcessor.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/FlushProcessor.java
index 7da3787d1..5f227ac0d 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/FlushProcessor.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/FlushProcessor.java
@@ -1,67 +1,75 @@
 package com.wavefront.agent.logsharvesting;
 
-import com.beust.jcommander.internal.Lists;
+import com.tdunning.math.stats.AVLTreeDigest;
+import com.tdunning.math.stats.Centroid;
+import com.tdunning.math.stats.TDigest;
 import com.wavefront.common.MetricsToTimeseries;
+import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
 import com.yammer.metrics.core.DeltaCounter;
 import com.yammer.metrics.core.Gauge;
 import com.yammer.metrics.core.Histogram;
-import com.yammer.metrics.core.WavefrontHistogram;
-import com.yammer.metrics.core.MetricProcessor;
 import com.yammer.metrics.core.Metered;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricProcessor;
+import com.yammer.metrics.core.Sampling;
+import com.yammer.metrics.core.Summarizable;
 import com.yammer.metrics.core.Timer;
-
+import com.yammer.metrics.core.WavefrontHistogram;
+import com.yammer.metrics.stats.Snapshot;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
-
 import wavefront.report.HistogramType;
 
 /**
- * Wrapper for {@link com.yammer.metrics.core.MetricProcessor}. It provides additional support
- * for Delta Counters and WavefrontHistogram.
+ * Wrapper for {@link com.yammer.metrics.core.MetricProcessor}. It provides additional support for
+ * Delta Counters and WavefrontHistogram.
  *
  * @author Mori Bellamy (mori@wavefront.com)
  */
-class FlushProcessor implements MetricProcessor {
+public class FlushProcessor implements MetricProcessor {
 
-  private final Counter sentCounter;
+  private final Counter sentCounter =
+      Metrics.newCounter(new MetricName("logsharvesting", "", "sent"));
+  private final Counter histogramCounter =
+      Metrics.newCounter(new MetricName("logsharvesting", "", "histograms-sent"));
   private final Supplier currentMillis;
   private final boolean useWavefrontHistograms;
   private final boolean reportEmptyHistogramStats;
 
-  FlushProcessor(Counter sentCounter, Supplier currentMillis) {
-    this(sentCounter, currentMillis, false, true);
-  }
-
   /**
    * Create new FlushProcessor instance
    *
-   * @param sentCounter               counter metric to increment for every sent data point
-   * @param currentMillis             {@link Supplier} of time (in milliseconds)
-   * @param useWavefrontHistograms    export data in {@link com.yammer.metrics.core.WavefrontHistogram} format
-   * @param reportEmptyHistogramStats enable legacy {@link com.yammer.metrics.core.Histogram} behavior and send zero
-   *                                  values for every stat
+   * @param currentMillis {@link Supplier} of time (in milliseconds)
+   * @param useWavefrontHistograms export data in {@link com.yammer.metrics.core.WavefrontHistogram}
+   *     format
+   * @param reportEmptyHistogramStats enable legacy {@link com.yammer.metrics.core.Histogram}
+   *     behavior and send zero values for every stat
    */
-  FlushProcessor(Counter sentCounter, Supplier currentMillis, boolean useWavefrontHistograms,
-                 boolean reportEmptyHistogramStats) {
-    this.sentCounter = sentCounter;
+  FlushProcessor(
+      Supplier currentMillis,
+      boolean useWavefrontHistograms,
+      boolean reportEmptyHistogramStats) {
     this.currentMillis = currentMillis;
     this.useWavefrontHistograms = useWavefrontHistograms;
     this.reportEmptyHistogramStats = reportEmptyHistogramStats;
   }
 
   @Override
-  public void processMeter(MetricName name, Metered meter, FlushProcessorContext context) throws Exception {
+  public void processMeter(MetricName name, Metered meter, FlushProcessorContext context) {
     throw new UnsupportedOperationException();
   }
 
   @Override
-  public void processCounter(MetricName name, Counter counter, FlushProcessorContext context) throws Exception {
+  public void processCounter(MetricName name, Counter counter, FlushProcessorContext context) {
     long count;
     // handle delta counter
     if (counter instanceof DeltaCounter) {
       count = DeltaCounter.processDeltaCounter((DeltaCounter) counter);
+      if (count == 0) return; // do not report 0-value delta counters
     } else {
       count = counter.count();
     }
@@ -70,42 +78,172 @@ public void processCounter(MetricName name, Counter counter, FlushProcessorConte
   }
 
   @Override
-  public void processHistogram(MetricName name, Histogram histogram, FlushProcessorContext context) throws Exception {
-    if (histogram instanceof WavefrontHistogram && useWavefrontHistograms) {
+  public void processHistogram(
+      MetricName name, Histogram histogram, FlushProcessorContext context) {
+    if (histogram instanceof WavefrontHistogram) {
       WavefrontHistogram wavefrontHistogram = (WavefrontHistogram) histogram;
-      wavefront.report.Histogram.Builder builder = wavefront.report.Histogram.newBuilder();
-      builder.setBins(Lists.newLinkedList());
-      builder.setCounts(Lists.newLinkedList());
-      long minMillis = Long.MAX_VALUE;
-      if (wavefrontHistogram.count() == 0) return;
-      for (WavefrontHistogram.MinuteBin minuteBin : wavefrontHistogram.bins(true)) {
-        builder.getBins().add(minuteBin.getDist().quantile(.5));
-        builder.getCounts().add(Math.toIntExact(minuteBin.getDist().size()));
-        minMillis = Long.min(minMillis, minuteBin.getMinMillis());
+      if (useWavefrontHistograms) {
+        // export Wavefront histograms in its native format
+        if (wavefrontHistogram.count() == 0) return;
+        for (WavefrontHistogram.MinuteBin bin : wavefrontHistogram.bins(true)) {
+          if (bin.getDist().size() == 0) continue;
+          int size = bin.getDist().centroids().size();
+          List centroids = new ArrayList<>(size);
+          List counts = new ArrayList<>(size);
+          for (Centroid centroid : bin.getDist().centroids()) {
+            centroids.add(centroid.mean());
+            counts.add(centroid.count());
+          }
+          context.report(
+              wavefront.report.Histogram.newBuilder()
+                  .setDuration(60_000)
+                  . // minute bins
+                  setType(HistogramType.TDIGEST)
+                  .setBins(centroids)
+                  .setCounts(counts)
+                  .build(),
+              bin.getMinMillis());
+          histogramCounter.inc();
+        }
+      } else {
+        // convert Wavefront histogram to Yammer-style histogram
+        TDigest tDigest = new AVLTreeDigest(100);
+        List bins = wavefrontHistogram.bins(true);
+        bins.stream().map(WavefrontHistogram.MinuteBin::getDist).forEach(tDigest::add);
+        context.reportSubMetric(
+            tDigest.centroids().stream().mapToLong(Centroid::count).sum(), "count");
+        Summarizable summarizable =
+            new Summarizable() {
+              @Override
+              public double max() {
+                return tDigest.centroids().stream()
+                    .map(Centroid::mean)
+                    .max(Comparator.naturalOrder())
+                    .orElse(Double.NaN);
+              }
+
+              @Override
+              public double min() {
+                return tDigest.centroids().stream()
+                    .map(Centroid::mean)
+                    .min(Comparator.naturalOrder())
+                    .orElse(Double.NaN);
+              }
+
+              @Override
+              public double mean() {
+                Centroid mean =
+                    tDigest.centroids().stream()
+                        .reduce(
+                            (x, y) ->
+                                new Centroid(
+                                    x.mean() + (y.mean() * y.count()), x.count() + y.count()))
+                        .orElse(null);
+                return mean == null || tDigest.centroids().size() == 0
+                    ? Double.NaN
+                    : mean.mean() / mean.count();
+              }
+
+              @Override
+              public double stdDev() {
+                return Double.NaN;
+              }
+
+              @Override
+              public double sum() {
+                return Double.NaN;
+              }
+            };
+        for (Map.Entry entry :
+            MetricsToTimeseries.explodeSummarizable(summarizable, reportEmptyHistogramStats)
+                .entrySet()) {
+          if (!entry.getValue().isNaN()) {
+            context.reportSubMetric(entry.getValue(), entry.getKey());
+          }
+        }
+        Sampling sampling =
+            () ->
+                new Snapshot(new double[0]) {
+                  @Override
+                  public double get75thPercentile() {
+                    return tDigest.quantile(.75);
+                  }
+
+                  @Override
+                  public double get95thPercentile() {
+                    return tDigest.quantile(.95);
+                  }
+
+                  @Override
+                  public double get98thPercentile() {
+                    return tDigest.quantile(.98);
+                  }
+
+                  @Override
+                  public double get999thPercentile() {
+                    return tDigest.quantile(.999);
+                  }
+
+                  @Override
+                  public double get99thPercentile() {
+                    return tDigest.quantile(.99);
+                  }
+
+                  @Override
+                  public double getMedian() {
+                    return tDigest.quantile(.50);
+                  }
+
+                  @Override
+                  public double getValue(double quantile) {
+                    return tDigest.quantile(quantile);
+                  }
+
+                  @Override
+                  public double[] getValues() {
+                    return new double[0];
+                  }
+
+                  @Override
+                  public int size() {
+                    return (int) tDigest.size();
+                  }
+                };
+        for (Map.Entry entry :
+            MetricsToTimeseries.explodeSampling(sampling, reportEmptyHistogramStats).entrySet()) {
+          if (!entry.getValue().isNaN()) {
+            context.reportSubMetric(entry.getValue(), entry.getKey());
+          }
+        }
+        sentCounter.inc();
       }
-      builder.setType(HistogramType.TDIGEST);
-      builder.setDuration(Math.toIntExact(currentMillis.get() - minMillis));
-      context.report(builder.build());
     } else {
       context.reportSubMetric(histogram.count(), "count");
-      for (Map.Entry entry : MetricsToTimeseries.explodeSummarizable(histogram, reportEmptyHistogramStats).entrySet()) {
-        context.reportSubMetric(entry.getValue(), entry.getKey());
+      for (Map.Entry entry :
+          MetricsToTimeseries.explodeSummarizable(histogram, reportEmptyHistogramStats)
+              .entrySet()) {
+        if (!entry.getValue().isNaN()) {
+          context.reportSubMetric(entry.getValue(), entry.getKey());
+        }
       }
-      for (Map.Entry entry : MetricsToTimeseries.explodeSampling(histogram, reportEmptyHistogramStats).entrySet()) {
-        context.reportSubMetric(entry.getValue(), entry.getKey());
+      for (Map.Entry entry :
+          MetricsToTimeseries.explodeSampling(histogram, reportEmptyHistogramStats).entrySet()) {
+        if (!entry.getValue().isNaN()) {
+          context.reportSubMetric(entry.getValue(), entry.getKey());
+        }
       }
+      sentCounter.inc();
       histogram.clear();
     }
-    sentCounter.inc();
   }
 
   @Override
-  public void processTimer(MetricName name, Timer timer, FlushProcessorContext context) throws Exception {
+  public void processTimer(MetricName name, Timer timer, FlushProcessorContext context) {
     throw new UnsupportedOperationException();
   }
 
   @Override
-  public void processGauge(MetricName name, Gauge gauge, FlushProcessorContext context) throws Exception {
+  public void processGauge(MetricName name, Gauge gauge, FlushProcessorContext context) {
     @SuppressWarnings("unchecked")
     ChangeableGauge changeableGauge = (ChangeableGauge) gauge;
     Double value = changeableGauge.value();
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/FlushProcessorContext.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/FlushProcessorContext.java
index 1033a9512..a59afa09d 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/FlushProcessorContext.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/FlushProcessorContext.java
@@ -1,25 +1,29 @@
 package com.wavefront.agent.logsharvesting;
 
-import com.wavefront.agent.PointHandler;
-
+import com.wavefront.agent.handlers.ReportableEntityHandler;
 import com.wavefront.common.MetricConstants;
+import java.util.function.Supplier;
 import wavefront.report.Histogram;
 import wavefront.report.ReportPoint;
 import wavefront.report.TimeSeries;
 
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public class FlushProcessorContext {
   private final long timestamp;
-  private TimeSeries timeSeries;
-  private PointHandler pointHandler;
-  private String prefix;
+  private final TimeSeries timeSeries;
+  private final Supplier> pointHandlerSupplier;
+  private final Supplier> histogramHandlerSupplier;
+  private final String prefix;
 
-  FlushProcessorContext(TimeSeries timeSeries, String prefix, PointHandler pointHandler) {
+  FlushProcessorContext(
+      TimeSeries timeSeries,
+      String prefix,
+      Supplier> pointHandlerSupplier,
+      Supplier> histogramHandlerSupplier) {
     this.timeSeries = TimeSeries.newBuilder(timeSeries).build();
-    this.pointHandler = pointHandler;
     this.prefix = prefix;
+    this.pointHandlerSupplier = pointHandlerSupplier;
+    this.histogramHandlerSupplier = histogramHandlerSupplier;
     timestamp = System.currentTimeMillis();
   }
 
@@ -27,13 +31,17 @@ String getMetricName() {
     return timeSeries.getMetric();
   }
 
-  private ReportPoint.Builder reportPointBuilder() {
+  private ReportPoint.Builder reportPointBuilder(long timestamp) {
     String newName = timeSeries.getMetric();
     // if prefix is provided then add the delta before the prefix
-    if (prefix != null && (newName.startsWith(MetricConstants.DELTA_PREFIX) ||
-            newName.startsWith(MetricConstants.DELTA_PREFIX_2))) {
-      newName = MetricConstants.DELTA_PREFIX + prefix + "." + newName.substring(MetricConstants
-              .DELTA_PREFIX.length());
+    if (prefix != null
+        && (newName.startsWith(MetricConstants.DELTA_PREFIX)
+            || newName.startsWith(MetricConstants.DELTA_PREFIX_2))) {
+      newName =
+          MetricConstants.DELTA_PREFIX
+              + prefix
+              + "."
+              + newName.substring(MetricConstants.DELTA_PREFIX.length());
     } else {
       newName = prefix == null ? timeSeries.getMetric() : prefix + "." + timeSeries.getMetric();
     }
@@ -45,24 +53,23 @@ private ReportPoint.Builder reportPointBuilder() {
   }
 
   void report(ReportPoint reportPoint) {
-    pointHandler.reportPoint(reportPoint, reportPoint.toString());
+    pointHandlerSupplier.get().report(reportPoint);
   }
 
   void report(double value) {
-    report(reportPointBuilder().setValue(value).build());
+    report(reportPointBuilder(this.timestamp).setValue(value).build());
   }
 
   void report(long value) {
-    report(reportPointBuilder().setValue(value).build());
+    report(reportPointBuilder(this.timestamp).setValue(value).build());
   }
 
-  void report(Histogram value) {
-    report(reportPointBuilder().setValue(value).build());
+  void report(Histogram value, long timestamp) {
+    histogramHandlerSupplier.get().report(reportPointBuilder(timestamp).setValue(value).build());
   }
 
   void reportSubMetric(double value, String subMetric) {
-    ReportPoint.Builder builder = reportPointBuilder();
+    ReportPoint.Builder builder = reportPointBuilder(this.timestamp);
     report(builder.setValue(value).setMetric(builder.getMetric() + "." + subMetric).build());
   }
-
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/InteractiveLogsTester.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/InteractiveLogsTester.java
index a8cb9db55..a32691d5c 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/InteractiveLogsTester.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/InteractiveLogsTester.java
@@ -1,79 +1,107 @@
 package com.wavefront.agent.logsharvesting;
 
-import com.wavefront.agent.PointHandler;
-import com.wavefront.agent.PointHandlerImpl;
+import com.wavefront.agent.InteractiveTester;
 import com.wavefront.agent.config.ConfigurationException;
 import com.wavefront.agent.config.LogsIngestionConfig;
-
+import com.wavefront.agent.formatter.DataFormat;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.ingester.ReportPointSerializer;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.util.List;
 import java.util.Scanner;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
-
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-
 import wavefront.report.ReportPoint;
 
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
-public class InteractiveLogsTester {
+/** @author Mori Bellamy (mori@wavefront.com) */
+public class InteractiveLogsTester implements InteractiveTester {
 
   private final Supplier logsIngestionConfigSupplier;
   private final String prefix;
   private final Scanner stdin;
 
-  public InteractiveLogsTester(Supplier logsIngestionConfigSupplier, String prefix)
-      throws ConfigurationException {
+  public InteractiveLogsTester(
+      Supplier logsIngestionConfigSupplier, String prefix) {
     this.logsIngestionConfigSupplier = logsIngestionConfigSupplier;
     this.prefix = prefix;
     stdin = new Scanner(System.in);
   }
 
-  /**
-   * Read one line of stdin and print a message to stdout.
-   */
+  /** Read one line of stdin and print a message to stdout. */
+  @Override
   public boolean interactiveTest() throws ConfigurationException {
     final AtomicBoolean reported = new AtomicBoolean(false);
 
-    LogsIngester logsIngester = new LogsIngester(
-        new PointHandler() {
+    ReportableEntityHandlerFactory factory =
+        new ReportableEntityHandlerFactory() {
+          @SuppressWarnings("unchecked")
           @Override
-          public void reportPoint(ReportPoint point, @Nullable String debugLine) {
-            reported.set(true);
-            System.out.println(PointHandlerImpl.pointToString(point));
+          public  ReportableEntityHandler getHandler(HandlerKey handlerKey) {
+            return (ReportableEntityHandler)
+                new ReportableEntityHandler() {
+                  @Override
+                  public void report(ReportPoint reportPoint) {
+                    reported.set(true);
+                    System.out.println(ReportPointSerializer.pointToString(reportPoint));
+                  }
+
+                  @Override
+                  public void block(ReportPoint reportPoint) {
+                    System.out.println("Blocked: " + reportPoint);
+                  }
+
+                  @Override
+                  public void block(@Nullable ReportPoint reportPoint, @Nullable String message) {
+                    System.out.println("Blocked: " + reportPoint);
+                  }
+
+                  @Override
+                  public void reject(@Nullable ReportPoint reportPoint, @Nullable String message) {
+                    System.out.println("Rejected: " + reportPoint);
+                  }
+
+                  @Override
+                  public void reject(@Nonnull String t, @Nullable String message) {
+                    System.out.println("Rejected: " + t);
+                  }
+
+                  @Override
+                  public void setLogFormat(DataFormat format) {
+                    throw new UnsupportedOperationException();
+                  }
+
+                  @Override
+                  public void shutdown() {}
+                };
           }
 
           @Override
-          public void reportPoints(List points) {
-            for (ReportPoint point : points) reportPoint(point, "");
-          }
+          public void shutdown(@Nonnull String handle) {}
+        };
 
+    LogsIngester logsIngester = new LogsIngester(factory, logsIngestionConfigSupplier, prefix);
+
+    String line = stdin.nextLine();
+    logsIngester.ingestLog(
+        new LogsMessage() {
           @Override
-          public void handleBlockedPoint(@Nullable String pointLine) {
-            System.out.println("Blocked point: " + pointLine);
+          public String getLogLine() {
+            return line;
           }
-        },
-        logsIngestionConfigSupplier, prefix, System::currentTimeMillis);
 
-    String line = stdin.nextLine();
-    logsIngester.ingestLog(new LogsMessage() {
-      @Override
-      public String getLogLine() {
-        return line;
-      }
-
-      @Override
-      public String hostOrDefault(String fallbackHost) {
-        try {
-          return InetAddress.getLocalHost().getHostName();
-        } catch (UnknownHostException e) {
-          return "localhost";
-        }
-      }
-    });
+          @Override
+          public String hostOrDefault(String fallbackHost) {
+            try {
+              return InetAddress.getLocalHost().getHostName();
+            } catch (UnknownHostException e) {
+              return "localhost";
+            }
+          }
+        });
 
     logsIngester.flush();
     if (!reported.get()) {
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsIngester.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsIngester.java
index d2dabf097..41e93f02f 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsIngester.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsIngester.java
@@ -1,27 +1,27 @@
 package com.wavefront.agent.logsharvesting;
 
+import com.github.benmanes.caffeine.cache.Ticker;
 import com.google.common.annotations.VisibleForTesting;
-
-import com.wavefront.agent.PointHandler;
 import com.wavefront.agent.config.ConfigurationException;
 import com.wavefront.agent.config.LogsIngestionConfig;
 import com.wavefront.agent.config.MetricMatcher;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Counter;
 import com.yammer.metrics.core.Metric;
 import com.yammer.metrics.core.MetricName;
-
+import com.yammer.metrics.core.MetricsRegistry;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-
 import wavefront.report.TimeSeries;
 
 /**
- * Consumes log messages sent to {@link #ingestLog(LogsMessage)}. Configures and starts the periodic flush of
- * consumed metric data to Wavefront.
+ * Consumes log messages sent to {@link #ingestLog(LogsMessage)}. Configures and starts the periodic
+ * flush of consumed metric data to Wavefront.
  *
  * @author Mori Bellamy (mori@wavefront.com)
  */
@@ -29,53 +29,99 @@ public class LogsIngester {
   protected static final Logger logger = Logger.getLogger(LogsIngester.class.getCanonicalName());
   private static final ReadProcessor readProcessor = new ReadProcessor();
   private final FlushProcessor flushProcessor;
-  private final PointHandler pointHandler;
   // A map from "true" to the currently loaded logs ingestion config.
-  @VisibleForTesting
-  final LogsIngestionConfigManager logsIngestionConfigManager;
-  private final Counter unparsed, parsed, sent;
+  @VisibleForTesting final LogsIngestionConfigManager logsIngestionConfigManager;
+  private final Counter unparsed, parsed;
   private final Supplier currentMillis;
   private final MetricsReporter metricsReporter;
   private EvictingMetricsRegistry evictingMetricsRegistry;
 
   /**
-   * @param pointHandler                play parsed metrics
-   * @param logsIngestionConfigSupplier supplied configuration object for logs harvesting. May be reloaded. Must return
-   *                                    "null" on any problems, as opposed to throwing
-   * @param prefix                      all harvested metrics start with this prefix
-   * @param currentMillis               supplier of the current time in millis
-   * @throws ConfigurationException if the first config from logsIngestionConfigSupplier is null
+   * Create an instance using system clock.
+   *
+   * @param handlerFactory factory for point handlers and histogram handlers
+   * @param logsIngestionConfigSupplier supplied configuration object for logs harvesting. May be
+   *     reloaded. Must return "null" on any problems, as opposed to throwing.
+   * @param prefix all harvested metrics start with this prefix
    */
-  public LogsIngester(PointHandler pointHandler, Supplier logsIngestionConfigSupplier,
-                      String prefix, Supplier currentMillis) throws ConfigurationException {
-    logsIngestionConfigManager = new LogsIngestionConfigManager(
+  public LogsIngester(
+      ReportableEntityHandlerFactory handlerFactory,
+      Supplier logsIngestionConfigSupplier,
+      String prefix)
+      throws ConfigurationException {
+    this(
+        handlerFactory,
         logsIngestionConfigSupplier,
-        removedMetricMatcher -> evictingMetricsRegistry.evict(removedMetricMatcher));
+        prefix,
+        System::currentTimeMillis,
+        Ticker.systemTicker());
+  }
+
+  /**
+   * Create an instance using provided clock and nano.
+   *
+   * @param handlerFactory factory for point handlers and histogram handlers
+   * @param logsIngestionConfigSupplier supplied configuration object for logs harvesting. May be
+   *     reloaded. Must return "null" on any problems, as opposed to throwing.
+   * @param prefix all harvested metrics start with this prefix
+   * @param currentMillis supplier of the current time in millis
+   * @param ticker nanosecond-precision clock for Caffeine cache.
+   * @throws ConfigurationException if the first config from logsIngestionConfigSupplier is null
+   */
+  @VisibleForTesting
+  LogsIngester(
+      ReportableEntityHandlerFactory handlerFactory,
+      Supplier logsIngestionConfigSupplier,
+      String prefix,
+      Supplier currentMillis,
+      Ticker ticker)
+      throws ConfigurationException {
+    logsIngestionConfigManager =
+        new LogsIngestionConfigManager(
+            logsIngestionConfigSupplier,
+            removedMetricMatcher -> evictingMetricsRegistry.evict(removedMetricMatcher));
     LogsIngestionConfig logsIngestionConfig = logsIngestionConfigManager.getConfig();
 
-    this.evictingMetricsRegistry = new EvictingMetricsRegistry(
-        logsIngestionConfig.expiryMillis, true, currentMillis);
+    MetricsRegistry metricsRegistry = new MetricsRegistry();
+    this.evictingMetricsRegistry =
+        new EvictingMetricsRegistry(
+            metricsRegistry,
+            logsIngestionConfig.expiryMillis,
+            true,
+            logsIngestionConfig.useDeltaCounters,
+            currentMillis,
+            ticker);
 
     // Logs harvesting metrics.
     this.unparsed = Metrics.newCounter(new MetricName("logsharvesting", "", "unparsed"));
     this.parsed = Metrics.newCounter(new MetricName("logsharvesting", "", "parsed"));
-    this.sent = Metrics.newCounter(new MetricName("logsharvesting", "", "sent"));
     this.currentMillis = currentMillis;
-    this.flushProcessor = new FlushProcessor(sent, currentMillis, logsIngestionConfig.useWavefrontHistograms,
-        logsIngestionConfig.reportEmptyHistogramStats);
-
-    // Set up user specified metric harvesting.
-    this.pointHandler = pointHandler;
+    this.flushProcessor =
+        new FlushProcessor(
+            currentMillis,
+            logsIngestionConfig.useWavefrontHistograms,
+            logsIngestionConfig.reportEmptyHistogramStats);
 
     // Continually flush user metrics to Wavefront.
-    this.metricsReporter = new MetricsReporter(
-        evictingMetricsRegistry.metricsRegistry(), flushProcessor, "FilebeatMetricsReporter", pointHandler, prefix);
+    this.metricsReporter =
+        new MetricsReporter(
+            metricsRegistry, flushProcessor, "FilebeatMetricsReporter", handlerFactory, prefix);
   }
 
   public void start() {
-    this.metricsReporter.start(
-        this.logsIngestionConfigManager.getConfig().aggregationIntervalSeconds,
-        TimeUnit.SECONDS);
+    long interval = this.logsIngestionConfigManager.getConfig().aggregationIntervalSeconds;
+    this.metricsReporter.start(interval, TimeUnit.SECONDS);
+    // check for expired cached items and trigger evictions every 2x aggregationIntervalSeconds
+    // but no more than once a minute. This is a workaround for the issue that surfaces mostly
+    // during testing, when there are no matching log messages at all for more than
+    // expiryMillis,
+    // which means there is no cache access and no time-based evictions are performed.
+    Executors.newSingleThreadScheduledExecutor()
+        .scheduleWithFixedDelay(
+            evictingMetricsRegistry::cleanUp,
+            interval * 3 / 2,
+            Math.max(60, interval * 2),
+            TimeUnit.SECONDS);
   }
 
   public void flush() {
@@ -111,7 +157,8 @@ public void ingestLog(LogsMessage logsMessage) {
   }
 
   private boolean maybeIngestLog(
-      BiFunction metricLoader, MetricMatcher metricMatcher,
+      BiFunction metricLoader,
+      MetricMatcher metricMatcher,
       LogsMessage logsMessage) {
     Double[] output = {null};
     TimeSeries timeSeries = metricMatcher.timeSeries(logsMessage, output);
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsIngestionConfigManager.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsIngestionConfigManager.java
index 90ab8942f..4314a64b3 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsIngestionConfigManager.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsIngestionConfigManager.java
@@ -1,13 +1,14 @@
 package com.wavefront.agent.logsharvesting;
 
-import com.google.common.annotations.VisibleForTesting;
-
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.google.common.annotations.VisibleForTesting;
 import com.wavefront.agent.config.ConfigurationException;
 import com.wavefront.agent.config.LogsIngestionConfig;
 import com.wavefront.agent.config.MetricMatcher;
-
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.TimeUnit;
@@ -22,58 +23,102 @@
  * @author Mori Bellamy (mori@wavefront.com)
  */
 public class LogsIngestionConfigManager {
-  protected static final Logger logger = Logger.getLogger(LogsIngestionConfigManager.class.getCanonicalName());
+  protected static final Logger logger =
+      Logger.getLogger(LogsIngestionConfigManager.class.getCanonicalName());
+  private static final Counter configReloads =
+      Metrics.newCounter(new MetricName("logsharvesting", "", "config-reloads.successful"));
+  private static final Counter failedConfigReloads =
+      Metrics.newCounter(new MetricName("logsharvesting", "", "config-reloads.failed"));
   private LogsIngestionConfig lastParsedConfig;
   // The only key in this cache is "true". Basically we want the cache expiry and reloading logic.
   private final LoadingCache logsIngestionConfigLoadingCache;
   private final Consumer removalListener;
 
-  public LogsIngestionConfigManager(Supplier logsIngestionConfigSupplier,
-                                    Consumer removalListener) throws ConfigurationException {
+  public LogsIngestionConfigManager(
+      Supplier logsIngestionConfigSupplier,
+      Consumer removalListener)
+      throws ConfigurationException {
     this.removalListener = removalListener;
     lastParsedConfig = logsIngestionConfigSupplier.get();
-    if (lastParsedConfig == null) throw new ConfigurationException("Could not load initial config.");
+    if (lastParsedConfig == null)
+      throw new ConfigurationException("Could not load initial config.");
     lastParsedConfig.verifyAndInit();
-    this.logsIngestionConfigLoadingCache = Caffeine.newBuilder()
-        .expireAfterWrite(lastParsedConfig.configReloadIntervalSeconds, TimeUnit.SECONDS)
-        .build((ignored) -> {
-          LogsIngestionConfig nextConfig = logsIngestionConfigSupplier.get();
-          if (nextConfig == null) {
-            logger.warning("Could not load a new logs ingestion config file, check above for a stack trace.");
-          } else if (!lastParsedConfig.equals(nextConfig)) {
-            nextConfig.verifyAndInit();  // If it throws, we keep the last (good) config.
-            processConfigChange(nextConfig);
-            logger.info("Loaded new config: " + lastParsedConfig.toString());
-          }
-          return lastParsedConfig;
-        });
+    this.logsIngestionConfigLoadingCache =
+        Caffeine.newBuilder()
+            .expireAfterWrite(lastParsedConfig.configReloadIntervalSeconds, TimeUnit.SECONDS)
+            .build(
+                (ignored) -> {
+                  LogsIngestionConfig nextConfig = logsIngestionConfigSupplier.get();
+                  if (nextConfig == null) {
+                    logger.warning("Unable to reload logs ingestion config file!");
+                    failedConfigReloads.inc();
+                  } else if (!lastParsedConfig.equals(nextConfig)) {
+                    nextConfig.verifyAndInit(); // If it throws, we keep the last
+                    // (good) config.
+                    processConfigChange(nextConfig);
+                    logger.info("Loaded new config: " + lastParsedConfig.toString());
+                    configReloads.inc();
+                  }
+                  return lastParsedConfig;
+                });
 
     // Force reload every N seconds.
-    new Timer().schedule(new TimerTask() {
-      @Override
-      public void run() {
-        try {
-          logsIngestionConfigLoadingCache.get(true);
-        } catch (Exception e) {
-          logger.log(Level.SEVERE, "Cannot load a new logs ingestion config.", e);
-        }
-      }
-    }, lastParsedConfig.aggregationIntervalSeconds, lastParsedConfig.aggregationIntervalSeconds);
+    new Timer("Timer-logsingestion-configmanager")
+        .schedule(
+            new TimerTask() {
+              @Override
+              public void run() {
+                try {
+                  logsIngestionConfigLoadingCache.get(true);
+                } catch (Exception e) {
+                  logger.log(Level.SEVERE, "Cannot load a new logs ingestion config.", e);
+                }
+              }
+            },
+            lastParsedConfig.aggregationIntervalSeconds,
+            lastParsedConfig.aggregationIntervalSeconds);
   }
 
   public LogsIngestionConfig getConfig() {
     return logsIngestionConfigLoadingCache.get(true);
   }
 
-  /**
-   * Forces the next call to {@link #getConfig()} to call the config supplier.
-   */
+  /** Forces the next call to {@link #getConfig()} to call the config supplier. */
   @VisibleForTesting
   public void forceConfigReload() {
     logsIngestionConfigLoadingCache.invalidate(true);
   }
 
   private void processConfigChange(LogsIngestionConfig nextConfig) {
+    if (nextConfig.useWavefrontHistograms != lastParsedConfig.useWavefrontHistograms) {
+      logger.warning(
+          "useWavefrontHistograms property cannot be changed at runtime, "
+              + "proxy restart required!");
+    }
+    if (nextConfig.useDeltaCounters != lastParsedConfig.useDeltaCounters) {
+      logger.warning(
+          "useDeltaCounters property cannot be changed at runtime, " + "proxy restart required!");
+    }
+    if (nextConfig.reportEmptyHistogramStats != lastParsedConfig.reportEmptyHistogramStats) {
+      logger.warning(
+          "reportEmptyHistogramStats property cannot be changed at runtime, "
+              + "proxy restart required!");
+    }
+    if (!nextConfig.aggregationIntervalSeconds.equals(
+        lastParsedConfig.aggregationIntervalSeconds)) {
+      logger.warning(
+          "aggregationIntervalSeconds property cannot be changed at runtime, "
+              + "proxy restart required!");
+    }
+    if (nextConfig.configReloadIntervalSeconds != lastParsedConfig.configReloadIntervalSeconds) {
+      logger.warning(
+          "configReloadIntervalSeconds property cannot be changed at runtime, "
+              + "proxy restart required!");
+    }
+    if (nextConfig.expiryMillis != lastParsedConfig.expiryMillis) {
+      logger.warning(
+          "expiryMillis property cannot be changed at runtime, " + "proxy restart required!");
+    }
     for (MetricMatcher oldMatcher : lastParsedConfig.counters) {
       if (!nextConfig.counters.contains(oldMatcher)) removalListener.accept(oldMatcher);
     }
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsMessage.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsMessage.java
index 542e0635c..d81c46255 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsMessage.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/LogsMessage.java
@@ -1,8 +1,6 @@
 package com.wavefront.agent.logsharvesting;
 
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public interface LogsMessage {
   String getLogLine();
 
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/MalformedMessageException.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/MalformedMessageException.java
index 87b6f7681..d6acd9aac 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/MalformedMessageException.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/MalformedMessageException.java
@@ -1,8 +1,6 @@
 package com.wavefront.agent.logsharvesting;
 
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public class MalformedMessageException extends Exception {
   MalformedMessageException(String msg) {
     super(msg);
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/MetricsReporter.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/MetricsReporter.java
index bba02d47d..bbe389aeb 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/MetricsReporter.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/MetricsReporter.java
@@ -1,39 +1,57 @@
 package com.wavefront.agent.logsharvesting;
 
-import com.wavefront.agent.PointHandler;
+import static com.wavefront.common.Utils.lazySupplier;
+
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.data.ReportableEntityType;
 import com.yammer.metrics.core.Metric;
 import com.yammer.metrics.core.MetricName;
 import com.yammer.metrics.core.MetricsRegistry;
 import com.yammer.metrics.reporting.AbstractPollingReporter;
-
 import java.util.Map;
 import java.util.SortedMap;
+import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-
+import wavefront.report.ReportPoint;
 import wavefront.report.TimeSeries;
 
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public class MetricsReporter extends AbstractPollingReporter {
 
   protected static final Logger logger = Logger.getLogger(MetricsReporter.class.getCanonicalName());
   private final FlushProcessor flushProcessor;
-  private final PointHandler pointHandler;
+  private final Supplier> pointHandlerSupplier;
+  private final Supplier> histogramHandlerSupplier;
   private final String prefix;
 
-  public MetricsReporter(MetricsRegistry metricsRegistry, FlushProcessor flushProcessor, String name,
-                         PointHandler pointHandler, String prefix) {
+  public MetricsReporter(
+      MetricsRegistry metricsRegistry,
+      FlushProcessor flushProcessor,
+      String name,
+      ReportableEntityHandlerFactory handlerFactory,
+      String prefix) {
     super(metricsRegistry, name);
     this.flushProcessor = flushProcessor;
-    this.pointHandler = pointHandler;
+    this.pointHandlerSupplier =
+        lazySupplier(
+            () ->
+                handlerFactory.getHandler(
+                    HandlerKey.of(ReportableEntityType.POINT, "logs-ingester")));
+    this.histogramHandlerSupplier =
+        lazySupplier(
+            () ->
+                handlerFactory.getHandler(
+                    HandlerKey.of(ReportableEntityType.HISTOGRAM, "logs-ingester")));
     this.prefix = prefix;
   }
 
   @Override
   public void run() {
-    for (Map.Entry> group : getMetricsRegistry().groupedMetrics().entrySet()) {
+    for (Map.Entry> group :
+        getMetricsRegistry().groupedMetrics().entrySet()) {
       for (Map.Entry entry : group.getValue().entrySet()) {
         if (entry.getValue() == null || entry.getKey() == null) {
           logger.severe("Application Error! Pulled null value from metrics registry.");
@@ -42,12 +60,15 @@ public void run() {
         Metric metric = entry.getValue();
         try {
           TimeSeries timeSeries = TimeSeriesUtils.fromMetricName(metricName);
-          metric.processWith(flushProcessor, metricName, new FlushProcessorContext(timeSeries, prefix, pointHandler));
+          metric.processWith(
+              flushProcessor,
+              metricName,
+              new FlushProcessorContext(
+                  timeSeries, prefix, pointHandlerSupplier, histogramHandlerSupplier));
         } catch (Exception e) {
           logger.log(Level.SEVERE, "Uncaught exception in MetricsReporter", e);
         }
       }
     }
   }
-
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/RawLogsIngester.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/RawLogsIngester.java
deleted file mode 100644
index 15e5a4404..000000000
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/RawLogsIngester.java
+++ /dev/null
@@ -1,149 +0,0 @@
-package com.wavefront.agent.logsharvesting;
-
-import com.wavefront.common.TaggedMetricName;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
-
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
-import java.util.logging.Logger;
-
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.ChannelDuplexHandler;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelOption;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.ServerChannel;
-import io.netty.channel.SimpleChannelInboundHandler;
-import io.netty.channel.epoll.Epoll;
-import io.netty.channel.epoll.EpollEventLoopGroup;
-import io.netty.channel.epoll.EpollServerSocketChannel;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.codec.LineBasedFrameDecoder;
-import io.netty.handler.codec.string.StringDecoder;
-import io.netty.handler.timeout.IdleState;
-import io.netty.handler.timeout.IdleStateEvent;
-import io.netty.handler.timeout.IdleStateHandler;
-import io.netty.util.CharsetUtil;
-
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
-public class RawLogsIngester {
-
-  private static final Logger logger = Logger.getLogger(RawLogsIngester.class.getCanonicalName());
-  private LogsIngester logsIngester;
-  private int port;
-  private Supplier now;
-  private Counter received;
-  private final Counter connectionsAccepted;
-  private final Counter connectionsIdleClosed;
-  private int maxLength = 4096;
-  private int channelIdleTimeout = (int) TimeUnit.HOURS.toSeconds(1);
-
-  public RawLogsIngester(LogsIngester logsIngester, int port, Supplier now) {
-    this.logsIngester = logsIngester;
-    this.port = port;
-    this.now = now;
-    this.received = Metrics.newCounter(new MetricName("logsharvesting", "", "raw-received"));
-    this.connectionsAccepted = Metrics.newCounter(new TaggedMetricName("listeners", "connections.accepted",
-        "port", String.valueOf(port)));
-    this.connectionsIdleClosed = Metrics.newCounter(new TaggedMetricName("listeners", "connections.idle.closed",
-        "port", String.valueOf(port)));
-  }
-
-  public RawLogsIngester withMaxLength(int maxLength) {
-    this.maxLength = maxLength;
-    return this;
-  }
-
-  public RawLogsIngester withChannelIdleTimeout(int channelIdleTimeout) {
-    this.channelIdleTimeout = channelIdleTimeout;
-    return this;
-  }
-
-  public void listen() throws InterruptedException {
-    ServerBootstrap serverBootstrap = new ServerBootstrap();
-    EventLoopGroup acceptorGroup;
-    EventLoopGroup handlerGroup;
-    Class socketChannelClass;
-    if (Epoll.isAvailable()) {
-      logger.fine("Using native socket transport for port " + port);
-      acceptorGroup = new EpollEventLoopGroup(2);
-      handlerGroup = new EpollEventLoopGroup(10);
-      socketChannelClass = EpollServerSocketChannel.class;
-    } else {
-      logger.fine("Using NIO socket transport for port " + port);
-      acceptorGroup = new NioEventLoopGroup(2);
-      handlerGroup = new NioEventLoopGroup(10);
-      socketChannelClass = NioServerSocketChannel.class;
-    }
-
-    serverBootstrap.group(acceptorGroup, handlerGroup)
-        .channel(socketChannelClass)
-        .childHandler(new SocketInitializer())
-        .option(ChannelOption.SO_BACKLOG, 5)
-        .option(ChannelOption.SO_KEEPALIVE, true);
-
-    serverBootstrap.bind(port).sync();
-  }
-
-  public void ingestLog(ChannelHandlerContext ctx, String log) {
-    logsIngester.ingestLog(new LogsMessage() {
-      @Override
-      public String getLogLine() {
-        return log;
-      }
-
-      @Override
-      public String hostOrDefault(String fallbackHost) {
-        if (!(ctx.channel().remoteAddress() instanceof InetSocketAddress)) return fallbackHost;
-        InetSocketAddress inetSocketAddress = (InetSocketAddress) ctx.channel().remoteAddress();
-        InetAddress inetAddress = inetSocketAddress.getAddress();
-        String host = inetAddress.getCanonicalHostName();
-        if (host == null || host.equals("")) return fallbackHost;
-        return host;
-      }
-    });
-  }
-
-  private class SocketInitializer extends ChannelInitializer {
-    @Override
-    protected void initChannel(SocketChannel ch) throws Exception {
-      connectionsAccepted.inc();
-      ChannelPipeline channelPipeline = ch.pipeline();
-      channelPipeline.addLast(LineBasedFrameDecoder.class.getName(), new LineBasedFrameDecoder(maxLength));
-      channelPipeline.addLast(StringDecoder.class.getName(), new StringDecoder(CharsetUtil.UTF_8));
-      channelPipeline.addLast("logsIngestionHandler", new SimpleChannelInboundHandler() {
-        @Override
-        protected void channelRead0(ChannelHandlerContext ctx, String msg) throws Exception {
-          received.inc();
-          ingestLog(ctx, msg);
-        }
-      });
-      channelPipeline.addLast("idleStateHandler", new IdleStateHandler(channelIdleTimeout, 0, 0));
-      channelPipeline.addLast("idleChannelTerminator", new ChannelDuplexHandler() {
-        @Override
-        public void userEventTriggered(ChannelHandlerContext ctx,
-                                       Object evt) throws Exception {
-          if (evt instanceof IdleStateEvent) {
-            if (((IdleStateEvent) evt).state() == IdleState.READER_IDLE) {
-              connectionsIdleClosed.inc();
-              logger.info("Closing idle connection to raw logs client, inactivity timeout " +
-                  channelIdleTimeout + "s expired: " + ctx.channel());
-              ctx.close();
-            }
-          }
-        }
-      });
-    }
-  }
-
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/ReadProcessor.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/ReadProcessor.java
index 36a6f3fbb..861c944e9 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/ReadProcessor.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/ReadProcessor.java
@@ -9,22 +9,20 @@
 import com.yammer.metrics.core.Timer;
 import com.yammer.metrics.core.WavefrontHistogram;
 
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public class ReadProcessor implements MetricProcessor {
   @Override
-  public void processMeter(MetricName name, Metered meter, ReadProcessorContext context) throws Exception {
+  public void processMeter(MetricName name, Metered meter, ReadProcessorContext context) {
     throw new UnsupportedOperationException();
   }
 
   @Override
-  public void processCounter(MetricName name, Counter counter, ReadProcessorContext context) throws Exception {
+  public void processCounter(MetricName name, Counter counter, ReadProcessorContext context) {
     counter.inc(context.getValue() == null ? 1L : Math.round(context.getValue()));
   }
 
   @Override
-  public void processHistogram(MetricName name, Histogram histogram, ReadProcessorContext context) throws Exception {
+  public void processHistogram(MetricName name, Histogram histogram, ReadProcessorContext context) {
     if (histogram instanceof WavefrontHistogram) {
       ((WavefrontHistogram) histogram).update(context.getValue());
     } else {
@@ -33,13 +31,14 @@ public void processHistogram(MetricName name, Histogram histogram, ReadProcessor
   }
 
   @Override
-  public void processTimer(MetricName name, Timer timer, ReadProcessorContext context) throws Exception {
+  public void processTimer(MetricName name, Timer timer, ReadProcessorContext context) {
     throw new UnsupportedOperationException();
   }
 
   @Override
   @SuppressWarnings("unchecked")
-  public void processGauge(MetricName name, Gauge gauge, ReadProcessorContext context) throws Exception {
+  public void processGauge(MetricName name, Gauge gauge, ReadProcessorContext context)
+      throws Exception {
     if (context.getValue() == null) {
       throw new MalformedMessageException("Need an explicit value for updating a gauge.");
     }
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/ReadProcessorContext.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/ReadProcessorContext.java
index d0394bfa0..be1b2b055 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/ReadProcessorContext.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/ReadProcessorContext.java
@@ -1,10 +1,8 @@
 package com.wavefront.agent.logsharvesting;
 
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public class ReadProcessorContext {
-  private Double value;
+  private final Double value;
 
   public ReadProcessorContext(Double value) {
     this.value = value;
diff --git a/proxy/src/main/java/com/wavefront/agent/logsharvesting/TimeSeriesUtils.java b/proxy/src/main/java/com/wavefront/agent/logsharvesting/TimeSeriesUtils.java
index 3c23bccf7..af808dfba 100644
--- a/proxy/src/main/java/com/wavefront/agent/logsharvesting/TimeSeriesUtils.java
+++ b/proxy/src/main/java/com/wavefront/agent/logsharvesting/TimeSeriesUtils.java
@@ -2,19 +2,14 @@
 
 import com.yammer.metrics.core.DeltaCounter;
 import com.yammer.metrics.core.MetricName;
-
+import java.io.IOException;
 import org.apache.avro.io.DatumReader;
 import org.apache.avro.io.Decoder;
 import org.apache.avro.io.DecoderFactory;
 import org.apache.avro.specific.SpecificDatumReader;
-
-import java.io.IOException;
-
 import wavefront.report.TimeSeries;
 
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public class TimeSeriesUtils {
 
   private static DatumReader datumReader = new SpecificDatumReader<>(TimeSeries.class);
@@ -43,5 +38,4 @@ public static TimeSeries fromMetricName(MetricName metricName) throws IOExceptio
   public static MetricName toMetricName(TimeSeries timeSeries) {
     return new MetricName("group", "type", timeSeries.toString());
   }
-
 }
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/AgentPreprocessorConfiguration.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/AgentPreprocessorConfiguration.java
deleted file mode 100644
index dc36b2e58..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/AgentPreprocessorConfiguration.java
+++ /dev/null
@@ -1,188 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Sets;
-
-import com.wavefront.common.TaggedMetricName;
-import com.yammer.metrics.Metrics;
-
-import org.apache.commons.lang.StringUtils;
-import org.yaml.snakeyaml.Yaml;
-
-import java.io.InputStream;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Logger;
-
-import javax.validation.constraints.NotNull;
-
-/**
- * Parses and stores all preprocessor rules (organized by listening port)
- *
- * Created by Vasily on 9/15/16.
- */
-public class AgentPreprocessorConfiguration {
-
-  private static final Logger logger = Logger.getLogger(AgentPreprocessorConfiguration.class.getCanonicalName());
-
-  private final Map portMap = new HashMap<>();
-
-  @VisibleForTesting
-  int totalInvalidRules = 0;
-  @VisibleForTesting
-  int totalValidRules = 0;
-
-  public ReportableEntityPreprocessor forPort(final String strPort) {
-    ReportableEntityPreprocessor preprocessor = portMap.get(strPort);
-    if (preprocessor == null) {
-      preprocessor = new ReportableEntityPreprocessor();
-      portMap.put(strPort, preprocessor);
-    }
-    return preprocessor;
-  }
-
-  private void requireArguments(@NotNull Map rule, String... arguments) {
-    if (rule == null)
-      throw new IllegalArgumentException("Rule is empty");
-    for (String argument : arguments) {
-      if (rule.get(argument) == null || rule.get(argument).replaceAll("[^a-z0-9_-]", "").isEmpty())
-        throw new IllegalArgumentException("'" + argument + "' is missing or empty");
-    }
-  }
-
-  private void allowArguments(@NotNull Map rule, String... arguments) {
-    Sets.SetView invalidArguments = Sets.difference(rule.keySet(), Sets.newHashSet(arguments));
-    if (invalidArguments.size() > 0) {
-      throw new IllegalArgumentException("Invalid or not applicable argument(s): " +
-          StringUtils.join(invalidArguments, ","));
-    }
-  }
-
-  public void loadFromStream(InputStream stream) {
-    totalValidRules = 0;
-    totalInvalidRules = 0;
-    Yaml yaml = new Yaml();
-    try {
-      //noinspection unchecked
-      Map rulesByPort = (Map) yaml.load(stream);
-      for (String strPort : rulesByPort.keySet()) {
-        int validRules = 0;
-        //noinspection unchecked
-        List> rules = (List>) rulesByPort.get(strPort);
-        for (Map rule : rules) {
-          try {
-            requireArguments(rule, "rule", "action");
-            allowArguments(rule, "rule", "action", "scope", "search", "replace", "match",
-                "tag", "newtag", "value", "source", "iterations", "replaceSource", "replaceInput");
-            String ruleName = rule.get("rule").replaceAll("[^a-z0-9_-]", "");
-            PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(
-                Metrics.newCounter(new TaggedMetricName("preprocessor." + ruleName, "count", "port", strPort)),
-                Metrics.newCounter(new TaggedMetricName("preprocessor." + ruleName, "cpu_nanos", "port", strPort)),
-                Metrics.newCounter(new TaggedMetricName("preprocessor." + ruleName, "checked.count", "port", strPort)));
-
-            if (rule.get("scope") != null && rule.get("scope").equals("pointLine")) {
-              switch (rule.get("action")) {
-                case "replaceRegex":
-                  allowArguments(rule, "rule", "action", "scope", "search", "replace", "match", "iterations");
-                  this.forPort(strPort).forPointLine().addTransformer(
-                      new PointLineReplaceRegexTransformer(rule.get("search"), rule.get("replace"), rule.get("match"),
-                          Integer.parseInt(rule.getOrDefault("iterations", "1")), ruleMetrics));
-                  break;
-                case "blacklistRegex":
-                  allowArguments(rule, "rule", "action", "scope", "match");
-                  this.forPort(strPort).forPointLine().addFilter(
-                      new PointLineBlacklistRegexFilter(rule.get("match"), ruleMetrics));
-                  break;
-                case "whitelistRegex":
-                  allowArguments(rule, "rule", "action", "scope", "match");
-                  this.forPort(strPort).forPointLine().addFilter(
-                      new PointLineWhitelistRegexFilter(rule.get("match"), ruleMetrics));
-                  break;
-                default:
-                  throw new IllegalArgumentException("Action '" + rule.get("action") +
-                      "' is not valid or cannot be applied to pointLine");
-              }
-            } else {
-              switch (rule.get("action")) {
-                case "replaceRegex":
-                  allowArguments(rule, "rule", "action", "scope", "search", "replace", "match", "iterations");
-                  this.forPort(strPort).forReportPoint().addTransformer(
-                      new ReportPointReplaceRegexTransformer(rule.get("scope"), rule.get("search"), rule.get("replace"),
-                          rule.get("match"), Integer.parseInt(rule.getOrDefault("iterations", "1")), ruleMetrics));
-                  break;
-                case "forceLowercase":
-                  allowArguments(rule, "rule", "action", "scope", "match");
-                  this.forPort(strPort).forReportPoint().addTransformer(
-                      new ReportPointForceLowercaseTransformer(rule.get("scope"), rule.get("match"), ruleMetrics));
-                  break;
-                case "addTag":
-                  allowArguments(rule, "rule", "action", "tag", "value");
-                  this.forPort(strPort).forReportPoint().addTransformer(
-                      new ReportPointAddTagTransformer(rule.get("tag"), rule.get("value"), ruleMetrics));
-                  break;
-                case "addTagIfNotExists":
-                  allowArguments(rule, "rule", "action", "tag", "value");
-                  this.forPort(strPort).forReportPoint().addTransformer(
-                      new ReportPointAddTagIfNotExistsTransformer(rule.get("tag"), rule.get("value"), ruleMetrics));
-                  break;
-                case "dropTag":
-                  allowArguments(rule, "rule", "action", "tag", "match");
-                  this.forPort(strPort).forReportPoint().addTransformer(
-                      new ReportPointDropTagTransformer(rule.get("tag"), rule.get("match"), ruleMetrics));
-                  break;
-                case "extractTag":
-                  allowArguments(rule, "rule", "action", "tag", "source", "search", "replace", "replaceSource",
-                      "replaceInput", "match");
-                  this.forPort(strPort).forReportPoint().addTransformer(
-                      new ReportPointExtractTagTransformer(rule.get("tag"), rule.get("source"), rule.get("search"),
-                          rule.get("replace"), rule.getOrDefault("replaceInput", rule.get("replaceSource")),
-                          rule.get("match"), ruleMetrics));
-                  break;
-                case "extractTagIfNotExists":
-                  allowArguments(rule, "rule", "action", "tag", "source", "search", "replace", "replaceSource",
-                      "replaceInput", "match");
-                  this.forPort(strPort).forReportPoint().addTransformer(
-                      new ReportPointExtractTagIfNotExistsTransformer(rule.get("tag"), rule.get("source"),
-                          rule.get("search"), rule.get("replace"), rule.getOrDefault("replaceInput",
-                          rule.get("replaceSource")), rule.get("match"), ruleMetrics));
-                  break;
-                case "renameTag":
-                  allowArguments(rule, "rule", "action", "tag", "newtag", "match");
-                  this.forPort(strPort).forReportPoint().addTransformer(
-                      new ReportPointRenameTagTransformer(
-                          rule.get("tag"), rule.get("newtag"), rule.get("match"), ruleMetrics));
-                  break;
-                case "blacklistRegex":
-                  allowArguments(rule, "rule", "action", "scope", "match");
-                  this.forPort(strPort).forReportPoint().addFilter(
-                      new ReportPointBlacklistRegexFilter(rule.get("scope"), rule.get("match"), ruleMetrics));
-                  break;
-                case "whitelistRegex":
-                  allowArguments(rule, "rule", "action", "scope", "match");
-                  this.forPort(strPort).forReportPoint().addFilter(
-                      new ReportPointWhitelistRegexFilter(rule.get("scope"), rule.get("match"), ruleMetrics));
-                  break;
-                default:
-                  throw new IllegalArgumentException("Action '" + rule.get("action") + "' is not valid");
-              }
-            }
-            validRules++;
-          } catch (IllegalArgumentException | NullPointerException ex) {
-            logger.warning("Invalid rule " + (rule == null || rule.get("rule") == null ? "" : rule.get("rule")) +
-                " (port " + strPort + "): " + ex);
-            totalInvalidRules++;
-          }
-        }
-        logger.info("Loaded " + validRules + " rules for port " + strPort);
-        totalValidRules += validRules;
-      }
-      logger.info("Total " + totalValidRules + " rules loaded");
-      if (totalInvalidRules > 0) {
-        throw new RuntimeException("Total " + totalInvalidRules + " invalid rules detected, aborting start-up");
-      }
-    } catch (ClassCastException e) {
-      throw new RuntimeException("Can't parse preprocessor configuration - aborting start-up");
-    }
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/AnnotatedPredicate.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/AnnotatedPredicate.java
deleted file mode 100644
index eeb32fc68..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/AnnotatedPredicate.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Predicate;
-
-import javax.annotation.Nullable;
-
-/**
- * This is the base class for all "filter"-type rules.
- *
- * Created by Vasily on 9/15/16.
- */
-public abstract class AnnotatedPredicate implements Predicate {
-
-  public abstract boolean apply(T input);
-
-  /**
-   * If special handling is needed based on the result of apply(), override getMessage() to return more details
-   */
-  @Nullable
-  public String getMessage(T input) {
-    return null;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/InteractivePreprocessorTester.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/InteractivePreprocessorTester.java
new file mode 100644
index 000000000..ed4273c09
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/preprocessor/InteractivePreprocessorTester.java
@@ -0,0 +1,166 @@
+package com.wavefront.agent.preprocessor;
+
+import com.wavefront.agent.InteractiveTester;
+import com.wavefront.agent.formatter.DataFormat;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.agent.listeners.WavefrontPortUnificationHandler;
+import com.wavefront.agent.listeners.tracing.SpanUtils;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.ingester.HistogramDecoder;
+import com.wavefront.ingester.ReportPointDecoder;
+import com.wavefront.ingester.ReportPointDecoderWrapper;
+import com.wavefront.ingester.ReportPointSerializer;
+import com.wavefront.ingester.ReportableEntityDecoder;
+import com.wavefront.ingester.SpanDecoder;
+import com.wavefront.ingester.SpanSerializer;
+import java.util.List;
+import java.util.Scanner;
+import java.util.function.Supplier;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import wavefront.report.ReportPoint;
+import wavefront.report.Span;
+
+/**
+ * Interactive tester for preprocessor rules.
+ *
+ * @author vasily@wavefront.com
+ */
+public class InteractivePreprocessorTester implements InteractiveTester {
+  private static final SpanSerializer SPAN_SERIALIZER = new SpanSerializer();
+  private static final ReportableEntityDecoder SPAN_DECODER =
+      new SpanDecoder("unknown");
+
+  private final Scanner stdin = new Scanner(System.in);
+  private final Supplier preprocessorSupplier;
+  private final ReportableEntityType entityType;
+  private final String port;
+  private final List customSourceTags;
+
+  private final ReportableEntityHandlerFactory factory =
+      new ReportableEntityHandlerFactory() {
+        @SuppressWarnings("unchecked")
+        @Override
+        public  ReportableEntityHandler getHandler(HandlerKey handlerKey) {
+          if (handlerKey.getEntityType() == ReportableEntityType.TRACE) {
+            return (ReportableEntityHandler)
+                new ReportableEntityHandler() {
+                  @Override
+                  public void report(Span reportSpan) {
+                    System.out.println(SPAN_SERIALIZER.apply(reportSpan));
+                  }
+
+                  @Override
+                  public void block(Span reportSpan) {
+                    System.out.println("Blocked: " + reportSpan);
+                  }
+
+                  @Override
+                  public void block(@Nullable Span reportSpan, @Nullable String message) {
+                    System.out.println("Blocked: " + SPAN_SERIALIZER.apply(reportSpan));
+                  }
+
+                  @Override
+                  public void reject(@Nullable Span reportSpan, @Nullable String message) {
+                    System.out.println("Rejected: " + SPAN_SERIALIZER.apply(reportSpan));
+                  }
+
+                  @Override
+                  public void reject(@Nonnull String t, @Nullable String message) {
+                    System.out.println("Rejected: " + t);
+                  }
+
+                  @Override
+                  public void setLogFormat(DataFormat format) {
+                    throw new UnsupportedOperationException();
+                  }
+
+                  @Override
+                  public void shutdown() {}
+                };
+          }
+          return (ReportableEntityHandler)
+              new ReportableEntityHandler() {
+                @Override
+                public void report(ReportPoint reportPoint) {
+                  System.out.println(ReportPointSerializer.pointToString(reportPoint));
+                }
+
+                @Override
+                public void block(ReportPoint reportPoint) {
+                  System.out.println(
+                      "Blocked: " + ReportPointSerializer.pointToString(reportPoint));
+                }
+
+                @Override
+                public void block(@Nullable ReportPoint reportPoint, @Nullable String message) {
+                  System.out.println(
+                      "Blocked: " + ReportPointSerializer.pointToString(reportPoint));
+                }
+
+                @Override
+                public void reject(@Nullable ReportPoint reportPoint, @Nullable String message) {
+                  System.out.println(
+                      "Rejected: " + ReportPointSerializer.pointToString(reportPoint));
+                }
+
+                @Override
+                public void reject(@Nonnull String t, @Nullable String message) {
+                  System.out.println("Rejected: " + t);
+                }
+
+                @Override
+                public void setLogFormat(DataFormat format) {
+                  throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public void shutdown() {}
+              };
+        }
+
+        @Override
+        public void shutdown(@Nonnull String handle) {}
+      };
+
+  /**
+   * @param preprocessorSupplier supplier for {@link ReportableEntityPreprocessor}
+   * @param entityType entity type (to determine whether it's a span or a point)
+   * @param port handler key
+   * @param customSourceTags list of custom source tags (for parsing)
+   */
+  public InteractivePreprocessorTester(
+      Supplier preprocessorSupplier,
+      ReportableEntityType entityType,
+      String port,
+      List customSourceTags) {
+    this.preprocessorSupplier = preprocessorSupplier;
+    this.entityType = entityType;
+    this.port = port;
+    this.customSourceTags = customSourceTags;
+  }
+
+  @Override
+  public boolean interactiveTest() {
+    String line = stdin.nextLine();
+    if (entityType == ReportableEntityType.TRACE) {
+      ReportableEntityHandler handler = factory.getHandler(entityType, port);
+      SpanUtils.preprocessAndHandleSpan(
+          line, SPAN_DECODER, handler, handler::report, preprocessorSupplier, null, x -> true);
+    } else {
+      ReportableEntityHandler handler = factory.getHandler(entityType, port);
+      ReportableEntityDecoder decoder;
+      if (DataFormat.autodetect(line) == DataFormat.HISTOGRAM) {
+        decoder = new ReportPointDecoderWrapper(new HistogramDecoder());
+      } else {
+        decoder = new ReportPointDecoder(() -> "unknown", customSourceTags);
+      }
+      WavefrontPortUnificationHandler.preprocessAndHandlePoint(
+          line, decoder, handler, preprocessorSupplier, null, "");
+    }
+    return stdin.hasNext();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/PointLineBlacklistRegexFilter.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/PointLineBlacklistRegexFilter.java
deleted file mode 100644
index 73f29138b..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/PointLineBlacklistRegexFilter.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Preconditions;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-
-/**
- * Blacklist regex filter. Reject a point line if it matches the regex
- *
- * Created by Vasily on 9/13/16.
- */
-public class PointLineBlacklistRegexFilter extends AnnotatedPredicate {
-
-  private final Pattern compiledPattern;
-  private final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public PointLineBlacklistRegexFilter(final String patternMatch,
-                                       @Nullable final Counter ruleAppliedCounter) {
-    this(patternMatch, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public PointLineBlacklistRegexFilter(final String patternMatch,
-                                       final PreprocessorRuleMetrics ruleMetrics) {
-    this.compiledPattern = Pattern.compile(Preconditions.checkNotNull(patternMatch, "[match] can't be null"));
-    Preconditions.checkArgument(!patternMatch.isEmpty(), "[match] can't be blank");
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  @Override
-  public boolean apply(String pointLine) {
-    long startNanos = ruleMetrics.ruleStart();
-    if (compiledPattern.matcher(pointLine).matches()) {
-      ruleMetrics.incrementRuleAppliedCounter();
-      ruleMetrics.ruleEnd(startNanos);
-      return false;
-    }
-    ruleMetrics.ruleEnd(startNanos);
-    return true;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/PointLineReplaceRegexTransformer.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/PointLineReplaceRegexTransformer.java
deleted file mode 100644
index 397a13dba..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/PointLineReplaceRegexTransformer.java
+++ /dev/null
@@ -1,81 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-
-/**
- * Replace regex transformer. Performs search and replace on an entire point line string
- *
- * Created by Vasily on 9/13/16.
- */
-public class PointLineReplaceRegexTransformer implements Function {
-
-  private final String patternReplace;
-  private final Pattern compiledSearchPattern;
-  private final Integer maxIterations;
-  @Nullable
-  private final Pattern compiledMatchPattern;
-  private final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public PointLineReplaceRegexTransformer(final String patternSearch,
-                                          final String patternReplace,
-                                          @Nullable final String patternMatch,
-                                          @Nullable final Integer maxIterations,
-                                          @Nullable final Counter ruleAppliedCounter) {
-    this(patternSearch, patternReplace, patternMatch, maxIterations, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public PointLineReplaceRegexTransformer(final String patternSearch,
-                                          final String patternReplace,
-                                          @Nullable final String patternMatch,
-                                          @Nullable final Integer maxIterations,
-                                          final PreprocessorRuleMetrics ruleMetrics) {
-    this.compiledSearchPattern = Pattern.compile(Preconditions.checkNotNull(patternSearch, "[search] can't be null"));
-    Preconditions.checkArgument(!patternSearch.isEmpty(), "[search] can't be blank");
-    this.compiledMatchPattern = patternMatch != null ? Pattern.compile(patternMatch) : null;
-    this.patternReplace = Preconditions.checkNotNull(patternReplace, "[replace] can't be null");
-    this.maxIterations = maxIterations != null ? maxIterations : 1;
-    Preconditions.checkArgument(this.maxIterations > 0, "[iterations] must be > 0");
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  @Override
-  public String apply(String pointLine) {
-    long startNanos = ruleMetrics.ruleStart();
-    if (compiledMatchPattern != null && !compiledMatchPattern.matcher(pointLine).matches()) {
-      ruleMetrics.ruleEnd(startNanos);
-      return pointLine;
-    }
-    Matcher patternMatcher = compiledSearchPattern.matcher(pointLine);
-
-    if (!patternMatcher.find()) {
-      ruleMetrics.ruleEnd(startNanos);
-      return pointLine;
-    }
-    ruleMetrics.incrementRuleAppliedCounter(); // count the rule only once regardless of the number of iterations
-
-    int currentIteration = 0;
-    while (true) {
-      pointLine = patternMatcher.replaceAll(patternReplace);
-      currentIteration++;
-      if (currentIteration >= maxIterations) {
-        break;
-      }
-      patternMatcher = compiledSearchPattern.matcher(pointLine);
-      if (!patternMatcher.find()) {
-        break;
-      }
-    }
-    ruleMetrics.ruleEnd(startNanos);
-    return pointLine;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/PointLineWhitelistRegexFilter.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/PointLineWhitelistRegexFilter.java
deleted file mode 100644
index f11509714..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/PointLineWhitelistRegexFilter.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Preconditions;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-
-/**
- * Whitelist regex filter. Reject a point line if it doesn't match the regex
- *
- * Created by Vasily on 9/13/16.
- */
-public class PointLineWhitelistRegexFilter extends AnnotatedPredicate {
-
-  private final Pattern compiledPattern;
-  private final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public PointLineWhitelistRegexFilter(final String patternMatch,
-                                       @Nullable final Counter ruleAppliedCounter) {
-    this(patternMatch, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public PointLineWhitelistRegexFilter(final String patternMatch,
-                                       final PreprocessorRuleMetrics ruleMetrics) {
-    this.compiledPattern = Pattern.compile(Preconditions.checkNotNull(patternMatch, "[match] can't be null"));
-    Preconditions.checkArgument(!patternMatch.isEmpty(), "[match] can't be blank");
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  @Override
-  public boolean apply(String pointLine) {
-  long startNanos = ruleMetrics.ruleStart();
-    if (!compiledPattern.matcher(pointLine).matches()) {
-      ruleMetrics.incrementRuleAppliedCounter();
-      ruleMetrics.ruleEnd(startNanos);
-      return false;
-    }
-    ruleMetrics.ruleEnd(startNanos);
-    return true;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/Preprocessor.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/Preprocessor.java
deleted file mode 100644
index 7a8ae0fcf..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/Preprocessor.java
+++ /dev/null
@@ -1,110 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-/**
- * Generic container class for storing transformation and filter rules
- *
- * Created by Vasily on 9/13/16.
- */
-public class Preprocessor {
-
-  private final List> transformers = new ArrayList<>();
-  private final List> filters = new ArrayList<>();
-
-  @Nullable
-  private String message;
-
-  /**
-   * Apply all transformation rules sequentially
-   * @param item input point
-   * @return transformed point
-   */
-  public T transform(@NotNull T item) {
-    for (final Function func : transformers) {
-      item = func.apply(item);
-    }
-    return item;
-  }
-
-  /**
-   * Apply all filter predicates sequentially, stop at the first "false" result
-   * @param item item to apply predicates to
-   * @return true if all predicates returned "true"
-   */
-  public boolean filter(@NotNull T item) {
-    message = null;
-    for (final AnnotatedPredicate predicate : filters) {
-      if (!predicate.apply(item)) {
-        message = predicate.getMessage(item);
-        return false;
-      }
-    }
-    return true;
-  }
-
-  /**
-   * Check if any filters are registered
-   * @return true if it has at least one filter
-   */
-  public boolean hasFilters() {
-    return !filters.isEmpty();
-  }
-
-  /**
-   * Check if any transformation rules are registered
-   * @return true if it has at least one transformer
-   */
-  public boolean hasTransformers() {
-    return !transformers.isEmpty();
-  }
-
-  /**
-   * Get the detailed message, if available, with the result of the last filter() operation
-   * @return message
-   */
-  @Nullable
-  public String getLastFilterResult() {
-    return message;
-  }
-
-  /**
-   * Register a transformation rule
-   * @param transformer rule
-   */
-  public void addTransformer(Function transformer) {
-    transformers.add(transformer);
-  }
-
-  /**
-   * Register a filter rule
-   * @param filter rule
-   */
-  public void addFilter(AnnotatedPredicate filter) {
-    filters.add(filter);
-  }
-
-  /**
-   * Register a transformation rule and place it at a specific index
-   * @param index zero-based index
-   * @param transformer rule
-   */
-  public void addTransformer(int index, Function transformer) {
-    transformers.add(index, transformer);
-  }
-
-  /**
-   * Register a filter rule and place it at a specific index
-   * @param index zero-based index
-   * @param filter rule
-   */
-  public void addFilter(int index, AnnotatedPredicate filter) {
-    filters.add(index, filter);
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/PreprocessorRuleMetrics.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/PreprocessorRuleMetrics.java
deleted file mode 100644
index 4e296d619..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/PreprocessorRuleMetrics.java
+++ /dev/null
@@ -1,83 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.yammer.metrics.core.Counter;
-
-import javax.annotation.Nullable;
-
-/**
- * A helper class for instrumenting preprocessor rules.
- * Tracks two counters: number of times the rule has been successfully applied, and counter of CPU time (nanos)
- * spent on applying the rule to troubleshoot possible performance issues.
- *
- * @author vasily@wavefront.com
- */
-public class PreprocessorRuleMetrics {
-  @Nullable
-  private final Counter ruleAppliedCounter;
-  @Nullable
-  private final Counter ruleCpuTimeNanosCounter;
-  @Nullable
-  private final Counter ruleCheckedCounter;
-
-  public PreprocessorRuleMetrics(@Nullable Counter ruleAppliedCounter, @Nullable Counter ruleCpuTimeNanosCounter,
-                                 @Nullable Counter ruleCheckedCounter) {
-    this.ruleAppliedCounter = ruleAppliedCounter;
-    this.ruleCpuTimeNanosCounter = ruleCpuTimeNanosCounter;
-    this.ruleCheckedCounter = ruleCheckedCounter;
-  }
-
-  @Deprecated
-  public PreprocessorRuleMetrics(@Nullable Counter ruleAppliedCounter, @Nullable Counter ruleCpuTimeNanosCounter) {
-    this(ruleAppliedCounter, ruleCpuTimeNanosCounter, null);
-  }
-
-  @Deprecated
-  public PreprocessorRuleMetrics(@Nullable Counter ruleAppliedCounter) {
-    this(ruleAppliedCounter, null);
-  }
-
-  /**
-   * Increment ruleAppliedCounter (if available) by 1
-   */
-  public void incrementRuleAppliedCounter() {
-    if (this.ruleAppliedCounter != null) {
-      this.ruleAppliedCounter.inc();
-    }
-  }
-
-  /**
-   * Increment ruleCpuTimeNanosCounter (if available) by {@code n}
-   *
-   * @param n the amount by which the counter will be increased
-   */
-  @Deprecated
-  public void countCpuNanos(long n) {
-    if (this.ruleCpuTimeNanosCounter != null) {
-      this.ruleCpuTimeNanosCounter.inc(n);
-    }
-  }
-
-  /**
-   * Measure rule execution time and add it to ruleCpuTimeNanosCounter (if available)
-   *
-   * @param ruleStartTime rule start time
-   */
-  public void ruleEnd(long ruleStartTime) {
-    if (this.ruleCpuTimeNanosCounter != null) {
-      this.ruleCpuTimeNanosCounter.inc(System.nanoTime() - ruleStartTime);
-    }
-  }
-
-
-  /**
-   * Mark rule start time, increment ruleCheckedCounter (if available) by 1
-   *
-   * @return start time in nanos
-   */
-  public long ruleStart() {
-    if (this.ruleCheckedCounter != null) {
-      this.ruleCheckedCounter.inc();
-    }
-    return System.nanoTime();
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/PreprocessorUtil.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/PreprocessorUtil.java
deleted file mode 100644
index 22a666db1..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/PreprocessorUtil.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Utility class for methods used by preprocessors
- *
- * @author vasily@wavefront.com
- */
-public abstract class PreprocessorUtil {
-
-  /**
-   * Substitute {{...}} placeholders with corresponding components of the point
-   * {{metricName}} {{sourceName}} are replaced with the metric name and source respectively
-   * {{anyTagK}} is replaced with the value of the anyTagK point tag
-   *
-   * @param input        input string with {{...}} placeholders
-   * @param reportPoint  ReportPoint object to extract components from
-   * @return string with substituted placeholders
-   */
-  public static String expandPlaceholders(String input, @NotNull ReportPoint reportPoint) {
-    if (input.contains("{{")) {
-      StringBuffer result = new StringBuffer();
-      Matcher placeholders = Pattern.compile("\\{\\{(.*?)}}").matcher(input);
-      while (placeholders.find()) {
-        if (placeholders.group(1).isEmpty()) {
-          placeholders.appendReplacement(result, placeholders.group(0));
-        } else {
-          String substitution;
-          switch (placeholders.group(1)) {
-            case "metricName":
-              substitution = reportPoint.getMetric();
-              break;
-            case "sourceName":
-              substitution = reportPoint.getHost();
-              break;
-            default:
-              substitution = reportPoint.getAnnotations().get(placeholders.group(1));
-          }
-          if (substitution != null) {
-            placeholders.appendReplacement(result, substitution);
-          } else {
-            placeholders.appendReplacement(result, placeholders.group(0));
-          }
-        }
-      }
-      placeholders.appendTail(result);
-      return result.toString();
-    }
-    return input;
-  }
-
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ProxyPreprocessorConfigManager.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ProxyPreprocessorConfigManager.java
new file mode 100644
index 000000000..a644c239f
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/preprocessor/ProxyPreprocessorConfigManager.java
@@ -0,0 +1,127 @@
+package com.wavefront.agent.preprocessor;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.wavefront.agent.ProxyCheckInScheduler;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import com.wavefront.api.agent.preprocessor.PreprocessorConfigManager;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+
+/**
+ * Extends the PreprocessorConfigManager in java-lib, which parses preprocessor rules (organized by listening port)
+ *
+ * 
Created by Vasily on 9/15/16.
+ */
+public class ProxyPreprocessorConfigManager extends PreprocessorConfigManager {
+  private static final Logger logger =
+      Logger.getLogger(ProxyPreprocessorConfigManager.class.getCanonicalName());
+  private static final Counter configReloads =
+      Metrics.newCounter(new MetricName("preprocessor", "", "config-reloads.successful"));
+  private static final Counter failedConfigReloads =
+      Metrics.newCounter(new MetricName("preprocessor", "", "config-reloads.failed"));
+
+  private final Supplier timeSupplier;
+
+  @VisibleForTesting public Map userPreprocessors;
+
+  private volatile long userPreprocessorsTs;
+  private static String proxyConfigRules;
+
+  public ProxyPreprocessorConfigManager() {
+    this(System::currentTimeMillis);
+  }
+
+  /** @param timeSupplier Supplier for current time (in millis). */
+  @VisibleForTesting
+  ProxyPreprocessorConfigManager(@Nonnull Supplier timeSupplier) {
+    this.timeSupplier = timeSupplier;
+    userPreprocessorsTs = timeSupplier.get();
+    userPreprocessors = Collections.emptyMap();
+  }
+
+  /**
+   * Schedules periodic checks for config file modification timestamp and performs hot-reload
+   *
+   * @param fileName Path name of the file to be monitored.
+   *
+   * @param fileCheckIntervalMillis Timestamp check interval.
+   */
+  public void setUpConfigFileMonitoring(String fileName, int fileCheckIntervalMillis) {
+    new Timer("Timer-preprocessor-configmanager")
+        .schedule(
+            new TimerTask() {
+              @Override
+              public void run() {
+                loadFileIfModified(fileName);
+              }
+            },
+            fileCheckIntervalMillis,
+            fileCheckIntervalMillis);
+  }
+
+  @VisibleForTesting
+  void loadFileIfModified(String fileName) {
+    // skip loading file completely if rules are already set from FE
+    if (ProxyCheckInScheduler.isRulesSetInFE.get()) return;
+    try {
+      File file = new File(fileName);
+      long lastModified = file.lastModified();
+      if (lastModified > userPreprocessorsTs) {
+        logger.info("File " + file + " has been modified on disk, reloading preprocessor rules");
+        loadFile(fileName);
+        configReloads.inc();
+      }
+    } catch (Exception e) {
+      logger.log(Level.SEVERE, "Unable to load preprocessor rules", e);
+      failedConfigReloads.inc();
+    }
+  }
+
+  public void loadFile(String filename) throws FileNotFoundException {
+    File file = new File(filename);
+    super.loadFromStream(new FileInputStream(file));
+    proxyConfigRules = getFileRules(filename);
+    ProxyCheckInScheduler.preprocessorRulesNeedUpdate.set(true);
+  }
+
+  public void loadFERules(String rules) {
+    logger.info("New preprocessor rules detected! Loading preprocessor rules from FE Configuration");
+    InputStream is = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8));
+    super.loadFromStream(is);
+    proxyConfigRules = rules;
+    ProxyCheckInScheduler.preprocessorRulesNeedUpdate.set(true);
+  }
+
+  public static String getProxyConfigRules() {
+    return proxyConfigRules;
+  }
+
+  public static String getFileRules(String filename) {
+    try {
+      if (filename == null || filename.isEmpty()) return null;
+      return new String(
+              Files.readAllBytes(Paths.get(filename)),
+              StandardCharsets.UTF_8
+      );
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to read file rules as string", e);
+    }
+  }
+
+  // For testing purposes
+  @VisibleForTesting
+  public static void clearProxyConfigRules() {
+    proxyConfigRules = null;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointAddPrefixTransformer.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointAddPrefixTransformer.java
deleted file mode 100644
index e16473551..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointAddPrefixTransformer.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Add prefix transformer. Add a metric name prefix, if defined, to all points.
- *
- * Created by Vasily on 9/15/16.
- */
-public class ReportPointAddPrefixTransformer implements Function {
-
-  @Nullable
-  private final String prefix;
-
-  public ReportPointAddPrefixTransformer(@Nullable final String prefix) {
-    this.prefix = prefix;
-  }
-
-  @Override
-  public ReportPoint apply(@NotNull ReportPoint reportPoint) {
-    if (prefix != null && !prefix.isEmpty()) {
-      reportPoint.setMetric(prefix + "." + reportPoint.getMetric());
-    }
-    return reportPoint;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointAddTagIfNotExistsTransformer.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointAddTagIfNotExistsTransformer.java
deleted file mode 100644
index fa6cf1489..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointAddTagIfNotExistsTransformer.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-import com.yammer.metrics.core.Counter;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Creates a new point tag with a specified value. If such point tag already exists, the value won't be overwritten.
- *
- * Created by Vasily on 9/13/16.
- */
-public class ReportPointAddTagIfNotExistsTransformer extends ReportPointAddTagTransformer {
-
-  @Deprecated
-  public ReportPointAddTagIfNotExistsTransformer(final String tag,
-                                                 final String value,
-                                                 @Nullable final Counter ruleAppliedCounter) {
-    this(tag, value, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public ReportPointAddTagIfNotExistsTransformer(final String tag,
-                                                 final String value,
-                                                 final PreprocessorRuleMetrics ruleMetrics) {
-    super(tag, value, ruleMetrics);
-  }
-
-  @Override
-  public ReportPoint apply(@NotNull ReportPoint reportPoint) {
-    long startNanos = ruleMetrics.ruleStart();
-    if (reportPoint.getAnnotations() == null) {
-      reportPoint.setAnnotations(Maps.newHashMap());
-    }
-    if (reportPoint.getAnnotations().get(tag) == null) {
-      reportPoint.getAnnotations().put(tag, PreprocessorUtil.expandPlaceholders(value, reportPoint));
-      ruleMetrics.incrementRuleAppliedCounter();
-    }
-    ruleMetrics.ruleEnd(startNanos);
-    return reportPoint;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointAddTagTransformer.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointAddTagTransformer.java
deleted file mode 100644
index b78009ab9..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointAddTagTransformer.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-import com.yammer.metrics.core.Counter;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Creates a new point tag with a specified value, or overwrite an existing one.
- *
- * Created by Vasily on 9/13/16.
- */
-public class ReportPointAddTagTransformer implements Function {
-
-  protected final String tag;
-  protected final String value;
-  protected final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public ReportPointAddTagTransformer(final String tag,
-                                      final String value,
-                                      @Nullable final Counter ruleAppliedCounter) {
-    this(tag, value, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public ReportPointAddTagTransformer(final String tag,
-                                      final String value,
-                                      final PreprocessorRuleMetrics ruleMetrics) {
-    this.tag = Preconditions.checkNotNull(tag, "[tag] can't be null");
-    this.value = Preconditions.checkNotNull(value, "[value] can't be null");
-    Preconditions.checkArgument(!tag.isEmpty(), "[tag] can't be blank");
-    Preconditions.checkArgument(!value.isEmpty(), "[value] can't be blank");
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  @Override
-  public ReportPoint apply(@NotNull ReportPoint reportPoint) {
-    long startNanos = ruleMetrics.ruleStart();
-    if (reportPoint.getAnnotations() == null) {
-      reportPoint.setAnnotations(Maps.newHashMap());
-    }
-    reportPoint.getAnnotations().put(tag, PreprocessorUtil.expandPlaceholders(value, reportPoint));
-    ruleMetrics.incrementRuleAppliedCounter();
-    ruleMetrics.ruleEnd(startNanos);
-    return reportPoint;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointBlacklistRegexFilter.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointBlacklistRegexFilter.java
deleted file mode 100644
index 7c599c1df..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointBlacklistRegexFilter.java
+++ /dev/null
@@ -1,77 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Preconditions;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Blacklist regex filter. Rejects a point if a specified component (metric, source, or point tag value, depending
- * on the "scope" parameter) doesn't match the regex.
- *
- * Created by Vasily on 9/13/16.
- */
-public class ReportPointBlacklistRegexFilter extends AnnotatedPredicate {
-
-  private final String scope;
-  private final Pattern compiledPattern;
-  private final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public ReportPointBlacklistRegexFilter(final String scope,
-                                         final String patternMatch,
-                                         @Nullable final Counter ruleAppliedCounter) {
-    this(scope, patternMatch, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public ReportPointBlacklistRegexFilter(final String scope,
-                                         final String patternMatch,
-                                         final PreprocessorRuleMetrics ruleMetrics) {
-    this.compiledPattern = Pattern.compile(Preconditions.checkNotNull(patternMatch, "[match] can't be null"));
-    Preconditions.checkArgument(!patternMatch.isEmpty(), "[match] can't be blank");
-    this.scope = Preconditions.checkNotNull(scope, "[scope] can't be null");
-    Preconditions.checkArgument(!scope.isEmpty(), "[scope] can't be blank");
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  @Override
-  public boolean apply(@NotNull ReportPoint reportPoint) {
-    long startNanos = ruleMetrics.ruleStart();
-    switch (scope) {
-      case "metricName":
-        if (compiledPattern.matcher(reportPoint.getMetric()).matches()) {
-          ruleMetrics.incrementRuleAppliedCounter();
-          ruleMetrics.ruleEnd(startNanos);
-          return false;
-        }
-        break;
-      case "sourceName":
-        if (compiledPattern.matcher(reportPoint.getHost()).matches()) {
-          ruleMetrics.incrementRuleAppliedCounter();
-          ruleMetrics.ruleEnd(startNanos);
-          return false;
-        }
-        break;
-      default:
-        if (reportPoint.getAnnotations() != null) {
-          String tagValue = reportPoint.getAnnotations().get(scope);
-          if (tagValue != null) {
-            if (compiledPattern.matcher(tagValue).matches()) {
-              ruleMetrics.incrementRuleAppliedCounter();
-              ruleMetrics.ruleEnd(startNanos);
-              return false;
-            }
-          }
-        }
-    }
-    ruleMetrics.ruleEnd(startNanos);
-    return true;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointDropTagTransformer.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointDropTagTransformer.java
deleted file mode 100644
index b0ffe40e0..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointDropTagTransformer.java
+++ /dev/null
@@ -1,67 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Removes a point tag if its value matches an optional regex pattern (always remove if null)
- *
- * Created by Vasily on 9/13/16.
- */
-public class ReportPointDropTagTransformer implements Function {
-
-  @Nullable
-  private final Pattern compiledTagPattern;
-  @Nullable
-  private final Pattern compiledValuePattern;
-  private final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public ReportPointDropTagTransformer(final String tag,
-                                       @Nullable final String patternMatch,
-                                       @Nullable final Counter ruleAppliedCounter) {
-    this(tag, patternMatch, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public ReportPointDropTagTransformer(final String tag,
-                                       @Nullable final String patternMatch,
-                                       final PreprocessorRuleMetrics ruleMetrics) {
-    this.compiledTagPattern = Pattern.compile(Preconditions.checkNotNull(tag, "[tag] can't be null"));
-    Preconditions.checkArgument(!tag.isEmpty(), "[tag] can't be blank");
-    this.compiledValuePattern = patternMatch != null ? Pattern.compile(patternMatch) : null;
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  @Override
-  public ReportPoint apply(@NotNull ReportPoint reportPoint) {
-    long startNanos = ruleMetrics.ruleStart();
-    if (reportPoint.getAnnotations() == null || compiledTagPattern == null) {
-      ruleMetrics.ruleEnd(startNanos);
-      return reportPoint;
-    }
-    Iterator> iterator = reportPoint.getAnnotations().entrySet().iterator();
-    while (iterator.hasNext()) {
-      Map.Entry entry = iterator.next();
-      if (compiledTagPattern.matcher(entry.getKey()).matches()) {
-        if (compiledValuePattern == null || compiledValuePattern.matcher(entry.getValue()).matches()) {
-          iterator.remove();
-          ruleMetrics.incrementRuleAppliedCounter();
-        }
-      }
-    }
-    ruleMetrics.ruleEnd(startNanos);
-    return reportPoint;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointExtractTagIfNotExistsTransformer.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointExtractTagIfNotExistsTransformer.java
deleted file mode 100644
index ad3645851..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointExtractTagIfNotExistsTransformer.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Create a point tag by extracting a portion of a metric name, source name or another point tag.
- * If such point tag already exists, the value won't be overwritten.
- *
- * @author vasily@wavefront.com
- * Created 5/18/18
- */
-public class ReportPointExtractTagIfNotExistsTransformer extends ReportPointExtractTagTransformer {
-
-  @Deprecated
-  public ReportPointExtractTagIfNotExistsTransformer(final String tag,
-                                                     final String source,
-                                                     final String patternSearch,
-                                                     final String patternReplace,
-                                                     @Nullable final String patternMatch,
-                                                     @Nullable final Counter ruleAppliedCounter) {
-    this(tag, source, patternSearch, patternReplace, null, patternMatch,
-        new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public ReportPointExtractTagIfNotExistsTransformer(final String tag,
-                                                     final String source,
-                                                     final String patternSearch,
-                                                     final String patternReplace,
-                                                     @Nullable final String replaceSource,
-                                                     @Nullable final String patternMatch,
-                                                     final PreprocessorRuleMetrics ruleMetrics) {
-    super(tag, source, patternSearch, patternReplace, replaceSource, patternMatch, ruleMetrics);
-  }
-
-  @Override
-  public ReportPoint apply(@NotNull ReportPoint reportPoint) {
-    long startNanos = ruleMetrics.ruleStart();
-    if (reportPoint.getAnnotations() == null) {
-      reportPoint.setAnnotations(Maps.newHashMap());
-    }
-    if (reportPoint.getAnnotations().get(tag) == null) {
-      internalApply(reportPoint);
-    }
-    ruleMetrics.ruleEnd(startNanos);
-    return reportPoint;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointExtractTagTransformer.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointExtractTagTransformer.java
deleted file mode 100644
index 5ac540569..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointExtractTagTransformer.java
+++ /dev/null
@@ -1,120 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Create a point tag by extracting a portion of a metric name, source name or another point tag
- *
- * Created by Vasily on 11/15/16.
- */
-public class ReportPointExtractTagTransformer implements Function{
-
-  protected final String tag;
-  protected final String source;
-  protected final String patternReplace;
-  protected final Pattern compiledSearchPattern;
-  @Nullable
-  protected final Pattern compiledMatchPattern;
-  @Nullable
-  protected final String patternReplaceSource;
-  protected final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public ReportPointExtractTagTransformer(final String tag,
-                                          final String source,
-                                          final String patternSearch,
-                                          final String patternReplace,
-                                          @Nullable final String patternMatch,
-                                          @Nullable final Counter ruleAppliedCounter) {
-    this(tag, source, patternSearch, patternReplace, null, patternMatch,
-        new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public ReportPointExtractTagTransformer(final String tag,
-                                          final String source,
-                                          final String patternSearch,
-                                          final String patternReplace,
-                                          @Nullable final String replaceSource,
-                                          @Nullable final String patternMatch,
-                                          final PreprocessorRuleMetrics ruleMetrics) {
-    this.tag = Preconditions.checkNotNull(tag, "[tag] can't be null");
-    this.source = Preconditions.checkNotNull(source, "[source] can't be null");
-    this.compiledSearchPattern = Pattern.compile(Preconditions.checkNotNull(patternSearch, "[search] can't be null"));
-    this.patternReplace = Preconditions.checkNotNull(patternReplace, "[replace] can't be null");
-    Preconditions.checkArgument(!tag.isEmpty(), "[tag] can't be blank");
-    Preconditions.checkArgument(!source.isEmpty(), "[source] can't be blank");
-    Preconditions.checkArgument(!patternSearch.isEmpty(), "[search] can't be blank");
-    this.compiledMatchPattern = patternMatch != null ? Pattern.compile(patternMatch) : null;
-    this.patternReplaceSource = replaceSource;
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  protected boolean extractTag(@NotNull ReportPoint reportPoint, final String extractFrom) {
-    Matcher patternMatcher;
-    if (extractFrom == null || (compiledMatchPattern != null && !compiledMatchPattern.matcher(extractFrom).matches())) {
-      return false;
-    }
-    patternMatcher = compiledSearchPattern.matcher(extractFrom);
-    if (!patternMatcher.find()) {
-      return false;
-    }
-    if (reportPoint.getAnnotations() == null) {
-      reportPoint.setAnnotations(Maps.newHashMap());
-    }
-    String value = patternMatcher.replaceAll(PreprocessorUtil.expandPlaceholders(patternReplace, reportPoint));
-    if (!value.isEmpty()) {
-      reportPoint.getAnnotations().put(tag, value);
-      ruleMetrics.incrementRuleAppliedCounter();
-    }
-    return true;
-  }
-
-  protected void internalApply(@NotNull ReportPoint reportPoint) {
-    switch (source) {
-      case "metricName":
-        if (extractTag(reportPoint, reportPoint.getMetric()) && patternReplaceSource != null) {
-          reportPoint.setMetric(compiledSearchPattern.matcher(reportPoint.getMetric()).
-              replaceAll(PreprocessorUtil.expandPlaceholders(patternReplaceSource, reportPoint)));
-        }
-        break;
-      case "sourceName":
-        if (extractTag(reportPoint, reportPoint.getHost()) && patternReplaceSource != null) {
-          reportPoint.setHost(compiledSearchPattern.matcher(reportPoint.getHost()).
-              replaceAll(PreprocessorUtil.expandPlaceholders(patternReplaceSource, reportPoint)));
-        }
-        break;
-      default:
-        if (reportPoint.getAnnotations() != null && reportPoint.getAnnotations().get(source) != null) {
-          if (extractTag(reportPoint, reportPoint.getAnnotations().get(source)) && patternReplaceSource != null) {
-            reportPoint.getAnnotations().put(source,
-                compiledSearchPattern.matcher(reportPoint.getAnnotations().get(source)).
-                    replaceAll(PreprocessorUtil.expandPlaceholders(patternReplaceSource, reportPoint)));
-          }
-        }
-    }
-  }
-
-  @Override
-  public ReportPoint apply(@NotNull ReportPoint reportPoint) {
-    long startNanos = ruleMetrics.ruleStart();
-    if (reportPoint.getAnnotations() == null) {
-      reportPoint.setAnnotations(Maps.newHashMap());
-    }
-    internalApply(reportPoint);
-    ruleMetrics.ruleEnd(startNanos);
-    return reportPoint;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointForceLowercaseTransformer.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointForceLowercaseTransformer.java
deleted file mode 100644
index ae052e3db..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointForceLowercaseTransformer.java
+++ /dev/null
@@ -1,78 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Force lowercase transformer. Converts a specified component of a point (metric name, source name or a point tag
- * value, depending on "scope" parameter) to lower case to enforce consistency.
- *
- * @author vasily@wavefront.com
- */
-public class ReportPointForceLowercaseTransformer implements Function {
-
-  private final String scope;
-  @Nullable
-  private final Pattern compiledMatchPattern;
-  private final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public ReportPointForceLowercaseTransformer(final String scope,
-                                              @Nullable final String patternMatch,
-                                              @Nullable final Counter ruleAppliedCounter) {
-    this(scope, patternMatch, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public ReportPointForceLowercaseTransformer(final String scope,
-                                              @Nullable final String patternMatch,
-                                              final PreprocessorRuleMetrics ruleMetrics) {
-    this.scope = Preconditions.checkNotNull(scope, "[scope] can't be null");
-    Preconditions.checkArgument(!scope.isEmpty(), "[scope] can't be blank");
-    this.compiledMatchPattern = patternMatch != null ? Pattern.compile(patternMatch) : null;
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  @Override
-  public ReportPoint apply(@NotNull ReportPoint reportPoint) {
-    long startNanos = ruleMetrics.ruleStart();
-    switch (scope) {
-      case "metricName":
-        if (compiledMatchPattern != null && !compiledMatchPattern.matcher(reportPoint.getMetric()).matches()) {
-          break;
-        }
-        reportPoint.setMetric(reportPoint.getMetric().toLowerCase());
-        ruleMetrics.incrementRuleAppliedCounter();
-        break;
-      case "sourceName": // source name is not case sensitive in Wavefront, but we'll do it anyway
-        if (compiledMatchPattern != null && !compiledMatchPattern.matcher(reportPoint.getHost()).matches()) {
-          break;
-        }
-        reportPoint.setHost(reportPoint.getHost().toLowerCase());
-        ruleMetrics.incrementRuleAppliedCounter();
-        break;
-      default:
-        if (reportPoint.getAnnotations() != null) {
-          String tagValue = reportPoint.getAnnotations().get(scope);
-          if (tagValue != null) {
-            if (compiledMatchPattern != null && !compiledMatchPattern.matcher(tagValue).matches()) {
-              break;
-            }
-            reportPoint.getAnnotations().put(scope, tagValue.toLowerCase());
-            ruleMetrics.incrementRuleAppliedCounter();
-          }
-        }
-    }
-    ruleMetrics.ruleEnd(startNanos);
-    return reportPoint;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointRenameTagTransformer.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointRenameTagTransformer.java
deleted file mode 100644
index 74d9f5068..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointRenameTagTransformer.java
+++ /dev/null
@@ -1,67 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Rename a point tag (optional: if its value matches a regex pattern)
- *
- * Created by Vasily on 9/13/16.
- */
-public class ReportPointRenameTagTransformer implements Function {
-
-  private final String tag;
-  private final String newTag;
-  @Nullable
-  private final Pattern compiledPattern;
-  private final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public ReportPointRenameTagTransformer(final String tag,
-                                         final String newTag,
-                                         @Nullable final String patternMatch,
-                                         @Nullable final Counter ruleAppliedCounter) {
-    this(tag, newTag, patternMatch, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public ReportPointRenameTagTransformer(final String tag,
-                                         final String newTag,
-                                         @Nullable final String patternMatch,
-                                         final PreprocessorRuleMetrics ruleMetrics) {
-    this.tag = Preconditions.checkNotNull(tag, "[tag] can't be null");
-    this.newTag = Preconditions.checkNotNull(newTag, "[newtag] can't be null");
-    Preconditions.checkArgument(!tag.isEmpty(), "[tag] can't be blank");
-    Preconditions.checkArgument(!newTag.isEmpty(), "[newtag] can't be blank");
-    this.compiledPattern = patternMatch != null ? Pattern.compile(patternMatch) : null;
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  @Override
-  public ReportPoint apply(@NotNull ReportPoint reportPoint) {
-    long startNanos = ruleMetrics.ruleStart();
-    if (reportPoint.getAnnotations() == null) {
-      ruleMetrics.ruleEnd(startNanos);
-      return reportPoint;
-    }
-    String tagValue = reportPoint.getAnnotations().get(tag);
-    if (tagValue == null || (compiledPattern != null && !compiledPattern.matcher(tagValue).matches())) {
-      ruleMetrics.ruleEnd(startNanos);
-      return reportPoint;
-    }
-    reportPoint.getAnnotations().remove(tag);
-    reportPoint.getAnnotations().put(newTag, tagValue);
-    ruleMetrics.incrementRuleAppliedCounter();
-    ruleMetrics.ruleEnd(startNanos);
-    return reportPoint;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointReplaceRegexTransformer.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointReplaceRegexTransformer.java
deleted file mode 100644
index 451f3681d..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointReplaceRegexTransformer.java
+++ /dev/null
@@ -1,112 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Replace regex transformer. Performs search and replace on a specified component of a point (metric name,
- * source name or a point tag value, depending on "scope" parameter.
- *
- * Created by Vasily on 9/13/16.
- */
-public class ReportPointReplaceRegexTransformer implements Function {
-
-  private final String patternReplace;
-  private final String scope;
-  private final Pattern compiledSearchPattern;
-  private final Integer maxIterations;
-  @Nullable
-  private final Pattern compiledMatchPattern;
-  private final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public ReportPointReplaceRegexTransformer(final String scope,
-                                            final String patternSearch,
-                                            final String patternReplace,
-                                            @Nullable final String patternMatch,
-                                            @Nullable final Integer maxIterations,
-                                            @Nullable final Counter ruleAppliedCounter) {
-    this(scope, patternSearch, patternReplace, patternMatch, maxIterations, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public ReportPointReplaceRegexTransformer(final String scope,
-                                            final String patternSearch,
-                                            final String patternReplace,
-                                            @Nullable final String patternMatch,
-                                            @Nullable final Integer maxIterations,
-                                            final PreprocessorRuleMetrics ruleMetrics) {
-    this.compiledSearchPattern = Pattern.compile(Preconditions.checkNotNull(patternSearch, "[search] can't be null"));
-    Preconditions.checkArgument(!patternSearch.isEmpty(), "[search] can't be blank");
-    this.scope = Preconditions.checkNotNull(scope, "[scope] can't be null");
-    Preconditions.checkArgument(!scope.isEmpty(), "[scope] can't be blank");
-    this.patternReplace = Preconditions.checkNotNull(patternReplace, "[replace] can't be null");
-    this.compiledMatchPattern = patternMatch != null ? Pattern.compile(patternMatch) : null;
-    this.maxIterations = maxIterations != null ? maxIterations : 1;
-    Preconditions.checkArgument(this.maxIterations > 0, "[iterations] must be > 0");
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  private String replaceString(@NotNull ReportPoint reportPoint, String content) {
-    Matcher patternMatcher;
-    patternMatcher = compiledSearchPattern.matcher(content);
-    if (!patternMatcher.find()) {
-      return content;
-    }
-    ruleMetrics.incrementRuleAppliedCounter();
-
-    String replacement = PreprocessorUtil.expandPlaceholders(patternReplace, reportPoint);
-
-    int currentIteration = 0;
-    while (currentIteration < maxIterations) {
-      content = patternMatcher.replaceAll(replacement);
-      patternMatcher = compiledSearchPattern.matcher(content);
-      if (!patternMatcher.find()) {
-        break;
-      }
-      currentIteration++;
-    }
-    return content;
-  }
-
-  @Override
-  public ReportPoint apply(@NotNull ReportPoint reportPoint) {
-    long startNanos = ruleMetrics.ruleStart();
-    switch (scope) {
-      case "metricName":
-        if (compiledMatchPattern != null && !compiledMatchPattern.matcher(reportPoint.getMetric()).matches()) {
-          break;
-        }
-        reportPoint.setMetric(replaceString(reportPoint, reportPoint.getMetric()));
-        break;
-      case "sourceName":
-        if (compiledMatchPattern != null && !compiledMatchPattern.matcher(reportPoint.getHost()).matches()) {
-          break;
-        }
-        reportPoint.setHost(replaceString(reportPoint, reportPoint.getHost()));
-        break;
-      default:
-        if (reportPoint.getAnnotations() != null) {
-          String tagValue = reportPoint.getAnnotations().get(scope);
-          if (tagValue != null) {
-            if (compiledMatchPattern != null && !compiledMatchPattern.matcher(tagValue).matches()) {
-              break;
-            }
-           reportPoint.getAnnotations().put(scope, replaceString(reportPoint, tagValue));
-          }
-        }
-    }
-    ruleMetrics.ruleEnd(startNanos);
-    return reportPoint;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointTimestampInRangeFilter.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointTimestampInRangeFilter.java
deleted file mode 100644
index cc3df1b61..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointTimestampInRangeFilter.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.wavefront.common.Clock;
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.MetricName;
-
-import org.apache.commons.lang.time.DateUtils;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Filter condition for valid timestamp - should be no more than 1 day in the future
- * and no more than X hours (usually 8760, or 1 year) in the past
- *
- * Created by Vasily on 9/16/16.
- * Updated by Howard on 1/10/18 
- * - to add support for hoursInFutureAllowed
- * - changed variable names to hoursInPastAllowed and hoursInFutureAllowed
- */
-public class ReportPointTimestampInRangeFilter extends AnnotatedPredicate {
-
-  private final int hoursInPastAllowed;
-  private final int hoursInFutureAllowed;
-
-  private final Counter outOfRangePointTimes;
-  @Nullable
-  private String message = null;
-
-  public ReportPointTimestampInRangeFilter(final int hoursInPastAllowed, final int hoursInFutureAllowed) {
-    this.hoursInPastAllowed = hoursInPastAllowed;
-    this.hoursInFutureAllowed = hoursInFutureAllowed;
-    this.outOfRangePointTimes = Metrics.newCounter(new MetricName("point", "", "badtime"));
-  }
-
-  @Override
-  public boolean apply(@NotNull ReportPoint point) {
-    this.message = null;
-    long pointTime = point.getTimestamp();
-    long rightNow = Clock.now();
-
-    // within  ago and within 
-    boolean pointInRange = (pointTime > (rightNow - this.hoursInPastAllowed * DateUtils.MILLIS_PER_HOUR)) &&
-        (pointTime < (rightNow + (this.hoursInFutureAllowed * DateUtils.MILLIS_PER_HOUR)));
-    if (!pointInRange) {
-      outOfRangePointTimes.inc();
-      this.message = "WF-402: Point outside of reasonable timeframe (" + point.toString() + ")";
-    }
-    return pointInRange;
-  }
-
-  @Override
-  public String getMessage(ReportPoint point) {
-    return this.message;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointWhitelistRegexFilter.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointWhitelistRegexFilter.java
deleted file mode 100644
index a49645881..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportPointWhitelistRegexFilter.java
+++ /dev/null
@@ -1,77 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.base.Preconditions;
-
-import com.yammer.metrics.core.Counter;
-
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-import javax.validation.constraints.NotNull;
-
-import wavefront.report.ReportPoint;
-
-/**
- * Whitelist regex filter. Rejects a point if a specified component (metric, source, or point tag value, depending
- * on the "scope" parameter) doesn't match the regex.
- *
- * Created by Vasily on 9/13/16.
- */
-public class ReportPointWhitelistRegexFilter extends AnnotatedPredicate {
-
-  private final String scope;
-  private final Pattern compiledPattern;
-  private final PreprocessorRuleMetrics ruleMetrics;
-
-  @Deprecated
-  public ReportPointWhitelistRegexFilter(final String scope,
-                                         final String patternMatch,
-                                         @Nullable final Counter ruleAppliedCounter) {
-    this(scope, patternMatch, new PreprocessorRuleMetrics(ruleAppliedCounter));
-  }
-
-  public ReportPointWhitelistRegexFilter(final String scope,
-                                         final String patternMatch,
-                                         final PreprocessorRuleMetrics ruleMetrics) {
-    this.compiledPattern = Pattern.compile(Preconditions.checkNotNull(patternMatch, "[match] can't be null"));
-    Preconditions.checkArgument(!patternMatch.isEmpty(), "[match] can't be blank");
-    this.scope = Preconditions.checkNotNull(scope, "[scope] can't be null");
-    Preconditions.checkArgument(!scope.isEmpty(), "[scope] can't be blank");
-    Preconditions.checkNotNull(ruleMetrics, "PreprocessorRuleMetrics can't be null");
-    this.ruleMetrics = ruleMetrics;
-  }
-
-  @Override
-  public boolean apply(@NotNull ReportPoint reportPoint) {
-    long startNanos = ruleMetrics.ruleStart();
-    switch (scope) {
-      case "metricName":
-        if (!compiledPattern.matcher(reportPoint.getMetric()).matches()) {
-          ruleMetrics.incrementRuleAppliedCounter();
-          ruleMetrics.ruleEnd(startNanos);
-          return false;
-        }
-        break;
-      case "sourceName":
-        if (!compiledPattern.matcher(reportPoint.getHost()).matches()) {
-          ruleMetrics.incrementRuleAppliedCounter();
-          ruleMetrics.ruleEnd(startNanos);
-          return false;
-        }
-        break;
-      default:
-        if (reportPoint.getAnnotations() != null) {
-          String tagValue = reportPoint.getAnnotations().get(scope);
-          if (tagValue != null && compiledPattern.matcher(tagValue).matches()) {
-            ruleMetrics.ruleEnd(startNanos);
-            return true;
-          }
-        }
-        ruleMetrics.incrementRuleAppliedCounter();
-        ruleMetrics.ruleEnd(startNanos);
-        return false;
-    }
-    ruleMetrics.ruleEnd(startNanos);
-    return true;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportableEntityPreprocessor.java b/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportableEntityPreprocessor.java
deleted file mode 100644
index 68611fe76..000000000
--- a/proxy/src/main/java/com/wavefront/agent/preprocessor/ReportableEntityPreprocessor.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.wavefront.agent.handlers.ReportableEntityHandler;
-
-import wavefront.report.ReportPoint;
-import wavefront.report.Span;
-
-/**
- * A container class for multiple types of rules (point line-specific and parsed entity-specific)
- *
- * Created by Vasily on 9/15/16.
- */
-public class ReportableEntityPreprocessor {
-
-  private final Preprocessor pointLinePreprocessor = new Preprocessor<>();
-  private final Preprocessor reportPointPreprocessor = new Preprocessor<>();
-  private final Preprocessor spanPreprocessor = new Preprocessor<>();
-
-  public Preprocessor forPointLine() {
-    return pointLinePreprocessor;
-  }
-
-  public Preprocessor forReportPoint() {
-    return reportPointPreprocessor;
-  }
-
-  public Preprocessor forSpan() {
-    return spanPreprocessor;
-  }
-
-  public boolean preprocessPointLine(String pointLine, ReportableEntityHandler handler) {
-    pointLine = pointLinePreprocessor.transform(pointLine);
-
-    // apply white/black lists after formatting
-    if (!pointLinePreprocessor.filter(pointLine)) {
-      if (pointLinePreprocessor.getLastFilterResult() != null) {
-        handler.reject(pointLine, pointLinePreprocessor.getLastFilterResult());
-      } else {
-        handler.block(pointLine);
-      }
-      return false;
-    }
-    return true;
-  }
-}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/ConcurrentQueueFile.java b/proxy/src/main/java/com/wavefront/agent/queueing/ConcurrentQueueFile.java
new file mode 100644
index 000000000..1373852b4
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/ConcurrentQueueFile.java
@@ -0,0 +1,102 @@
+package com.wavefront.agent.queueing;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.concurrent.locks.ReentrantLock;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * A thread-safe wrapper for {@link QueueFile}. This version assumes that operations on the head and
+ * on the tail of the queue are mutually exclusive and should be synchronized. For a more
+ * fine-grained implementation, see {@link ConcurrentShardedQueueFile} that maintains separate locks
+ * on the head and the tail of the queue.
+ *
+ * @author vasily@wavefront.com
+ */
+public class ConcurrentQueueFile implements QueueFile {
+
+  private final QueueFile delegate;
+  private final ReentrantLock lock = new ReentrantLock(true);
+
+  public ConcurrentQueueFile(QueueFile delegate) {
+    this.delegate = delegate;
+  }
+
+  @Override
+  public void add(byte[] data, int offset, int count) throws IOException {
+    lock.lock();
+    try {
+      delegate.add(data, offset, count);
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  @Override
+  public void clear() throws IOException {
+    lock.lock();
+    try {
+      delegate.clear();
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  @Nullable
+  @Override
+  public byte[] peek() throws IOException {
+    lock.lock();
+    try {
+      return delegate.peek();
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  @Override
+  public void remove() throws IOException {
+    lock.lock();
+    try {
+      delegate.remove();
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  @Override
+  public int size() {
+    return delegate.size();
+  }
+
+  @Override
+  public long storageBytes() {
+    return delegate.storageBytes();
+  }
+
+  @Override
+  public long usedBytes() {
+    return delegate.usedBytes();
+  }
+
+  @Override
+  public long availableBytes() {
+    return delegate.availableBytes();
+  }
+
+  @Override
+  public void close() throws IOException {
+    lock.lock();
+    try {
+      delegate.close();
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  @NotNull
+  @Override
+  public Iterator iterator() {
+    return delegate.iterator();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/ConcurrentShardedQueueFile.java b/proxy/src/main/java/com/wavefront/agent/queueing/ConcurrentShardedQueueFile.java
new file mode 100644
index 000000000..12915e89f
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/ConcurrentShardedQueueFile.java
@@ -0,0 +1,367 @@
+package com.wavefront.agent.queueing;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterators;
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import com.wavefront.common.Utils;
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.ObjectUtils;
+
+/**
+ * A thread-safe {@link QueueFile} implementation, that uses multiple smaller "shard" files instead
+ * of one large file. This also improves concurrency - when we have more than one file, we can add
+ * and remove tasks at the same time without mutually exclusive locking.
+ *
+ * @author vasily@wavefront.com
+ */
+public class ConcurrentShardedQueueFile implements QueueFile {
+  private static final int HEADER_SIZE_BYTES = 36;
+  private static final int TASK_HEADER_SIZE_BYTES = 4;
+  private static final int SUFFIX_DIGITS = 4;
+
+  private final String fileNamePrefix;
+  private final String fileNameSuffix;
+  private final int shardSizeBytes;
+  private final QueueFileFactory queueFileFactory;
+
+  @VisibleForTesting final Deque shards = new ConcurrentLinkedDeque<>();
+  private final ReentrantLock globalLock = new ReentrantLock(true);
+  private final ReentrantLock tailLock = new ReentrantLock(true);
+  private final ReentrantLock headLock = new ReentrantLock(true);
+  private volatile boolean closed = false;
+  private volatile byte[] head;
+  private final AtomicLong modCount = new AtomicLong();
+
+  /**
+   * @param fileNamePrefix path + file name prefix for shard files
+   * @param fileNameSuffix file name suffix to identify shard files
+   * @param shardSizeBytes target shard size bytes
+   * @param queueFileFactory factory for {@link QueueFile} objects
+   * @throws IOException if file(s) could not be created or accessed
+   */
+  public ConcurrentShardedQueueFile(
+      String fileNamePrefix,
+      String fileNameSuffix,
+      int shardSizeBytes,
+      QueueFileFactory queueFileFactory)
+      throws IOException {
+    this.fileNamePrefix = fileNamePrefix;
+    this.fileNameSuffix = fileNameSuffix;
+    this.shardSizeBytes = shardSizeBytes;
+    this.queueFileFactory = queueFileFactory;
+    //noinspection unchecked
+    for (String filename :
+        ObjectUtils.firstNonNull(
+            listFiles(fileNamePrefix, fileNameSuffix), ImmutableList.of(getInitialFilename()))) {
+      Shard shard = new Shard(filename);
+      // don't keep the QueueFile open within the shard object until it's actually needed,
+      // as we don't want to keep too many files open.
+      shard.close();
+      this.shards.add(shard);
+    }
+  }
+
+  @Nullable
+  @Override
+  public byte[] peek() throws IOException {
+    checkForClosedState();
+    headLock.lock();
+    try {
+      if (this.head == null) {
+        globalLock.lock();
+        Shard shard = shards.getFirst().updateStats();
+        if (shards.size() > 1) {
+          globalLock.unlock();
+        }
+        this.head = Objects.requireNonNull(shard.queueFile).peek();
+      }
+      return this.head;
+    } finally {
+      headLock.unlock();
+      if (globalLock.isHeldByCurrentThread()) {
+        globalLock.unlock();
+      }
+    }
+  }
+
+  @Override
+  public void add(byte[] data, int offset, int count) throws IOException {
+    checkForClosedState();
+    tailLock.lock();
+    try {
+      globalLock.lock();
+      // check whether we need to allocate a new shard
+      Shard shard = shards.getLast();
+      if (shard.newShardRequired(count)) {
+        // allocate new shard unless the task is oversized and current shard is empty
+        if (shards.size() > 1) {
+          // we don't want to close if that shard was the head
+          shard.close();
+        }
+        String newFileName = incrementFileName(shard.shardFileName, fileNameSuffix);
+        shard = new Shard(newFileName);
+        shards.addLast(shard);
+      }
+      shard.updateStats();
+      modCount.incrementAndGet();
+      if (shards.size() > 2) {
+        globalLock.unlock();
+      }
+      Objects.requireNonNull(shard.queueFile).add(data, offset, count);
+      shard.updateStats();
+    } finally {
+      tailLock.unlock();
+      if (globalLock.isHeldByCurrentThread()) {
+        globalLock.unlock();
+      }
+    }
+  }
+
+  @Override
+  public void remove() throws IOException {
+    checkForClosedState();
+    headLock.lock();
+    try {
+      this.head = null;
+      Shard shard = shards.getFirst().updateStats();
+      if (shards.size() == 1) {
+        globalLock.lock();
+      }
+      modCount.incrementAndGet();
+      Objects.requireNonNull(shard.queueFile).remove();
+      shard.updateStats();
+      // check whether we have removed the last task in a shard
+      if (shards.size() > 1 && shard.numTasks == 0) {
+        shard.close();
+        shards.removeFirst();
+        new File(shard.shardFileName).delete();
+      }
+    } finally {
+      headLock.unlock();
+      if (globalLock.isHeldByCurrentThread()) {
+        globalLock.unlock();
+      }
+    }
+  }
+
+  @Override
+  public int size() {
+    return shards.stream().mapToInt(shard -> shard.numTasks).sum();
+  }
+
+  @Override
+  public long storageBytes() {
+    return shards.stream().mapToLong(shard -> shard.fileLength).sum();
+  }
+
+  @Override
+  public long usedBytes() {
+    return shards.stream().mapToLong(shard -> shard.usedBytes).sum();
+  }
+
+  @Override
+  public long availableBytes() {
+    Shard shard = shards.getLast();
+    return shard.fileLength - shard.usedBytes;
+  }
+
+  @Override
+  public void close() throws IOException {
+    this.closed = true;
+    for (Shard shard : shards) {
+      shard.close();
+    }
+  }
+
+  @Override
+  public void clear() throws IOException {
+    this.headLock.lock();
+    this.tailLock.lock();
+    try {
+      this.head = null;
+      for (Shard shard : shards) {
+        shard.close();
+        new File(shard.shardFileName).delete();
+      }
+      shards.clear();
+      shards.add(new Shard(getInitialFilename()));
+      modCount.incrementAndGet();
+    } finally {
+      this.headLock.unlock();
+      this.tailLock.unlock();
+    }
+  }
+
+  @Nonnull
+  @Override
+  public Iterator iterator() {
+    checkForClosedState();
+    return new ShardedIterator();
+  }
+
+  private final class ShardedIterator implements Iterator {
+    long expectedModCount = modCount.get();
+    Iterator currentIterator = Collections.emptyIterator();
+    Shard currentShard = null;
+    Iterator shardIterator = shards.iterator();
+    int nextElementIndex = 0;
+
+    ShardedIterator() {}
+
+    private void checkForComodification() {
+      checkForClosedState();
+      if (modCount.get() != expectedModCount) {
+        throw new ConcurrentModificationException();
+      }
+    }
+
+    @Override
+    public boolean hasNext() {
+      checkForComodification();
+      try {
+        while (!checkNotNull(currentIterator).hasNext()) {
+          if (!shardIterator.hasNext()) {
+            return false;
+          }
+          currentShard = shardIterator.next().updateStats();
+          currentIterator = Objects.requireNonNull(currentShard.queueFile).iterator();
+        }
+      } catch (IOException e) {
+        throw Utils.throwAny(e);
+      }
+      return true;
+    }
+
+    @Override
+    public byte[] next() {
+      checkForComodification();
+      if (hasNext()) {
+        nextElementIndex++;
+        return currentIterator.next();
+      } else {
+        throw new NoSuchElementException();
+      }
+    }
+
+    @Override
+    public void remove() {
+      checkForComodification();
+      if (nextElementIndex > 1) {
+        throw new UnsupportedOperationException("Removal is only permitted from the head.");
+      }
+      try {
+        currentIterator.remove();
+        currentShard.updateStats();
+        nextElementIndex--;
+      } catch (IOException e) {
+        throw Utils.throwAny(e);
+      }
+    }
+  }
+
+  private final class Shard {
+    private final String shardFileName;
+    @Nullable private QueueFile queueFile;
+    private long fileLength;
+    private Long usedBytes;
+    private int numTasks;
+
+    private Shard(String shardFileName) throws IOException {
+      this.shardFileName = shardFileName;
+      updateStats();
+    }
+
+    @CanIgnoreReturnValue
+    private Shard updateStats() throws IOException {
+      if (this.queueFile == null) {
+        this.queueFile = queueFileFactory.get(this.shardFileName);
+      }
+      if (this.queueFile != null) {
+        this.fileLength = this.queueFile.storageBytes();
+        this.numTasks = this.queueFile.size();
+        this.usedBytes = this.queueFile.usedBytes();
+      }
+      return this;
+    }
+
+    private void close() throws IOException {
+      if (this.queueFile != null) {
+        this.queueFile.close();
+        this.queueFile = null;
+      }
+    }
+
+    private boolean newShardRequired(int taskSize) {
+      return (taskSize > (shardSizeBytes - this.usedBytes - TASK_HEADER_SIZE_BYTES)
+          && (taskSize <= (shardSizeBytes - HEADER_SIZE_BYTES) || this.numTasks > 0));
+    }
+  }
+
+  private void checkForClosedState() {
+    if (closed) {
+      throw new IllegalStateException("closed");
+    }
+  }
+
+  private String getInitialFilename() {
+    return new File(fileNamePrefix).exists()
+        ? fileNamePrefix
+        : incrementFileName(fileNamePrefix, fileNameSuffix);
+  }
+
+  @VisibleForTesting
+  @Nullable
+  static List listFiles(String path, String suffix) {
+    String fnPrefix = Iterators.getLast(Splitter.on('/').split(path).iterator());
+    Pattern pattern = getSuffixMatchingPattern(suffix);
+    File bufferFilePath = new File(path);
+    File[] files =
+        bufferFilePath
+            .getParentFile()
+            .listFiles(
+                (dir, fileName) ->
+                    (fileName.endsWith(suffix) || pattern.matcher(fileName).matches())
+                        && fileName.startsWith(fnPrefix));
+    return (files == null || files.length == 0)
+        ? null
+        : Arrays.stream(files).map(File::getAbsolutePath).sorted().collect(Collectors.toList());
+  }
+
+  @VisibleForTesting
+  static String incrementFileName(String fileName, String suffix) {
+    Pattern pattern = getSuffixMatchingPattern(suffix);
+    String zeroes = StringUtils.repeat("0", SUFFIX_DIGITS);
+    if (pattern.matcher(fileName).matches()) {
+      int nextId = Integer.parseInt(StringUtils.right(fileName, SUFFIX_DIGITS), 16) + 1;
+      String newHex = StringUtils.right(zeroes + Long.toHexString(nextId), SUFFIX_DIGITS);
+      return StringUtils.left(fileName, fileName.length() - SUFFIX_DIGITS) + newHex;
+    } else {
+      return fileName + "_" + zeroes;
+    }
+  }
+
+  private static Pattern getSuffixMatchingPattern(String suffix) {
+    return Pattern.compile("^.*" + Pattern.quote(suffix) + "_[0-9a-f]{" + SUFFIX_DIGITS + "}$");
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/DirectByteArrayOutputStream.java b/proxy/src/main/java/com/wavefront/agent/queueing/DirectByteArrayOutputStream.java
new file mode 100644
index 000000000..8fe9c09d3
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/DirectByteArrayOutputStream.java
@@ -0,0 +1,15 @@
+package com.wavefront.agent.queueing;
+
+import java.io.ByteArrayOutputStream;
+
+/** Enables direct access to the internal array. Avoids unnecessary copying. */
+public final class DirectByteArrayOutputStream extends ByteArrayOutputStream {
+
+  /**
+   * Gets a reference to the internal byte array. The {@link #size()} method indicates how many
+   * bytes contain actual data added since the last {@link #reset()} call.
+   */
+  byte[] getArray() {
+    return buf;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/FileBasedTaskQueue.java b/proxy/src/main/java/com/wavefront/agent/queueing/FileBasedTaskQueue.java
new file mode 100644
index 000000000..fdc7a55e3
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/FileBasedTaskQueue.java
@@ -0,0 +1,143 @@
+package com.wavefront.agent.queueing;
+
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.common.Utils;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * Implements proxy-specific {@link TaskQueue} interface as a wrapper over {@link QueueFile}.
+ *
+ * @param  type of objects stored.
+ * @author vasily@wavefront.com
+ */
+public class FileBasedTaskQueue> implements TaskQueue {
+  private static final Logger log = Logger.getLogger(FileBasedTaskQueue.class.getCanonicalName());
+
+  private final DirectByteArrayOutputStream bytes = new DirectByteArrayOutputStream();
+
+  private volatile T head;
+
+  private final AtomicLong currentWeight = new AtomicLong();
+  private final QueueFile queueFile;
+  private final TaskConverter taskConverter;
+
+  /**
+   * @param queueFile file backing the queue
+   * @param taskConverter task converter
+   */
+  public FileBasedTaskQueue(QueueFile queueFile, TaskConverter taskConverter) {
+    this.queueFile = queueFile;
+    this.taskConverter = taskConverter;
+    log.fine("Enumerating queue");
+    this.queueFile
+        .iterator()
+        .forEachRemaining(
+            task -> {
+              Integer weight = taskConverter.getWeight(task);
+              if (weight != null) {
+                currentWeight.addAndGet(weight);
+              }
+            });
+    log.fine("Enumerated: " + currentWeight.get() + " items in " + queueFile.size() + " tasks");
+  }
+
+  @Override
+  public T peek() {
+    try {
+      if (this.head != null) {
+        return this.head;
+      }
+      byte[] task = queueFile.peek();
+      if (task == null) return null;
+      this.head = taskConverter.fromBytes(task);
+      return this.head;
+    } catch (IOException ex) {
+      throw Utils.throwAny(ex);
+    }
+  }
+
+  @Override
+  public void add(@Nonnull T entry) throws IOException {
+    bytes.reset();
+    taskConverter.serializeToStream(entry, bytes);
+    queueFile.add(bytes.getArray(), 0, bytes.size());
+    currentWeight.addAndGet(entry.weight());
+  }
+
+  @Override
+  public void clear() throws IOException {
+    queueFile.clear();
+    this.head = null;
+    this.currentWeight.set(0);
+  }
+
+  @Override
+  public void remove() throws IOException {
+    if (this.head == null) {
+      byte[] task = queueFile.peek();
+      if (task == null) return;
+      this.head = taskConverter.fromBytes(task);
+    }
+    queueFile.remove();
+    if (this.head != null) {
+      int weight = this.head.weight();
+      currentWeight.getAndUpdate(x -> x > weight ? x - weight : 0);
+      this.head = null;
+    }
+  }
+
+  @Override
+  public int size() {
+    return queueFile.size();
+  }
+
+  @Override
+  public void close() throws IOException {
+    queueFile.close();
+  }
+
+  @Nullable
+  @Override
+  public Long weight() {
+    return currentWeight.get();
+  }
+
+  @Nullable
+  @Override
+  public Long getAvailableBytes() {
+    return queueFile.storageBytes() - queueFile.usedBytes();
+  }
+
+  @Nonnull
+  @Override
+  public Iterator iterator() {
+    Iterator iterator = queueFile.iterator();
+    return new Iterator() {
+
+      @Override
+      public boolean hasNext() {
+        return iterator.hasNext();
+      }
+
+      @Override
+      public T next() {
+        byte[] data = iterator.next();
+        try {
+          return taskConverter.fromBytes(data);
+        } catch (IOException e) {
+          throw Utils.throwAny(e);
+        }
+      }
+
+      @Override
+      public void remove() {
+        iterator.remove();
+      }
+    };
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/InMemorySubmissionQueue.java b/proxy/src/main/java/com/wavefront/agent/queueing/InMemorySubmissionQueue.java
new file mode 100644
index 000000000..f5ef23bfa
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/InMemorySubmissionQueue.java
@@ -0,0 +1,99 @@
+package com.wavefront.agent.queueing;
+
+import com.squareup.tape2.ObjectQueue;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.common.Utils;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Implements proxy-specific in-memory-queue interface as a wrapper over tape {@link ObjectQueue}
+ *
+ * @param  type of objects stored.
+ * @author mike@wavefront.com
+ */
+public class InMemorySubmissionQueue> implements TaskQueue {
+  private static final Logger log =
+      Logger.getLogger(InMemorySubmissionQueue.class.getCanonicalName());
+  private static final int MAX_BUFFER_SIZE = 50_000;
+
+  private final ObjectQueue wrapped;
+
+  private final AtomicLong currentWeight = new AtomicLong();
+  private T head;
+
+  public InMemorySubmissionQueue() {
+    this.wrapped = ObjectQueue.createInMemory();
+  }
+
+  @Override
+  public int size() {
+    return wrapped.size();
+  }
+
+  @Nullable
+  @Override
+  public Long weight() {
+    return currentWeight.get();
+  }
+
+  @Nullable
+  @Override
+  public Long getAvailableBytes() {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public T peek() {
+    try {
+      if (this.head != null) return this.head;
+      this.head = wrapped.peek();
+      return this.head;
+    } catch (IOException ex) {
+      throw Utils.throwAny(ex);
+    }
+  }
+
+  @Override
+  public void add(@Nonnull T entry) throws IOException {
+    if (wrapped.size() >= MAX_BUFFER_SIZE) {
+      log.severe("Memory buffer full - too many outstanding tasks (" + MAX_BUFFER_SIZE + ")");
+      return;
+    }
+    wrapped.add(entry);
+    currentWeight.addAndGet(entry.weight());
+  }
+
+  @Override
+  public void clear() throws IOException {
+    wrapped.clear();
+    this.head = null;
+    this.currentWeight.set(0);
+  }
+
+  @Override
+  public void remove() throws IOException {
+    T t = peek();
+    long weight = t == null ? 0 : t.weight();
+    currentWeight.getAndUpdate(x -> x > weight ? x - weight : 0);
+    wrapped.remove();
+    head = null;
+  }
+
+  @Override
+  public void close() throws IOException {
+    wrapped.close();
+  }
+
+  @NotNull
+  @Override
+  public Iterator iterator() {
+    return wrapped.iterator();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/InstrumentedTaskQueueDelegate.java b/proxy/src/main/java/com/wavefront/agent/queueing/InstrumentedTaskQueueDelegate.java
new file mode 100644
index 000000000..3d5dc7b45
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/InstrumentedTaskQueueDelegate.java
@@ -0,0 +1,142 @@
+package com.wavefront.agent.queueing;
+
+import static org.apache.commons.lang3.ObjectUtils.firstNonNull;
+
+import com.google.common.collect.ImmutableMap;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.common.TaggedMetricName;
+import com.wavefront.data.ReportableEntityType;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * A thread-safe wrapper for {@link TaskQueue} that reports queue metrics.
+ *
+ * @param  type of objects stored.
+ * @author vasily@wavefront.com
+ */
+public class InstrumentedTaskQueueDelegate>
+    implements TaskQueue {
+  private static final Logger log =
+      Logger.getLogger(InstrumentedTaskQueueDelegate.class.getCanonicalName());
+
+  private final TaskQueue delegate;
+  private volatile T head;
+
+  private final String prefix;
+  private final Map tags;
+  private final Counter tasksAddedCounter;
+  private final Counter itemsAddedCounter;
+  private final Counter tasksRemovedCounter;
+  private final Counter itemsRemovedCounter;
+
+  /**
+   * @param delegate delegate {@link TaskQueue}.
+   * @param metricPrefix prefix for metric names (default: "buffer")
+   * @param metricTags point tags for metrics (default: none)
+   * @param entityType entity type (default: points)
+   */
+  public InstrumentedTaskQueueDelegate(
+      TaskQueue delegate,
+      @Nullable String metricPrefix,
+      @Nullable Map metricTags,
+      @Nullable ReportableEntityType entityType) {
+    this.delegate = delegate;
+    String entityName = entityType == null ? "points" : entityType.toString();
+    this.prefix = firstNonNull(metricPrefix, "buffer");
+    this.tags = metricTags == null ? ImmutableMap.of() : metricTags;
+    this.tasksAddedCounter = Metrics.newCounter(new TaggedMetricName(prefix, "task-added", tags));
+    this.itemsAddedCounter =
+        Metrics.newCounter(new TaggedMetricName(prefix, entityName + "-added", tags));
+    this.tasksRemovedCounter =
+        Metrics.newCounter(new TaggedMetricName(prefix, "task-removed", tags));
+    this.itemsRemovedCounter =
+        Metrics.newCounter(new TaggedMetricName(prefix, entityName + "-removed", tags));
+  }
+
+  @Override
+  public T peek() {
+    try {
+      if (this.head != null) return this.head;
+      this.head = delegate.peek();
+      return this.head;
+    } catch (Exception e) {
+      //noinspection ConstantConditions
+      if (e instanceof IOException) {
+        Metrics.newCounter(new TaggedMetricName(prefix, "failures", tags)).inc();
+        log.severe("I/O error retrieving data from the queue: " + e.getMessage());
+        this.head = null;
+        return null;
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  @Override
+  public void add(@Nonnull T t) throws IOException {
+    delegate.add(t);
+    tasksAddedCounter.inc();
+    itemsAddedCounter.inc(t.weight());
+  }
+
+  @Override
+  public void clear() {
+    try {
+      this.head = null;
+      delegate.clear();
+    } catch (IOException e) {
+      Metrics.newCounter(new TaggedMetricName(prefix, "failures", tags)).inc();
+      log.severe("I/O error clearing queue: " + e.getMessage());
+    }
+  }
+
+  @Override
+  public void remove() {
+    try {
+      T t = this.head == null ? delegate.peek() : head;
+      long size = t == null ? 0 : t.weight();
+      delegate.remove();
+      head = null;
+      tasksRemovedCounter.inc();
+      itemsRemovedCounter.inc(size);
+    } catch (IOException e) {
+      Metrics.newCounter(new TaggedMetricName(prefix, "failures", tags)).inc();
+      log.severe("I/O error removing task from the queue: " + e.getMessage());
+    }
+  }
+
+  @Override
+  public int size() {
+    return delegate.size();
+  }
+
+  @Override
+  public void close() throws IOException {
+    delegate.close();
+  }
+
+  @Nullable
+  @Override
+  public Long weight() {
+    return delegate.weight();
+  }
+
+  @Nullable
+  @Override
+  public Long getAvailableBytes() {
+    return delegate.getAvailableBytes();
+  }
+
+  @Nonnull
+  @Override
+  public Iterator iterator() {
+    return delegate.iterator();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/QueueController.java b/proxy/src/main/java/com/wavefront/agent/queueing/QueueController.java
new file mode 100644
index 000000000..86772e245
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/QueueController.java
@@ -0,0 +1,238 @@
+package com.wavefront.agent.queueing;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.common.Managed;
+import com.wavefront.common.NamedThreadFactory;
+import com.wavefront.common.Pair;
+import com.wavefront.common.TaggedMetricName;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Gauge;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.TimerTask;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+/**
+ * A queue controller (one per entity/port). Responsible for reporting queue-related metrics and
+ * adjusting priority across queues.
+ *
+ * @param  submission task type
+ */
+public class QueueController> extends TimerTask implements Managed {
+  private static final Logger logger = Logger.getLogger(QueueController.class.getCanonicalName());
+
+  // min difference in queued timestamps for the schedule adjuster to kick in
+  private static final int TIME_DIFF_THRESHOLD_SECS = 60;
+  private static final double MIN_ADJ_FACTOR = 0.25d;
+  private static final double MAX_ADJ_FACTOR = 1.5d;
+  private static ScheduledExecutorService executor;
+
+  protected final HandlerKey handlerKey;
+  protected final List> processorTasks;
+  protected final Supplier timeProvider;
+  private final Consumer backlogSizeSink;
+
+  private final AtomicBoolean isRunning = new AtomicBoolean(false);
+  private final AtomicLong currentWeight = new AtomicLong();
+  private final AtomicInteger queueSize = new AtomicInteger();
+
+  /**
+   * @param handlerKey Pipeline handler key
+   * @param processorTasks List of {@link QueueProcessor} tasks responsible for processing the
+   *     backlog.
+   * @param backlogSizeSink Where to report backlog size.
+   */
+  public QueueController(
+      HandlerKey handlerKey,
+      List> processorTasks,
+      @Nullable Consumer backlogSizeSink) {
+    this(handlerKey, processorTasks, backlogSizeSink, System::currentTimeMillis);
+  }
+
+  /**
+   * @param handlerKey Pipeline handler key
+   * @param processorTasks List of {@link QueueProcessor} tasks responsible for processing the
+   *     backlog.
+   * @param backlogSizeSink Where to report backlog size.
+   * @param timeProvider current time provider (in millis).
+   */
+  QueueController(
+      HandlerKey handlerKey,
+      List> processorTasks,
+      @Nullable Consumer backlogSizeSink,
+      Supplier timeProvider) {
+    this.handlerKey = handlerKey;
+    this.processorTasks = processorTasks;
+    this.backlogSizeSink = backlogSizeSink;
+    this.timeProvider = timeProvider == null ? System::currentTimeMillis : timeProvider;
+
+    Metrics.newGauge(
+        new TaggedMetricName(
+            "buffer",
+            "task-count",
+            "port",
+            handlerKey.getHandle(),
+            "content",
+            handlerKey.getEntityType().toString()),
+        new Gauge() {
+          @Override
+          public Integer value() {
+            return queueSize.get();
+          }
+        });
+    Metrics.newGauge(
+        new TaggedMetricName(
+            "buffer", handlerKey.getEntityType() + "-count", "port", handlerKey.getHandle()),
+        new Gauge() {
+          @Override
+          public Long value() {
+            return currentWeight.get();
+          }
+        });
+  }
+
+  /**
+   * Compares timestamps of tasks at the head of all backing queues. If the time difference between
+   * most recently queued head and the oldest queued head (across all backing queues) is less than
+   * {@code TIME_DIFF_THRESHOLD_SECS}, restore timing factor to 1.0d for all processors. If the
+   * difference is higher, adjust timing factors proportionally (use linear interpolation to stretch
+   * timing factor between {@code MIN_ADJ_FACTOR} and {@code MAX_ADJ_FACTOR}.
+   *
+   * @param processors processors
+   */
+  @VisibleForTesting
+  static > void adjustTimingFactors(
+      List> processors) {
+    List, Long>> sortedProcessors =
+        processors.stream()
+            .map(x -> new Pair<>(x, x.getHeadTaskTimestamp()))
+            .filter(x -> x._2 < Long.MAX_VALUE)
+            .sorted(Comparator.comparing(o -> o._2))
+            .collect(Collectors.toList());
+    if (sortedProcessors.size() > 1) {
+      long minTs = sortedProcessors.get(0)._2;
+      long maxTs = sortedProcessors.get(sortedProcessors.size() - 1)._2;
+      if (maxTs - minTs > TIME_DIFF_THRESHOLD_SECS * 1000) {
+        sortedProcessors.forEach(
+            x ->
+                x._1.setTimingFactor(
+                    MIN_ADJ_FACTOR
+                        + ((double) (x._2 - minTs) / (maxTs - minTs))
+                            * (MAX_ADJ_FACTOR - MIN_ADJ_FACTOR)));
+      } else {
+        processors.forEach(x -> x.setTimingFactor(1.0d));
+      }
+    }
+  }
+
+  @Override
+  public void run() {
+    // 1. grab current queue sizes (tasks count) and report to EntityProperties
+    int backlog = processorTasks.stream().mapToInt(x -> x.getTaskQueue().size()).sum();
+    if (backlogSizeSink != null) {
+      backlogSizeSink.accept(backlog);
+    }
+    // 2. adjust timing
+    adjustTimingFactors(processorTasks);
+  }
+
+  private void printQueueStats() {
+    // 1. grab current queue sizes (tasks count)
+    int backlog = processorTasks.stream().mapToInt(x -> x.getTaskQueue().size()).sum();
+    queueSize.set(backlog);
+
+    // 2. grab queue sizes (points/etc count)
+    long actualWeight = 0L;
+    if (backlog > 0) {
+      for (QueueProcessor task : processorTasks) {
+        TaskQueue taskQueue = task.getTaskQueue();
+        if ((taskQueue != null) && (taskQueue.weight() != null)) {
+          actualWeight += taskQueue.weight();
+        }
+      }
+    }
+
+    long previousWeight = currentWeight.getAndSet(actualWeight);
+
+    // 4. print stats when there's backlog
+    if ((previousWeight != 0) || (actualWeight != 0)) {
+      logger.info(
+          "["
+              + handlerKey.getHandle()
+              + "] "
+              + handlerKey.getEntityType()
+              + " backlog status: "
+              + queueSize
+              + " tasks, "
+              + currentWeight
+              + " "
+              + handlerKey.getEntityType());
+      if (actualWeight == 0) {
+        logger.info(
+            "["
+                + handlerKey.getHandle()
+                + "] "
+                + handlerKey.getEntityType()
+                + " backlog has been cleared!");
+      }
+    }
+  }
+
+  @Override
+  public void start() {
+    if (isRunning.compareAndSet(false, true)) {
+      if ((executor == null) || (executor.isShutdown())) { // need it for unittests
+        executor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("QueueController"));
+      }
+      executor.scheduleAtFixedRate(this::run, 1, 1, TimeUnit.SECONDS);
+      executor.scheduleAtFixedRate(this::printQueueStats, 10, 10, TimeUnit.SECONDS);
+      processorTasks.forEach(QueueProcessor::start);
+    }
+  }
+
+  @Override
+  public void stop() {
+    if (isRunning.compareAndSet(true, false)) {
+      executor.shutdown();
+      processorTasks.forEach(QueueProcessor::stop);
+    }
+  }
+
+  public void truncateBuffers() {
+    processorTasks.forEach(
+        tQueueProcessor -> {
+          logger.info(
+              "["
+                  + handlerKey.getHandle()
+                  + "] "
+                  + handlerKey.getEntityType()
+                  + "-- size before truncate: "
+                  + tQueueProcessor.getTaskQueue().size());
+          try {
+            tQueueProcessor.getTaskQueue().clear();
+          } catch (IOException e) {
+            e.printStackTrace();
+          }
+          logger.info(
+              "["
+                  + handlerKey.getHandle()
+                  + "] "
+                  + handlerKey.getEntityType()
+                  + "--> size after truncate: "
+                  + tQueueProcessor.getTaskQueue().size());
+        });
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/QueueExporter.java b/proxy/src/main/java/com/wavefront/agent/queueing/QueueExporter.java
new file mode 100644
index 000000000..48a3a216c
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/QueueExporter.java
@@ -0,0 +1,167 @@
+package com.wavefront.agent.queueing;
+
+import static com.wavefront.agent.queueing.ConcurrentShardedQueueFile.listFiles;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.data.EntityPropertiesFactory;
+import com.wavefront.agent.data.EventDataSubmissionTask;
+import com.wavefront.agent.data.LineDelimitedDataSubmissionTask;
+import com.wavefront.agent.data.SourceTagSubmissionTask;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.dto.Event;
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.math.NumberUtils;
+
+/**
+ * Supports proxy's ability to export data from buffer files.
+ *
+ * @author vasily@wavefront.com
+ */
+public class QueueExporter {
+  private static final Logger logger = Logger.getLogger(QueueExporter.class.getCanonicalName());
+  private static final Pattern FILENAME =
+      Pattern.compile("^(.*)\\.(\\w+)\\.(\\w+)\\.(\\w+)\\.(\\w+)$");
+
+  private final String bufferFile;
+  private final String exportQueuePorts;
+  private final String exportQueueOutputFile;
+  private final boolean retainData;
+  private final TaskQueueFactory taskQueueFactory;
+  private final EntityPropertiesFactory entityPropertiesFactory;
+
+  /**
+   * @param bufferFile
+   * @param exportQueuePorts
+   * @param exportQueueOutputFile
+   * @param retainData
+   * @param taskQueueFactory Factory for task queues
+   * @param entityPropertiesFactory Entity properties factory
+   */
+  public QueueExporter(
+      String bufferFile,
+      String exportQueuePorts,
+      String exportQueueOutputFile,
+      boolean retainData,
+      TaskQueueFactory taskQueueFactory,
+      EntityPropertiesFactory entityPropertiesFactory) {
+    this.bufferFile = bufferFile;
+    this.exportQueuePorts = exportQueuePorts;
+    this.exportQueueOutputFile = exportQueueOutputFile;
+    this.retainData = retainData;
+    this.taskQueueFactory = taskQueueFactory;
+    this.entityPropertiesFactory = entityPropertiesFactory;
+  }
+
+  /** Starts data exporting process. */
+  public void export() {
+    Set handlerKeys =
+        getValidHandlerKeys(listFiles(bufferFile, ".spool"), exportQueuePorts);
+    handlerKeys.forEach(this::processHandlerKey);
+  }
+
+  @VisibleForTesting
+  > void processHandlerKey(HandlerKey key) {
+    logger.info("Processing " + key.getEntityType() + " queue for port " + key.getHandle());
+    int threads = entityPropertiesFactory.get(key.getEntityType()).getFlushThreads();
+    for (int i = 0; i < threads; i++) {
+      TaskQueue taskQueue = taskQueueFactory.getTaskQueue(key, i);
+      if (!(taskQueue instanceof TaskQueueStub)) {
+        String outputFileName =
+            exportQueueOutputFile
+                + "."
+                + key.getEntityType()
+                + "."
+                + key.getHandle()
+                + "."
+                + i
+                + ".txt";
+        logger.info("Exporting data to " + outputFileName);
+        try {
+          BufferedWriter writer = new BufferedWriter(new FileWriter(outputFileName));
+          processQueue(taskQueue, writer);
+          writer.close();
+          taskQueue.close();
+        } catch (IOException e) {
+          logger.log(Level.SEVERE, "IO error", e);
+        }
+      }
+    }
+  }
+
+  @VisibleForTesting
+  > void processQueue(TaskQueue queue, BufferedWriter writer)
+      throws IOException {
+    int tasksProcessed = 0;
+    int itemsExported = 0;
+    Iterator iterator = queue.iterator();
+    while (iterator.hasNext()) {
+      T task = iterator.next();
+      processTask(task, writer);
+      if (!retainData) {
+        iterator.remove();
+      }
+      tasksProcessed++;
+      itemsExported += task.weight();
+    }
+    logger.info(tasksProcessed + " tasks, " + itemsExported + " items exported");
+  }
+
+  @VisibleForTesting
+  > void processTask(T task, BufferedWriter writer)
+      throws IOException {
+    if (task instanceof LineDelimitedDataSubmissionTask) {
+      for (String line : ((LineDelimitedDataSubmissionTask) task).payload()) {
+        writer.write(line);
+        writer.newLine();
+      }
+    } else if (task instanceof SourceTagSubmissionTask) {
+      writer.write(((SourceTagSubmissionTask) task).payload().toString());
+      writer.newLine();
+    } else if (task instanceof EventDataSubmissionTask) {
+      for (Event event : ((EventDataSubmissionTask) task).payload()) {
+        writer.write(event.toString());
+        writer.newLine();
+      }
+    }
+  }
+
+  @VisibleForTesting
+  static Set getValidHandlerKeys(@Nullable List files, String portList) {
+    if (files == null) {
+      return Collections.emptySet();
+    }
+    Set ports =
+        new HashSet<>(Splitter.on(",").omitEmptyStrings().trimResults().splitToList(portList));
+    Set out = new HashSet<>();
+    files.forEach(
+        x -> {
+          Matcher matcher = FILENAME.matcher(x);
+          if (matcher.matches()) {
+            ReportableEntityType type = ReportableEntityType.fromString(matcher.group(2));
+            String handle = matcher.group(3);
+            if (type != null
+                && NumberUtils.isDigits(matcher.group(4))
+                && !handle.startsWith("_")
+                && (portList.equalsIgnoreCase("all") || ports.contains(handle))) {
+              out.add(HandlerKey.of(type, handle));
+            }
+          }
+        });
+    return out;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/QueueFile.java b/proxy/src/main/java/com/wavefront/agent/queueing/QueueFile.java
new file mode 100644
index 000000000..7ee856d04
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/QueueFile.java
@@ -0,0 +1,85 @@
+package com.wavefront.agent.queueing;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+
+/**
+ * Proxy-specific FIFO queue interface for storing {@code byte[]}. This allows us to potentially
+ * support multiple backing storages in the future.
+ *
+ * @author vasily@wavefront.com
+ */
+public interface QueueFile extends Closeable, Iterable {
+  /**
+   * Adds an element to the end of the queue.
+   *
+   * @param data to copy bytes from
+   */
+  default void add(byte[] data) throws IOException {
+    add(data, 0, data.length);
+  }
+
+  /**
+   * Adds an element to the end of the queue.
+   *
+   * @param data to copy bytes from
+   * @param offset to start from in buffer
+   * @param count number of bytes to copy
+   * @throws IndexOutOfBoundsException if {@code offset < 0} or {@code count < 0}, or if {@code
+   *     offset + count} is bigger than the length of {@code buffer}.
+   */
+  void add(byte[] data, int offset, int count) throws IOException;
+
+  /** Clears this queue. Truncates the file to the initial size. */
+  void clear() throws IOException;
+
+  /**
+   * Checks whether this queue is empty.
+   *
+   * @return true if this queue contains no entries
+   */
+  default boolean isEmpty() {
+    return size() == 0;
+  }
+
+  /**
+   * Reads the eldest element. Returns null if the queue is empty.
+   *
+   * @return the eldest element.
+   */
+  @Nullable
+  byte[] peek() throws IOException;
+
+  /**
+   * Removes the eldest element.
+   *
+   * @throws NoSuchElementException if the queue is empty
+   */
+  void remove() throws IOException;
+
+  /** Returns the number of elements in this queue. */
+  int size();
+
+  /**
+   * Returns the storage size (on-disk file size) in bytes.
+   *
+   * @return file size in bytes.
+   */
+  long storageBytes();
+
+  /**
+   * Returns the number of bytes used for data.
+   *
+   * @return bytes used.
+   */
+  long usedBytes();
+
+  /**
+   * Returns the number of bytes available for adding new tasks without growing the file.
+   *
+   * @return bytes available.
+   */
+  long availableBytes();
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/QueueFileFactory.java b/proxy/src/main/java/com/wavefront/agent/queueing/QueueFileFactory.java
new file mode 100644
index 000000000..199bbcafd
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/QueueFileFactory.java
@@ -0,0 +1,20 @@
+package com.wavefront.agent.queueing;
+
+import java.io.IOException;
+
+/**
+ * Factory for {@link QueueFile} instances.
+ *
+ * @author vasily@wavefront.com
+ */
+public interface QueueFileFactory {
+
+  /**
+   * Creates, or accesses an existing file, with the specified name.
+   *
+   * @param fileName file name to use
+   * @return queue file instance
+   * @throws IOException if file could not be created or accessed
+   */
+  QueueFile get(String fileName) throws IOException;
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/QueueProcessor.java b/proxy/src/main/java/com/wavefront/agent/queueing/QueueProcessor.java
new file mode 100644
index 000000000..de5c7075e
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/QueueProcessor.java
@@ -0,0 +1,212 @@
+package com.wavefront.agent.queueing;
+
+import com.google.common.base.Suppliers;
+import com.google.common.util.concurrent.RecyclableRateLimiter;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.data.EntityProperties;
+import com.wavefront.agent.data.GlobalProperties;
+import com.wavefront.agent.data.TaskInjector;
+import com.wavefront.agent.data.TaskResult;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.common.Managed;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+
+/**
+ * A thread responsible for processing the backlog from a single task queue.
+ *
+ * @param  type of queued tasks
+ * @author vasily@wavefront.com
+ */
+public class QueueProcessor> implements Runnable, Managed {
+  protected static final Logger logger = Logger.getLogger(QueueProcessor.class.getCanonicalName());
+
+  protected final HandlerKey handlerKey;
+  protected final TaskQueue taskQueue;
+  protected final ScheduledExecutorService scheduler;
+  private final GlobalProperties globalProps;
+  protected final TaskInjector taskInjector;
+  protected final EntityProperties runtimeProperties;
+  protected final RecyclableRateLimiter rateLimiter;
+  private volatile long headTaskTimestamp = Long.MAX_VALUE;
+  private volatile double schedulerTimingFactor = 1.0d;
+  private final AtomicBoolean isRunning = new AtomicBoolean(false);
+  private int backoffExponent = 1;
+  private Supplier storedTask;
+
+  /**
+   * @param handlerKey pipeline handler key
+   * @param taskQueue backing queue
+   * @param taskInjector injects members into task objects after deserialization
+   * @param entityProps container for mutable proxy settings.
+   * @param globalProps container for mutable global proxy settings.
+   */
+  public QueueProcessor(
+      final HandlerKey handlerKey,
+      @Nonnull final TaskQueue taskQueue,
+      final TaskInjector taskInjector,
+      final ScheduledExecutorService scheduler,
+      final EntityProperties entityProps,
+      final GlobalProperties globalProps) {
+    this.handlerKey = handlerKey;
+    this.taskQueue = taskQueue;
+    this.taskInjector = taskInjector;
+    this.runtimeProperties = entityProps;
+    this.rateLimiter = entityProps.getRateLimiter();
+    this.scheduler = scheduler;
+    this.globalProps = globalProps;
+  }
+
+  @Override
+  public void run() {
+    if (!isRunning.get()) return;
+    int successes = 0;
+    int failures = 0;
+    boolean rateLimiting = false;
+    try {
+      while (taskQueue.size() > 0 && taskQueue.size() > failures) {
+        if (!isRunning.get() || Thread.currentThread().isInterrupted()) return;
+        if (storedTask == null) {
+          storedTask = Suppliers.memoizeWithExpiration(taskQueue::peek, 500, TimeUnit.MILLISECONDS);
+        }
+        T task = storedTask.get();
+        int taskSize = task == null ? 0 : task.weight();
+        this.headTaskTimestamp = task == null ? Long.MAX_VALUE : task.getEnqueuedMillis();
+        int permitsNeeded = Math.min((int) rateLimiter.getRate(), taskSize);
+        if (!rateLimiter.immediatelyAvailable(permitsNeeded)) {
+          // if there's less than 1 second worth of accumulated credits,
+          // don't process the backlog queue
+          rateLimiting = true;
+          break;
+        }
+        if (taskSize > 0) {
+          rateLimiter.acquire(taskSize);
+        }
+        boolean removeTask = true;
+        try {
+          if (task != null) {
+            taskInjector.inject(task);
+            TaskResult result = task.execute();
+            switch (result) {
+              case DELIVERED:
+                successes++;
+                break;
+              case REMOVED:
+                failures++;
+                logger.warning(
+                    "["
+                        + handlerKey.getHandle()
+                        + "] "
+                        + handlerKey.getEntityType()
+                        + " will be dropped from backlog!");
+                break;
+              case PERSISTED:
+                rateLimiter.recyclePermits(taskSize);
+                failures++;
+                return;
+              case PERSISTED_RETRY:
+                rateLimiter.recyclePermits(taskSize);
+                failures++;
+                break;
+              case RETRY_LATER:
+                removeTask = false;
+                rateLimiter.recyclePermits(taskSize);
+                failures++;
+            }
+          }
+          if (failures >= 10) {
+            break;
+          }
+        } finally {
+          if (removeTask) {
+            taskQueue.remove();
+            if (taskQueue.size() == 0) schedulerTimingFactor = 1.0d;
+            storedTask = null;
+          }
+        }
+      }
+      if (taskQueue.size() == 0) headTaskTimestamp = Long.MAX_VALUE;
+    } catch (Throwable ex) {
+      logger.log(Level.WARNING, "Unexpected exception", ex);
+    } finally {
+      long nextFlush;
+      if (rateLimiting) {
+        logger.fine(
+            "["
+                + handlerKey.getHandle()
+                + "] Rate limiter active, will re-attempt later "
+                + "to prioritize eal-time traffic.");
+        // if proxy rate limit exceeded, try again in 1/4 to 1/2 flush interval
+        // (to introduce some degree of fairness)
+        nextFlush =
+            (int)
+                ((1 + Math.random())
+                    * runtimeProperties.getPushFlushInterval()
+                    / 4
+                    * schedulerTimingFactor);
+      } else {
+        if (successes == 0 && failures > 0) {
+          backoffExponent = Math.min(4, backoffExponent + 1); // caps at 2*base^4
+        } else {
+          backoffExponent = 1;
+        }
+        nextFlush =
+            (long)
+                ((Math.random() + 1.0)
+                    * runtimeProperties.getPushFlushInterval()
+                    * Math.pow(globalProps.getRetryBackoffBaseSeconds(), backoffExponent)
+                    * schedulerTimingFactor);
+        logger.fine("[" + handlerKey.getHandle() + "] Next run scheduled in " + nextFlush + "ms");
+      }
+      if (isRunning.get()) {
+        scheduler.schedule(this, nextFlush, TimeUnit.MILLISECONDS);
+      }
+    }
+  }
+
+  @Override
+  public void start() {
+    if (isRunning.compareAndSet(false, true)) {
+      scheduler.submit(this);
+    }
+  }
+
+  @Override
+  public void stop() {
+    isRunning.set(false);
+  }
+
+  /**
+   * Returns the timestamp of the task at the head of the queue.
+   *
+   * @return timestamp
+   */
+  long getHeadTaskTimestamp() {
+    return this.headTaskTimestamp;
+  }
+
+  /**
+   * Returns the backing queue.
+   *
+   * @return task queue
+   */
+  TaskQueue getTaskQueue() {
+    return this.taskQueue;
+  }
+
+  /**
+   * Adjusts the timing multiplier for this processor. If the timingFactor value is lower than 1,
+   * delays between cycles get shorter which results in higher priority for the queue; if it's
+   * higher than 1, delays get longer, which, naturally, lowers the priority.
+   *
+   * @param timingFactor timing multiplier
+   */
+  void setTimingFactor(double timingFactor) {
+    this.schedulerTimingFactor = timingFactor;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/QueueingFactory.java b/proxy/src/main/java/com/wavefront/agent/queueing/QueueingFactory.java
new file mode 100644
index 000000000..1b6060cc6
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/QueueingFactory.java
@@ -0,0 +1,23 @@
+package com.wavefront.agent.queueing;
+
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.handlers.HandlerKey;
+import javax.annotation.Nonnull;
+
+/**
+ * Factory for {@link QueueProcessor} instances.
+ *
+ * @author vasily@wavefront.com
+ */
+public interface QueueingFactory {
+  /**
+   * Create a new {@code QueueController} instance for the specified handler key.
+   *
+   * @param handlerKey {@link HandlerKey} for the queue controller.
+   * @param numThreads number of threads to create processor tasks for.
+   * @param  data submission task type.
+   * @return {@code QueueController} object
+   */
+  > QueueController getQueueController(
+      @Nonnull HandlerKey handlerKey, int numThreads);
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/QueueingFactoryImpl.java b/proxy/src/main/java/com/wavefront/agent/queueing/QueueingFactoryImpl.java
new file mode 100644
index 000000000..92b5a37cc
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/QueueingFactoryImpl.java
@@ -0,0 +1,187 @@
+package com.wavefront.agent.queueing;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.data.EntityPropertiesFactory;
+import com.wavefront.agent.data.EventDataSubmissionTask;
+import com.wavefront.agent.data.LineDelimitedDataSubmissionTask;
+import com.wavefront.agent.data.LogDataSubmissionTask;
+import com.wavefront.agent.data.SourceTagSubmissionTask;
+import com.wavefront.agent.data.TaskInjector;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.common.NamedThreadFactory;
+import com.wavefront.data.ReportableEntityType;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import javax.annotation.Nonnull;
+
+/**
+ * A caching implementation of {@link QueueingFactory}.
+ *
+ * @author vasily@wavefront.com
+ */
+public class QueueingFactoryImpl implements QueueingFactory {
+
+  private final Map executors = new ConcurrentHashMap<>();
+  private final Map>> queueProcessors =
+      new ConcurrentHashMap<>();
+  private final Map> queueControllers = new ConcurrentHashMap<>();
+  private final TaskQueueFactory taskQueueFactory;
+  private final APIContainer apiContainer;
+  private final UUID proxyId;
+  private final Map entityPropsFactoryMap;
+
+  /**
+   * @param apiContainer handles interaction with Wavefront servers as well as queueing.
+   * @param proxyId proxy ID.
+   * @param taskQueueFactory factory for backing queues.
+   * @param entityPropsFactoryMap map of factory for entity-specific wrappers for multiple
+   *     multicasting mutable proxy settings.
+   */
+  public QueueingFactoryImpl(
+      APIContainer apiContainer,
+      UUID proxyId,
+      final TaskQueueFactory taskQueueFactory,
+      final Map entityPropsFactoryMap) {
+    this.apiContainer = apiContainer;
+    this.proxyId = proxyId;
+    this.taskQueueFactory = taskQueueFactory;
+    this.entityPropsFactoryMap = entityPropsFactoryMap;
+  }
+
+  /**
+   * Create a new {@code QueueProcessor} instance for the specified handler key.
+   *
+   * @param handlerKey {@link HandlerKey} for the queue processor.
+   * @param executorService executor service
+   * @param threadNum thread number
+   * @param  data submission task type
+   * @return {@code QueueProcessor} object
+   */
+  > QueueProcessor getQueueProcessor(
+      @Nonnull HandlerKey handlerKey, ScheduledExecutorService executorService, int threadNum) {
+    TaskQueue taskQueue = taskQueueFactory.getTaskQueue(handlerKey, threadNum);
+    //noinspection unchecked
+    return (QueueProcessor)
+        queueProcessors
+            .computeIfAbsent(handlerKey, x -> new TreeMap<>())
+            .computeIfAbsent(
+                threadNum,
+                x ->
+                    new QueueProcessor<>(
+                        handlerKey,
+                        taskQueue,
+                        getTaskInjector(handlerKey, taskQueue),
+                        executorService,
+                        entityPropsFactoryMap
+                            .get(handlerKey.getTenantName())
+                            .get(handlerKey.getEntityType()),
+                        entityPropsFactoryMap
+                            .get(handlerKey.getTenantName())
+                            .getGlobalProperties()));
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public > QueueController getQueueController(
+      @Nonnull HandlerKey handlerKey, int numThreads) {
+    ScheduledExecutorService executor =
+        executors.computeIfAbsent(
+            handlerKey,
+            x ->
+                Executors.newScheduledThreadPool(
+                    numThreads,
+                    new NamedThreadFactory(
+                        "queueProcessor-"
+                            + handlerKey.getEntityType()
+                            + "-"
+                            + handlerKey.getHandle())));
+    List> queueProcessors =
+        IntStream.range(0, numThreads)
+            .mapToObj(i -> (QueueProcessor) getQueueProcessor(handlerKey, executor, i))
+            .collect(Collectors.toList());
+    return (QueueController)
+        queueControllers.computeIfAbsent(
+            handlerKey,
+            x ->
+                new QueueController<>(
+                    handlerKey,
+                    queueProcessors,
+                    backlogSize ->
+                        entityPropsFactoryMap
+                            .get(handlerKey.getTenantName())
+                            .get(handlerKey.getEntityType())
+                            .reportBacklogSize(handlerKey.getHandle(), backlogSize)));
+  }
+
+  @SuppressWarnings("unchecked")
+  private > TaskInjector getTaskInjector(
+      HandlerKey handlerKey, TaskQueue queue) {
+    ReportableEntityType entityType = handlerKey.getEntityType();
+    String tenantName = handlerKey.getTenantName();
+    switch (entityType) {
+      case POINT:
+      case DELTA_COUNTER:
+      case HISTOGRAM:
+      case TRACE:
+      case TRACE_SPAN_LOGS:
+        return task ->
+            ((LineDelimitedDataSubmissionTask) task)
+                .injectMembers(
+                    apiContainer.getProxyV2APIForTenant(tenantName),
+                    proxyId,
+                    entityPropsFactoryMap.get(tenantName).get(entityType),
+                    (TaskQueue) queue);
+      case SOURCE_TAG:
+        return task ->
+            ((SourceTagSubmissionTask) task)
+                .injectMembers(
+                    apiContainer.getSourceTagAPIForTenant(tenantName),
+                    entityPropsFactoryMap.get(tenantName).get(entityType),
+                    (TaskQueue) queue);
+      case EVENT:
+        return task ->
+            ((EventDataSubmissionTask) task)
+                .injectMembers(
+                    apiContainer.getEventAPIForTenant(tenantName),
+                    proxyId,
+                    entityPropsFactoryMap.get(tenantName).get(entityType),
+                    (TaskQueue) queue);
+      case LOGS:
+        return task ->
+            ((LogDataSubmissionTask) task)
+                .injectMembers(
+                    apiContainer.getLogAPI(),
+                    proxyId,
+                    entityPropsFactoryMap.get(tenantName).get(entityType),
+                    (TaskQueue) queue);
+      default:
+        throw new IllegalArgumentException("Unexpected entity type: " + entityType);
+    }
+  }
+
+  /**
+   * The parameter handlerKey is port specific rather than tenant specific, need to convert to port
+   * + tenant specific format so that correct task can be shut down properly.
+   *
+   * @param handlerKey port specific handlerKey
+   */
+  @VisibleForTesting
+  public void flushNow(@Nonnull HandlerKey handlerKey) {
+    ReportableEntityType entityType = handlerKey.getEntityType();
+    String handle = handlerKey.getHandle();
+    HandlerKey tenantHandlerKey;
+    for (String tenantName : apiContainer.getTenantNameList()) {
+      tenantHandlerKey = HandlerKey.of(entityType, handle, tenantName);
+      queueProcessors.get(tenantHandlerKey).values().forEach(QueueProcessor::run);
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/RetryTaskConverter.java b/proxy/src/main/java/com/wavefront/agent/queueing/RetryTaskConverter.java
new file mode 100644
index 000000000..b7b2fa3f7
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/RetryTaskConverter.java
@@ -0,0 +1,155 @@
+package com.wavefront.agent.queueing;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import com.fasterxml.jackson.databind.jsontype.impl.LaissezFaireSubTypeValidator;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.common.TaggedMetricName;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.logging.Logger;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorInputStream;
+import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorOutputStream;
+import org.apache.commons.io.IOUtils;
+
+/**
+ * A serializer + deserializer of {@link DataSubmissionTask} objects for storage.
+ *
+ * @param  task type
+ * @author vasily@wavefront.com
+ */
+public class RetryTaskConverter> implements TaskConverter {
+  private static final Logger logger =
+      Logger.getLogger(RetryTaskConverter.class.getCanonicalName());
+
+  static final byte[] TASK_HEADER = new byte[] {'W', 'F'};
+  static final byte FORMAT_RAW = 1;
+  static final byte FORMAT_GZIP = 2;
+  static final byte FORMAT_LZ4_OLD = 3;
+  static final byte WRAPPED = 4;
+  static final byte FORMAT_LZ4 = 5;
+  static final byte[] PREFIX = {'W', 'F', 6, 4};
+
+  private final ObjectMapper objectMapper =
+      JsonMapper.builder().activateDefaultTyping(LaissezFaireSubTypeValidator.instance).build();
+
+  private final CompressionType compressionType;
+  private final Counter errorCounter;
+
+  /**
+   * @param handle Handle (usually port number) of the pipeline where the data came from.
+   * @param compressionType compression type to use for storing tasks.
+   */
+  public RetryTaskConverter(String handle, CompressionType compressionType) {
+    this.compressionType = compressionType;
+    this.errorCounter =
+        Metrics.newCounter(new TaggedMetricName("buffer", "read-errors", "port", handle));
+  }
+
+  @SuppressWarnings("unchecked")
+  @Nullable
+  @Override
+  public T fromBytes(@Nonnull byte[] bytes) {
+    ByteArrayInputStream input = new ByteArrayInputStream(bytes);
+    int len = TASK_HEADER.length;
+    byte[] prefix = new byte[len];
+    if (input.read(prefix, 0, len) == len && Arrays.equals(prefix, TASK_HEADER)) {
+      int bytesToRead = input.read();
+      if (bytesToRead > 0) {
+        byte[] header = new byte[bytesToRead];
+        if (input.read(header, 0, bytesToRead) == bytesToRead) {
+          InputStream stream = null;
+          byte compression = header[0] == WRAPPED && bytesToRead > 1 ? header[1] : header[0];
+          try {
+            switch (compression) {
+              case FORMAT_LZ4_OLD:
+                input.skip(21); // old lz4 header, not need with the apache commons implementation
+                stream = new BlockLZ4CompressorInputStream(input);
+                break;
+              case FORMAT_LZ4:
+                stream = new BlockLZ4CompressorInputStream(input);
+                break;
+              case FORMAT_GZIP:
+                stream = new GZIPInputStream(input);
+                break;
+              case FORMAT_RAW:
+                stream = input;
+                break;
+              default:
+                logger.warning(
+                    "Unable to restore persisted task - unsupported data format "
+                        + "header detected: "
+                        + Arrays.toString(header));
+                return null;
+            }
+            return (T) objectMapper.readValue(stream, DataSubmissionTask.class);
+          } catch (Throwable t) {
+            logger.warning("Unable to restore persisted task: " + t);
+          } finally {
+            IOUtils.closeQuietly(stream);
+          }
+        } else {
+          logger.warning("Unable to restore persisted task - corrupted header, ignoring");
+        }
+      } else {
+        logger.warning("Unable to restore persisted task - missing header, ignoring");
+      }
+    } else {
+      logger.warning("Unable to restore persisted task - invalid or missing header, ignoring");
+    }
+    errorCounter.inc();
+    return null;
+  }
+
+  @Override
+  public void serializeToStream(@Nonnull T t, @Nonnull OutputStream bytes) throws IOException {
+    bytes.write(TASK_HEADER);
+    // 6 bytes: 1 for WRAPPED, 1 for compression method, 4 for task weight (integer)
+    bytes.write(6);
+    bytes.write(WRAPPED);
+    switch (compressionType) {
+      case LZ4:
+        bytes.write(FORMAT_LZ4);
+        bytes.write(ByteBuffer.allocate(4).putInt(t.weight()).array());
+        BlockLZ4CompressorOutputStream lz4BlockOutputStream =
+            new BlockLZ4CompressorOutputStream(bytes);
+        objectMapper.writeValue(lz4BlockOutputStream, t);
+        lz4BlockOutputStream.close();
+        return;
+      case GZIP:
+        bytes.write(FORMAT_GZIP);
+        bytes.write(ByteBuffer.allocate(4).putInt(t.weight()).array());
+        GZIPOutputStream gzipOutputStream = new GZIPOutputStream(bytes);
+        objectMapper.writeValue(gzipOutputStream, t);
+        gzipOutputStream.close();
+        return;
+      case NONE:
+        bytes.write(FORMAT_RAW);
+        bytes.write(ByteBuffer.allocate(4).putInt(t.weight()).array());
+        objectMapper.writeValue(bytes, t);
+    }
+  }
+
+  @Nullable
+  @Override
+  public Integer getWeight(@Nonnull byte[] bytes) {
+    if (bytes.length > 8 && Arrays.equals(Arrays.copyOf(bytes, PREFIX.length), PREFIX)) {
+      // take a shortcut - reconstruct an integer from bytes 5 thru 7
+      return bytes[5] << 24 | (bytes[6] & 0xFF) << 16 | (bytes[7] & 0xFF) << 8 | (bytes[8] & 0xFF);
+    } else {
+      T t = fromBytes(bytes);
+      if (t == null) return null;
+      return t.weight();
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/SQSQueueFactoryImpl.java b/proxy/src/main/java/com/wavefront/agent/queueing/SQSQueueFactoryImpl.java
new file mode 100644
index 000000000..2718916aa
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/SQSQueueFactoryImpl.java
@@ -0,0 +1,144 @@
+package com.wavefront.agent.queueing;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
+import com.amazonaws.services.sqs.model.CreateQueueRequest;
+import com.amazonaws.services.sqs.model.CreateQueueResult;
+import com.amazonaws.services.sqs.model.GetQueueUrlRequest;
+import com.amazonaws.services.sqs.model.GetQueueUrlResult;
+import com.amazonaws.services.sqs.model.QueueAttributeName;
+import com.amazonaws.services.sqs.model.QueueDoesNotExistException;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.data.ReportableEntityType;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * An AmazonSQS implementation of {@link TaskQueueFactory}
+ *
+ * @author mike@wavefront.com
+ */
+public class SQSQueueFactoryImpl implements TaskQueueFactory {
+  private static final Logger logger =
+      Logger.getLogger(SQSQueueFactoryImpl.class.getCanonicalName());
+
+  private final Map>> taskQueues = new ConcurrentHashMap<>();
+
+  private final String queueNameTemplate;
+  private final String region;
+  private final String queueId;
+  private final boolean purgeBuffer;
+  private final Map queues = new ConcurrentHashMap<>();
+  private final AmazonSQS client;
+
+  /**
+   * @param template The sqsTemplateName
+   * @param region The region in AWS to operate against
+   * @param queueId The unique identifier for the queues
+   * @param purgeBuffer Whether buffer files should be nuked before starting (this may cause data
+   *     loss if queue files are not empty)
+   */
+  public SQSQueueFactoryImpl(String template, String region, String queueId, boolean purgeBuffer) {
+    this.queueNameTemplate = template;
+    this.region = region;
+    this.purgeBuffer = purgeBuffer;
+    this.queueId = queueId;
+    this.client = AmazonSQSClientBuilder.standard().withRegion(region).build();
+  }
+
+  @Override
+  public > TaskQueue getTaskQueue(
+      @Nonnull HandlerKey key, int threadNum) {
+    // noinspection unchecked
+    return (TaskQueue)
+        taskQueues
+            .computeIfAbsent(key, x -> new TreeMap<>())
+            .computeIfAbsent(threadNum, x -> createTaskQueue(key));
+  }
+
+  private > TaskQueue createTaskQueue(
+      @Nonnull HandlerKey handlerKey) {
+    if (purgeBuffer) {
+      logger.warning(
+          "--purgeBuffer is set but purging buffers is not supported on " + "SQS implementation");
+    }
+
+    final String queueName = getQueueName(handlerKey);
+    String queueUrl = queues.computeIfAbsent(queueName, x -> getOrCreateQueue(queueName));
+    if (handlerKey.getEntityType() == ReportableEntityType.SOURCE_TAG) {
+      return new InstrumentedTaskQueueDelegate(
+          new InMemorySubmissionQueue<>(),
+          "buffer.in-memory",
+          ImmutableMap.of("port", handlerKey.getHandle()),
+          handlerKey.getEntityType());
+    }
+    if (StringUtils.isNotBlank(queueUrl)) {
+      return new InstrumentedTaskQueueDelegate<>(
+          new SQSSubmissionQueue<>(
+              queueUrl,
+              AmazonSQSClientBuilder.standard().withRegion(this.region).build(),
+              new RetryTaskConverter(
+                  handlerKey.getHandle(), RetryTaskConverter.CompressionType.LZ4)),
+          "buffer.sqs",
+          ImmutableMap.of("port", handlerKey.getHandle(), "sqsQueue", queueUrl),
+          handlerKey.getEntityType());
+    }
+    return new TaskQueueStub<>();
+  }
+
+  @VisibleForTesting
+  public String getQueueName(HandlerKey handlerKey) {
+    String queueName =
+        queueNameTemplate
+            .replace("{{id}}", this.queueId)
+            .replace("{{entity}}", handlerKey.getEntityType().toString())
+            .replace("{{port}}", handlerKey.getHandle());
+    queueName = queueName.replaceAll("[^A-Za-z0-9\\-_]", "_");
+    return queueName;
+  }
+
+  private String getOrCreateQueue(String queueName) {
+    String queueUrl = queues.getOrDefault(queueName, "");
+    if (StringUtils.isNotBlank(queueUrl)) return queueUrl;
+    try {
+      GetQueueUrlResult queueUrlResult =
+          client.getQueueUrl(new GetQueueUrlRequest().withQueueName(queueName));
+      queueUrl = queueUrlResult.getQueueUrl();
+    } catch (QueueDoesNotExistException e) {
+      logger.info("Queue " + queueName + " does not exist...creating for first time");
+    } catch (AmazonClientException e) {
+      logger.log(Level.SEVERE, "Unable to lookup queue by name in aws " + queueName, e);
+    }
+    try {
+      if (StringUtils.isBlank(queueUrl)) {
+        CreateQueueRequest request = new CreateQueueRequest();
+        request
+            .addAttributesEntry(QueueAttributeName.MessageRetentionPeriod.toString(), "1209600")
+            .addAttributesEntry(QueueAttributeName.ReceiveMessageWaitTimeSeconds.toString(), "20")
+            .addAttributesEntry(QueueAttributeName.VisibilityTimeout.toString(), "60")
+            .setQueueName(queueName);
+        CreateQueueResult result = client.createQueue(request);
+        queueUrl = result.getQueueUrl();
+      }
+    } catch (AmazonClientException e) {
+      logger.log(Level.SEVERE, "Error creating queue in AWS " + queueName, e);
+    }
+
+    return queueUrl;
+  }
+
+  public static boolean isValidSQSTemplate(String template) {
+    return template.contains("{{id}}")
+        && template.contains("{{entity}}")
+        && template.contains("{{port}}");
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/SQSSubmissionQueue.java b/proxy/src/main/java/com/wavefront/agent/queueing/SQSSubmissionQueue.java
new file mode 100644
index 000000000..af83489c6
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/SQSSubmissionQueue.java
@@ -0,0 +1,179 @@
+package com.wavefront.agent.queueing;
+
+import static javax.xml.bind.DatatypeConverter.parseBase64Binary;
+import static javax.xml.bind.DatatypeConverter.printBase64Binary;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.model.DeleteMessageRequest;
+import com.amazonaws.services.sqs.model.GetQueueAttributesRequest;
+import com.amazonaws.services.sqs.model.GetQueueAttributesResult;
+import com.amazonaws.services.sqs.model.Message;
+import com.amazonaws.services.sqs.model.PurgeQueueRequest;
+import com.amazonaws.services.sqs.model.QueueAttributeName;
+import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
+import com.amazonaws.services.sqs.model.ReceiveMessageResult;
+import com.amazonaws.services.sqs.model.SendMessageRequest;
+import com.google.common.annotations.VisibleForTesting;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.common.Utils;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Implements proxy-specific queue interface as a wrapper over {@link AmazonSQS}
+ *
+ * @param  type of objects stored.
+ * @author mike@wavefront.com
+ */
+public class SQSSubmissionQueue> implements TaskQueue {
+  private static final Logger log = Logger.getLogger(SQSSubmissionQueue.class.getCanonicalName());
+
+  private final String queueUrl;
+  private final TaskConverter converter;
+
+  private final AmazonSQS sqsClient;
+
+  private volatile String messageHandle = null;
+  private volatile T head = null;
+
+  /**
+   * @param queueUrl The FQDN of the SQS Queue
+   * @param sqsClient The {@link AmazonSQS} client.
+   * @param converter The {@link TaskQueue} for converting tasks into and from the Queue
+   */
+  public SQSSubmissionQueue(String queueUrl, AmazonSQS sqsClient, TaskConverter converter) {
+    this.queueUrl = queueUrl;
+    this.converter = converter;
+    this.sqsClient = sqsClient;
+  }
+
+  @Override
+  public T peek() {
+    try {
+      if (this.head != null) return head;
+      ReceiveMessageRequest receiveRequest = new ReceiveMessageRequest(this.queueUrl);
+      receiveRequest.setMaxNumberOfMessages(1);
+      receiveRequest.setWaitTimeSeconds(1);
+      ReceiveMessageResult result = sqsClient.receiveMessage(receiveRequest);
+      List messages = result.getMessages();
+      if (messages.size() <= 0) {
+        return null;
+      }
+      Message message = messages.get(0);
+      byte[] messageBytes = parseBase64Binary(message.getBody());
+      messageHandle = message.getReceiptHandle();
+      head = converter.fromBytes(messageBytes);
+      return head;
+    } catch (IOException e) {
+      throw Utils.throwAny(e);
+    } catch (AmazonClientException e) {
+      throw Utils.throwAny(
+          new IOException("AmazonClientException while trying to peek the queues, ", e));
+    }
+  }
+
+  @Override
+  public void add(@Nonnull T t) throws IOException {
+    try {
+      SendMessageRequest request = new SendMessageRequest();
+      String contents = encodeMessageForDelivery(t);
+      request.setMessageBody(contents);
+      request.setQueueUrl(queueUrl);
+      sqsClient.sendMessage(request);
+    } catch (AmazonClientException e) {
+      throw new IOException("AmazonClientException adding messages onto the queue", e);
+    }
+  }
+
+  @VisibleForTesting
+  public String encodeMessageForDelivery(T t) throws IOException {
+    try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
+      converter.serializeToStream(t, os);
+      byte[] contents = os.toByteArray();
+      return printBase64Binary(contents);
+    }
+  }
+
+  @Override
+  public void remove() throws IOException {
+    try {
+      // We have no head, do not remove
+      if (StringUtils.isBlank(messageHandle) || head == null) {
+        return;
+      }
+      int taskSize = head.weight();
+      DeleteMessageRequest deleteRequest =
+          new DeleteMessageRequest(this.queueUrl, this.messageHandle);
+      sqsClient.deleteMessage(deleteRequest);
+      this.head = null;
+      this.messageHandle = null;
+    } catch (AmazonClientException e) {
+      throw new IOException("AmazonClientException removing from the queue", e);
+    }
+  }
+
+  @Override
+  public void clear() throws IOException {
+    try {
+      sqsClient.purgeQueue(new PurgeQueueRequest(this.queueUrl));
+    } catch (AmazonClientException e) {
+      throw new IOException("AmazonClientException clearing the queue", e);
+    }
+  }
+
+  @Override
+  public int size() {
+    int queueSize = 0;
+    try {
+      GetQueueAttributesRequest request = new GetQueueAttributesRequest(this.queueUrl);
+      request.withAttributeNames(QueueAttributeName.ApproximateNumberOfMessages);
+      GetQueueAttributesResult result = sqsClient.getQueueAttributes(request);
+      queueSize =
+          Integer.parseInt(
+              result
+                  .getAttributes()
+                  .getOrDefault(QueueAttributeName.ApproximateNumberOfMessages.toString(), "0"));
+    } catch (AmazonClientException e) {
+      log.log(Level.SEVERE, "Unable to obtain ApproximateNumberOfMessages from queue", e);
+    } catch (NumberFormatException e) {
+      log.log(
+          Level.SEVERE,
+          "Value returned for approximate number of messages is not a " + "valid number",
+          e);
+    }
+    return queueSize;
+  }
+
+  @Override
+  public void close() {
+    // Nothing to close
+  }
+
+  @Nullable
+  @Override
+  public Long weight() {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public Long getAvailableBytes() {
+    throw new UnsupportedOperationException(
+        "Cannot obtain total bytes from SQS queue, " + "consider using size instead");
+  }
+
+  @NotNull
+  @Override
+  public Iterator iterator() {
+    throw new UnsupportedOperationException("iterator() is not supported on a SQS queue");
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/TapeQueueFile.java b/proxy/src/main/java/com/wavefront/agent/queueing/TapeQueueFile.java
new file mode 100644
index 000000000..7fbae3e41
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/TapeQueueFile.java
@@ -0,0 +1,135 @@
+package com.wavefront.agent.queueing;
+
+import com.wavefront.common.TimeProvider;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Iterator;
+import java.util.function.BiConsumer;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * A {@link com.squareup.tape2.QueueFile} to {@link QueueFile} adapter.
+ *
+ * @author vasily@wavefront.com
+ */
+public class TapeQueueFile implements QueueFile {
+  private static final Method usedBytes;
+  private static final Field fileLength;
+
+  static {
+    try {
+      Class classQueueFile = Class.forName("com.squareup.tape2.QueueFile");
+      usedBytes = classQueueFile.getDeclaredMethod("usedBytes");
+      usedBytes.setAccessible(true);
+      fileLength = classQueueFile.getDeclaredField("fileLength");
+      fileLength.setAccessible(true);
+    } catch (ClassNotFoundException | NoSuchMethodException | NoSuchFieldException e) {
+      throw new AssertionError(e);
+    }
+  }
+
+  private final com.squareup.tape2.QueueFile delegate;
+  @Nullable private final BiConsumer writeStatsConsumer;
+  private final TimeProvider clock;
+
+  /** @param delegate tape queue file */
+  public TapeQueueFile(com.squareup.tape2.QueueFile delegate) {
+    this(delegate, null, null);
+  }
+
+  /**
+   * @param delegate tape queue file
+   * @param writeStatsConsumer consumer for statistics on writes (bytes written and millis taken)
+   */
+  public TapeQueueFile(
+      com.squareup.tape2.QueueFile delegate,
+      @Nullable BiConsumer writeStatsConsumer) {
+    this(delegate, writeStatsConsumer, null);
+  }
+
+  /**
+   * @param delegate tape queue file
+   * @param writeStatsConsumer consumer for statistics on writes (bytes written and millis taken)
+   * @param clock time provider (in millis)
+   */
+  public TapeQueueFile(
+      com.squareup.tape2.QueueFile delegate,
+      @Nullable BiConsumer writeStatsConsumer,
+      @Nullable TimeProvider clock) {
+    this.delegate = delegate;
+    this.writeStatsConsumer = writeStatsConsumer;
+    this.clock = clock == null ? System::currentTimeMillis : clock;
+  }
+
+  @Override
+  public void add(byte[] data, int offset, int count) throws IOException {
+    long startTime = clock.currentTimeMillis();
+    delegate.add(data, offset, count);
+    if (writeStatsConsumer != null) {
+      writeStatsConsumer.accept(count, clock.currentTimeMillis() - startTime);
+    }
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return delegate.isEmpty();
+  }
+
+  @Override
+  @Nullable
+  public byte[] peek() throws IOException {
+    return delegate.peek();
+  }
+
+  @Nonnull
+  @Override
+  public Iterator iterator() {
+    return delegate.iterator();
+  }
+
+  @Override
+  public int size() {
+    return delegate.size();
+  }
+
+  @Override
+  public long storageBytes() {
+    try {
+      return (long) fileLength.get(delegate);
+    } catch (IllegalAccessException e) {
+      return 0;
+    }
+  }
+
+  @Override
+  public long usedBytes() {
+    try {
+      return (long) usedBytes.invoke(delegate);
+    } catch (InvocationTargetException | IllegalAccessException e) {
+      return 0;
+    }
+  }
+
+  @Override
+  public long availableBytes() {
+    return storageBytes() - usedBytes();
+  }
+
+  @Override
+  public void remove() throws IOException {
+    delegate.remove();
+  }
+
+  @Override
+  public void clear() throws IOException {
+    delegate.clear();
+  }
+
+  @Override
+  public void close() throws IOException {
+    delegate.close();
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/TaskConverter.java b/proxy/src/main/java/com/wavefront/agent/queueing/TaskConverter.java
new file mode 100644
index 000000000..aaa083bd2
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/TaskConverter.java
@@ -0,0 +1,47 @@
+package com.wavefront.agent.queueing;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * Proxy-specific interface for converting data into and from queues, this potentially allows us to
+ * support other converting mechanisms in the future.
+ *
+ * @param  type of objects stored.
+ * @author mike@wavefront.com
+ */
+public interface TaskConverter {
+
+  /**
+   * De-serializes an object from a byte array.
+   *
+   * @return de-serialized object.
+   */
+  T fromBytes(@Nonnull byte[] bytes) throws IOException;
+
+  /**
+   * Serializes {@code value} to bytes written to the specified stream.
+   *
+   * @param value value to serialize.
+   * @param bytes output stream to write a {@code byte[]} to.
+   */
+  void serializeToStream(@Nonnull T value, @Nonnull OutputStream bytes) throws IOException;
+
+  /**
+   * Attempts to retrieve task weight from a {@code byte[]}, without de-serializing the object, if
+   * at all possible.
+   *
+   * @return task weight or null if not applicable.
+   */
+  @Nullable
+  Integer getWeight(@Nonnull byte[] bytes);
+
+  /** Supported compression schemas */
+  enum CompressionType {
+    NONE,
+    GZIP,
+    LZ4
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueue.java b/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueue.java
new file mode 100644
index 000000000..871b07d96
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueue.java
@@ -0,0 +1,70 @@
+package com.wavefront.agent.queueing;
+
+import com.wavefront.agent.data.DataSubmissionTask;
+import java.io.IOException;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * Proxy-specific queue interface, which is basically a wrapper for a Tape queue. This allows us to
+ * potentially support more than one backing storage in the future.
+ *
+ * @param  type of objects stored.
+ * @author vasily@wavefront.com.
+ */
+public interface TaskQueue> extends Iterable {
+
+  /**
+   * Retrieve a task that is currently at the head of the queue.
+   *
+   * @return task object
+   */
+  @Nullable
+  T peek();
+
+  /**
+   * Add a task to the end of the queue
+   *
+   * @param entry task
+   * @throws IOException IO exceptions caught by the storage engine
+   */
+  void add(@Nonnull T entry) throws IOException;
+
+  /**
+   * Remove a task from the head of the queue. Requires peek() to be called first, otherwise an
+   * {@code IllegalStateException} is thrown.
+   *
+   * @throws IOException IO exceptions caught by the storage engine
+   */
+  void remove() throws IOException;
+
+  /** Empty and re-initialize the queue. */
+  void clear() throws IOException;
+
+  /**
+   * Returns a number of tasks currently in the queue.
+   *
+   * @return number of tasks
+   */
+  int size();
+
+  /** Close the queue. Should be invoked before a graceful shutdown. */
+  void close() throws IOException;
+
+  /**
+   * Returns the total weight of the queue (sum of weights of all tasks).
+   *
+   * @return weight of the queue (null if unknown)
+   */
+  @Nullable
+  Long weight();
+
+  /**
+   * Returns the total number of pre-allocated but unused bytes in the backing file. May return null
+   * if not applicable.
+   *
+   * @return total number of available bytes in the file or null
+   */
+  @Nullable
+  Long getAvailableBytes();
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueueFactory.java b/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueueFactory.java
new file mode 100644
index 000000000..0339789f7
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueueFactory.java
@@ -0,0 +1,23 @@
+package com.wavefront.agent.queueing;
+
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.handlers.HandlerKey;
+import javax.annotation.Nonnull;
+
+/**
+ * A factory for {@link TaskQueue} objects.
+ *
+ * @author vasily@wavefront.com.
+ */
+public interface TaskQueueFactory {
+
+  /**
+   * Create a task queue for a specified {@link HandlerKey} and thread number.
+   *
+   * @param handlerKey handler key for the {@code TaskQueue}. Usually part of the file name.
+   * @param threadNum thread number. Usually part of the file name.
+   * @return task queue for the specified thread
+   */
+  > TaskQueue getTaskQueue(
+      @Nonnull HandlerKey handlerKey, int threadNum);
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueueFactoryImpl.java b/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueueFactoryImpl.java
new file mode 100644
index 000000000..b67d943bb
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueueFactoryImpl.java
@@ -0,0 +1,196 @@
+package com.wavefront.agent.queueing;
+
+import com.google.common.collect.ImmutableMap;
+import com.squareup.tape2.QueueFile;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.common.Pair;
+import com.wavefront.common.TaggedMetricName;
+import com.wavefront.metrics.ExpectedAgentMetric;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.Gauge;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.BiConsumer;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+
+/**
+ * A caching implementation of a {@link TaskQueueFactory}.
+ *
+ * @author vasily@wavefront.com.
+ */
+public class TaskQueueFactoryImpl implements TaskQueueFactory {
+  private static final Logger logger =
+      Logger.getLogger(TaskQueueFactoryImpl.class.getCanonicalName());
+  private final Map>> taskQueues = new ConcurrentHashMap<>();
+  private final List> taskQueuesLocks = new ArrayList<>();
+
+  private final String bufferFile;
+  private final boolean purgeBuffer;
+  private final boolean disableSharding;
+  private final int shardSize;
+
+  private static final Counter bytesWritten =
+      Metrics.newCounter(new TaggedMetricName("buffer", "bytes-written"));
+  private static final Counter ioTimeWrites =
+      Metrics.newCounter(new TaggedMetricName("buffer", "io-time-writes"));
+
+  /**
+   * @param bufferFile File name prefix for queue file names.
+   * @param purgeBuffer Whether buffer files should be nuked before starting (this may cause data
+   *     loss if queue files are not empty).
+   * @param disableSharding disable buffer sharding (use single file)
+   * @param shardSize target shard size (in MBytes)
+   */
+  public TaskQueueFactoryImpl(
+      String bufferFile, boolean purgeBuffer, boolean disableSharding, int shardSize) {
+    this.bufferFile = bufferFile;
+    this.purgeBuffer = purgeBuffer;
+    this.disableSharding = disableSharding;
+    this.shardSize = shardSize;
+
+    Metrics.newGauge(
+        ExpectedAgentMetric.BUFFER_BYTES_LEFT.metricName,
+        new Gauge() {
+          @Override
+          public Long value() {
+            try {
+              long availableBytes =
+                  taskQueues.values().stream()
+                      .flatMap(x -> x.values().stream())
+                      .map(TaskQueue::getAvailableBytes)
+                      .filter(Objects::nonNull)
+                      .mapToLong(x -> x)
+                      .sum();
+
+              File bufferDirectory = new File(bufferFile).getAbsoluteFile();
+              while (bufferDirectory != null && bufferDirectory.getUsableSpace() == 0) {
+                bufferDirectory = bufferDirectory.getParentFile();
+              }
+              if (bufferDirectory != null) {
+                return bufferDirectory.getUsableSpace() + availableBytes;
+              }
+            } catch (Throwable t) {
+              logger.warning("cannot compute remaining space in buffer file partition: " + t);
+            }
+            return null;
+          }
+        });
+  }
+
+  public > TaskQueue getTaskQueue(
+      @Nonnull HandlerKey key, int threadNum) {
+    //noinspection unchecked
+    TaskQueue taskQueue =
+        (TaskQueue)
+            taskQueues
+                .computeIfAbsent(key, x -> new TreeMap<>())
+                .computeIfAbsent(threadNum, x -> createTaskQueue(key, threadNum));
+    try {
+      // check if queue is closed and re-create if it is.
+      taskQueue.peek();
+    } catch (IllegalStateException e) {
+      taskQueue = createTaskQueue(key, threadNum);
+      taskQueues.get(key).put(threadNum, taskQueue);
+    }
+    return taskQueue;
+  }
+
+  private > TaskQueue createTaskQueue(
+      @Nonnull HandlerKey handlerKey, int threadNum) {
+    String fileName =
+        bufferFile
+            + "."
+            + handlerKey.getEntityType().toString()
+            + "."
+            + handlerKey.getHandle()
+            + "."
+            + threadNum;
+    String lockFileName = fileName + ".lck";
+    String spoolFileName = fileName + ".spool";
+    // Having two proxy processes write to the same buffer file simultaneously causes buffer
+    // file corruption. To prevent concurrent access from another process, we try to obtain
+    // exclusive access to a .lck file. trylock() is platform-specific so there is no
+    // iron-clad guarantee, but it works well in most cases.
+    try {
+      File lockFile = new File(lockFileName);
+      FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel();
+      FileLock lock = channel.tryLock();
+      logger.fine(() -> "lockFile: " + lockFile);
+      if (lock == null) {
+        channel.close();
+        throw new OverlappingFileLockException();
+      }
+      logger.fine(() -> "lock isValid: " + lock.isValid() + " - isShared: " + lock.isShared());
+      taskQueuesLocks.add(new Pair<>(channel, lock));
+    } catch (SecurityException e) {
+      logger.severe(
+          "Error writing to the buffer lock file "
+              + lockFileName
+              + " - please make sure write permissions are correct for this file path and restart the "
+              + "proxy: "
+              + e);
+      return new TaskQueueStub<>();
+    } catch (OverlappingFileLockException e) {
+      logger.severe(
+          "Error requesting exclusive access to the buffer "
+              + "lock file "
+              + lockFileName
+              + " - please make sure that no other processes "
+              + "access this file and restart the proxy: "
+              + e);
+      return new TaskQueueStub<>();
+    } catch (IOException e) {
+      logger.severe(
+          "Error requesting access to buffer lock file "
+              + lockFileName
+              + " Channel is "
+              + "closed or an I/O error has occurred - please restart the proxy: "
+              + e);
+      return new TaskQueueStub<>();
+    }
+    try {
+      File buffer = new File(spoolFileName);
+      if (purgeBuffer) {
+        if (buffer.delete()) {
+          logger.warning("Retry buffer has been purged: " + spoolFileName);
+        }
+      }
+      BiConsumer statsUpdater =
+          (bytes, millis) -> {
+            bytesWritten.inc(bytes);
+            ioTimeWrites.inc(millis);
+          };
+      com.wavefront.agent.queueing.QueueFile queueFile =
+          disableSharding
+              ? new ConcurrentQueueFile(
+                  new TapeQueueFile(
+                      new QueueFile.Builder(new File(spoolFileName)).build(), statsUpdater))
+              : new ConcurrentShardedQueueFile(
+                  spoolFileName,
+                  ".spool",
+                  shardSize * 1024 * 1024,
+                  s -> new TapeQueueFile(new QueueFile.Builder(new File(s)).build(), statsUpdater));
+      // TODO: allow configurable compression types and levels
+      return new InstrumentedTaskQueueDelegate<>(
+          new FileBasedTaskQueue<>(
+              queueFile,
+              new RetryTaskConverter(handlerKey.getHandle(), TaskConverter.CompressionType.LZ4)),
+          "buffer",
+          ImmutableMap.of("port", handlerKey.getHandle()),
+          handlerKey.getEntityType());
+    } catch (Exception e) {
+      logger.severe(
+          "WF-006: Unable to open or create queue file " + spoolFileName + ": " + e.getMessage());
+      return new TaskQueueStub<>();
+    }
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueueStub.java b/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueueStub.java
new file mode 100644
index 000000000..3de4ea08f
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/TaskQueueStub.java
@@ -0,0 +1,60 @@
+package com.wavefront.agent.queueing;
+
+import com.wavefront.agent.data.DataSubmissionTask;
+import java.io.IOException;
+import java.util.Iterator;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.commons.collections.iterators.EmptyIterator;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * A non-functional empty {@code TaskQueue} that throws an error when attempting to add a task. To
+ * be used as a stub when dynamic provisioning of queues failed.
+ *
+ * @author vasily@wavefront.com
+ */
+public class TaskQueueStub> implements TaskQueue {
+
+  @Override
+  public T peek() {
+    return null;
+  }
+
+  @Override
+  public void add(@Nonnull T t) throws IOException {
+    throw new IOException("Storage queue is not available!");
+  }
+
+  @Override
+  public void remove() {}
+
+  @Override
+  public void clear() {}
+
+  @Override
+  public int size() {
+    return 0;
+  }
+
+  @Override
+  public void close() {}
+
+  @Nullable
+  @Override
+  public Long weight() {
+    return null;
+  }
+
+  @Nullable
+  @Override
+  public Long getAvailableBytes() {
+    return null;
+  }
+
+  @NotNull
+  @Override
+  public Iterator iterator() {
+    return EmptyIterator.INSTANCE;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/queueing/TaskSizeEstimator.java b/proxy/src/main/java/com/wavefront/agent/queueing/TaskSizeEstimator.java
new file mode 100644
index 000000000..607a74af3
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/queueing/TaskSizeEstimator.java
@@ -0,0 +1,122 @@
+package com.wavefront.agent.queueing;
+
+import com.google.common.util.concurrent.RateLimiter;
+import com.wavefront.agent.SharedMetricsRegistry;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.common.NamedThreadFactory;
+import com.wavefront.common.TaggedMetricName;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.Meter;
+import com.yammer.metrics.core.MetricsRegistry;
+import java.io.ByteArrayOutputStream;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+/**
+ * Calculates approximate task sizes to estimate how quickly we would run out of disk space if we
+ * are no longer able to send data to the server endpoint (i.e. network outage).
+ *
+ * @author vasily@wavefront.com.
+ */
+public class TaskSizeEstimator {
+  private static final MetricsRegistry REGISTRY = SharedMetricsRegistry.getInstance();
+  /**
+   * Biases result sizes to the last 5 minutes heavily. This histogram does not see all result
+   * sizes. The executor only ever processes one posting at any given time and drops the rest.
+   * {@link #resultPostingMeter} records the actual rate (i.e. sees all posting calls).
+   */
+  private final Histogram resultPostingSizes;
+
+  private final Meter resultPostingMeter;
+  /** A single threaded bounded work queue to update result posting sizes. */
+  private final ExecutorService resultPostingSizerExecutorService;
+
+  @SuppressWarnings("rawtypes")
+  private final TaskConverter taskConverter;
+
+  /** Only size postings once every 5 seconds. */
+  @SuppressWarnings("UnstableApiUsage")
+  private final RateLimiter resultSizingRateLimier = RateLimiter.create(0.2);
+
+  /** @param handle metric pipeline handle (usually port number). */
+  public TaskSizeEstimator(String handle) {
+    this.resultPostingSizes =
+        REGISTRY.newHistogram(
+            new TaggedMetricName("post-result", "result-size", "port", handle), true);
+    this.resultPostingMeter =
+        REGISTRY.newMeter(
+            new TaggedMetricName("post-result", "results", "port", handle),
+            "results",
+            TimeUnit.MINUTES);
+    this.resultPostingSizerExecutorService =
+        new ThreadPoolExecutor(
+            1,
+            1,
+            60L,
+            TimeUnit.SECONDS,
+            new ArrayBlockingQueue<>(1),
+            new NamedThreadFactory("result-posting-sizer-" + handle));
+    // for now, we can just use a generic task converter with default lz4 compression method
+    this.taskConverter = new RetryTaskConverter<>(handle, TaskConverter.CompressionType.LZ4);
+    Metrics.newGauge(
+        new TaggedMetricName("buffer", "fill-rate", "port", handle),
+        new Gauge() {
+          @Override
+          public Long value() {
+            return getBytesPerMinute();
+          }
+        });
+  }
+
+  /**
+   * Submit a candidate task to be sized. The task may or may not be accepted, depending on the rate
+   * limiter
+   *
+   * @param task task to be sized.
+   */
+  public > void scheduleTaskForSizing(T task) {
+    resultPostingMeter.mark();
+    try {
+      //noinspection UnstableApiUsage
+      if (resultSizingRateLimier.tryAcquire()) {
+        resultPostingSizerExecutorService.submit(getPostingSizerTask(task));
+      }
+    } catch (Exception ex) {
+      // ignored.
+    }
+  }
+
+  /**
+   * Calculates the bytes per minute buffer usage rate. Needs at
+   *
+   * @return bytes per minute for requests submissions. Null if no data is available yet (needs at
+   *     least
+   */
+  @Nullable
+  public Long getBytesPerMinute() {
+    if (resultPostingSizes.count() < 50) return null;
+    if (resultPostingMeter.fifteenMinuteRate() == 0 || resultPostingSizes.mean() == 0) return null;
+    return (long) (resultPostingSizes.mean() * resultPostingMeter.fifteenMinuteRate());
+  }
+
+  public void shutdown() {
+    resultPostingSizerExecutorService.shutdown();
+  }
+
+  private > Runnable getPostingSizerTask(final T task) {
+    return () -> {
+      try {
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        taskConverter.serializeToStream(task, outputStream);
+        resultPostingSizes.update(outputStream.size());
+      } catch (Throwable t) {
+        // ignored. this is a stats task.
+      }
+    };
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/sampler/SpanSampler.java b/proxy/src/main/java/com/wavefront/agent/sampler/SpanSampler.java
new file mode 100644
index 000000000..00f4049c2
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/agent/sampler/SpanSampler.java
@@ -0,0 +1,148 @@
+package com.wavefront.agent.sampler;
+
+import static com.wavefront.internal.SpanDerivedMetricsUtils.DEBUG_SPAN_TAG_VAL;
+import static com.wavefront.sdk.common.Constants.DEBUG_TAG_KEY;
+
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.wavefront.api.agent.SpanSamplingPolicy;
+import com.wavefront.predicates.ExpressionSyntaxException;
+import com.wavefront.predicates.Predicates;
+import com.wavefront.sdk.entities.tracing.sampling.Sampler;
+import com.yammer.metrics.core.Counter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+
+/**
+ * Sampler that takes a {@link Span} as input and delegates to a {@link Sampler} when evaluating the
+ * sampling decision.
+ *
+ * @author Han Zhang (zhanghan@vmware.com)
+ */
+public class SpanSampler {
+  public static final String SPAN_SAMPLING_POLICY_TAG = "_sampledByPolicy";
+  private static final int EXPIRE_AFTER_ACCESS_SECONDS = 3600;
+  private static final int POLICY_BASED_SAMPLING_MOD_FACTOR = 100;
+  private static final Logger logger = Logger.getLogger(SpanSampler.class.getCanonicalName());
+  private final Sampler delegate;
+  private final LoadingCache> spanPredicateCache =
+      Caffeine.newBuilder()
+          .expireAfterAccess(EXPIRE_AFTER_ACCESS_SECONDS, TimeUnit.SECONDS)
+          .build(
+              new CacheLoader>() {
+                @Override
+                @Nullable
+                public Predicate load(@NonNull String key) {
+                  try {
+                    return Predicates.fromPredicateEvalExpression(key);
+                  } catch (ExpressionSyntaxException ex) {
+                    logger.severe("Policy expression " + key + " is invalid: " + ex.getMessage());
+                    return null;
+                  }
+                }
+              });
+  private final Supplier> activeSpanSamplingPoliciesSupplier;
+
+  /**
+   * Creates a new instance from a {@Sampler} delegate.
+   *
+   * @param delegate The delegate {@Sampler}.
+   * @param activeSpanSamplingPoliciesSupplier Active span sampling policies to be applied.
+   */
+  public SpanSampler(
+      Sampler delegate,
+      @Nonnull Supplier> activeSpanSamplingPoliciesSupplier) {
+    this.delegate = delegate;
+    this.activeSpanSamplingPoliciesSupplier = activeSpanSamplingPoliciesSupplier;
+  }
+
+  /**
+   * Evaluates whether a span should be allowed or discarded.
+   *
+   * @param span The span to sample.
+   * @return true if the span should be allowed, false otherwise.
+   */
+  public boolean sample(Span span) {
+    return sample(span, null);
+  }
+
+  /**
+   * Evaluates whether a span should be allowed or discarded, and increment a counter if it should
+   * be discarded.
+   *
+   * @param span The span to sample.
+   * @param discarded The counter to increment if the decision is to discard the span.
+   * @return true if the span should be allowed, false otherwise.
+   */
+  public boolean sample(Span span, @Nullable Counter discarded) {
+    if (isForceSampled(span)) {
+      return true;
+    }
+    // Policy based span sampling
+    List activeSpanSamplingPolicies = activeSpanSamplingPoliciesSupplier.get();
+    if (activeSpanSamplingPolicies != null) {
+      int samplingPercent = 0;
+      String policyId = null;
+      for (SpanSamplingPolicy policy : activeSpanSamplingPolicies) {
+        Predicate spanPredicate = spanPredicateCache.get(policy.getExpression());
+        if (spanPredicate != null
+            && spanPredicate.test(span)
+            && policy.getSamplingPercent() > samplingPercent) {
+          samplingPercent = policy.getSamplingPercent();
+          policyId = policy.getPolicyId();
+        }
+      }
+      if (samplingPercent > 0
+          && Math.abs(UUID.fromString(span.getTraceId()).getLeastSignificantBits())
+                  % POLICY_BASED_SAMPLING_MOD_FACTOR
+              <= samplingPercent) {
+        if (span.getAnnotations() == null) {
+          span.setAnnotations(new ArrayList<>());
+        }
+        span.getAnnotations().add(new Annotation(SPAN_SAMPLING_POLICY_TAG, policyId));
+        return true;
+      }
+    }
+    if (delegate.sample(
+        span.getName(),
+        UUID.fromString(span.getTraceId()).getLeastSignificantBits(),
+        span.getDuration())) {
+      return true;
+    }
+    if (discarded != null) {
+      discarded.inc();
+    }
+    return false;
+  }
+
+  /**
+   * Util method to determine if a span is force sampled. Currently force samples if any of the
+   * below conditions are met. 1. The span annotation debug=true is present 2.
+   * alwaysSampleErrors=true and the span annotation error=true is present.
+   *
+   * @param span The span to sample
+   * @return true if the span should be force sampled.
+   */
+  private boolean isForceSampled(Span span) {
+    List annotations = span.getAnnotations();
+    for (Annotation annotation : annotations) {
+      if (DEBUG_TAG_KEY.equals(annotation.getKey())) {
+        if (annotation.getValue().equals(DEBUG_SPAN_TAG_VAL)) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/agent/sampler/SpanSamplerUtils.java b/proxy/src/main/java/com/wavefront/agent/sampler/SpanSamplerUtils.java
index 5d3938e6f..5394e8274 100644
--- a/proxy/src/main/java/com/wavefront/agent/sampler/SpanSamplerUtils.java
+++ b/proxy/src/main/java/com/wavefront/agent/sampler/SpanSamplerUtils.java
@@ -3,11 +3,10 @@
 import com.wavefront.sdk.entities.tracing.sampling.DurationSampler;
 import com.wavefront.sdk.entities.tracing.sampling.RateSampler;
 import com.wavefront.sdk.entities.tracing.sampling.Sampler;
-
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-
+import java.util.Objects;
+import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
 /**
@@ -19,7 +18,7 @@ public class SpanSamplerUtils {
 
   @Nullable
   public static Sampler getRateSampler(double rate) {
-    if (rate < 0.0 || rate > 1.0) {
+    if (rate < 0.0 || rate >= 1.0) {
       return null;
     }
     return new RateSampler(rate);
@@ -38,8 +37,6 @@ public static List fromSamplers(Sampler... samplers) {
     if (samplers == null || samplers.length == 0) {
       return null;
     }
-    List l = new ArrayList<>(Arrays.asList(samplers));
-    while (l.remove(null));
-    return l;
+    return Arrays.stream(samplers).filter(Objects::nonNull).collect(Collectors.toList());
   }
 }
diff --git a/proxy/src/main/java/com/wavefront/common/HostMetricTagsPair.java b/proxy/src/main/java/com/wavefront/common/HostMetricTagsPair.java
new file mode 100644
index 000000000..25b98e40c
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/common/HostMetricTagsPair.java
@@ -0,0 +1,62 @@
+package com.wavefront.common;
+
+import com.google.common.base.Preconditions;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Tuple class to store combination of { host, metric, tags } Two or more tuples with the same value
+ * of { host, metric and tags } are considered equal and will have the same hashcode.
+ *
+ * @author Jia Deng (djia@vmware.com).
+ */
+public class HostMetricTagsPair {
+  public final String metric;
+  public final String host;
+  @Nullable private final Map tags;
+
+  public HostMetricTagsPair(String host, String metric, @Nullable Map tags) {
+    Preconditions.checkNotNull(host, "HostMetricTagsPair.host cannot be null");
+    Preconditions.checkNotNull(metric, "HostMetricTagsPair.metric cannot be null");
+    this.metric = metric.trim();
+    this.host = host.trim().toLowerCase();
+    this.tags = tags;
+  }
+
+  public String getHost() {
+    return host;
+  }
+
+  public String getMetric() {
+    return metric;
+  }
+
+  @Nullable
+  public Map getTags() {
+    return tags;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    HostMetricTagsPair that = (HostMetricTagsPair) o;
+
+    if (!metric.equals(that.metric) || !host.equals(that.host)) return false;
+    return tags != null ? tags.equals(that.tags) : that.tags == null;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = host.hashCode();
+    result = 31 * result + metric.hashCode();
+    result = 31 * result + (tags != null ? tags.hashCode() : 0);
+    return result;
+  }
+
+  @Override
+  public String toString() {
+    return String.format("[host: %s, metric: %s, tags: %s]", host, metric, tags);
+  }
+}
diff --git a/proxy/src/main/java/com/wavefront/common/Managed.java b/proxy/src/main/java/com/wavefront/common/Managed.java
new file mode 100644
index 000000000..6acec24ae
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/common/Managed.java
@@ -0,0 +1,14 @@
+package com.wavefront.common;
+
+/**
+ * Background process that can be started and stopped.
+ *
+ * @author vasily@wavefront.com
+ */
+public interface Managed {
+  /** Starts the process. */
+  void start();
+
+  /** Stops the process. */
+  void stop();
+}
diff --git a/proxy/src/main/java/com/wavefront/common/Utils.java b/proxy/src/main/java/com/wavefront/common/Utils.java
new file mode 100644
index 000000000..ab265949b
--- /dev/null
+++ b/proxy/src/main/java/com/wavefront/common/Utils.java
@@ -0,0 +1,261 @@
+package com.wavefront.common;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.Inet4Address;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.util.*;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.ws.rs.core.Response;
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * A placeholder class for miscellaneous utility methods.
+ *
+ * @author vasily@wavefront.com
+ */
+public abstract class Utils {
+
+  private static final ObjectMapper JSON_PARSER = new ObjectMapper();
+  private static final ResourceBundle buildProps = ResourceBundle.getBundle("build");
+  private static final List UUID_SEGMENTS = ImmutableList.of(8, 4, 4, 4, 12);
+
+  private static final Logger log = Logger.getLogger(Utils.class.getCanonicalName());
+
+  /**
+   * A lazy initialization wrapper for {@code Supplier}
+   *
+   * @param supplier {@code Supplier} to lazy-initialize
+   * @return lazy wrapped supplier
+   */
+  public static  Supplier lazySupplier(Supplier supplier) {
+    return new Supplier() {
+      private volatile T value = null;
+
+      @Override
+      public T get() {
+        if (value == null) {
+          synchronized (this) {
+            if (value == null) {
+              value = supplier.get();
+            }
+          }
+        }
+        return value;
+      }
+    };
+  }
+
+  /**
+   * Requires an input uuid string Encoded as 32 hex characters. For example {@code
+   * cced093a76eea418ffdc9bb9a6453df3}
+   *
+   * @param uuid string encoded as 32 hex characters.
+   * @return uuid string encoded in 8-4-4-4-12 (rfc4122) format.
+   */
+  public static String addHyphensToUuid(String uuid) {
+    if (uuid.length() != 32) {
+      return uuid;
+    }
+    StringBuilder result = new StringBuilder();
+    int startOffset = 0;
+    for (Integer segmentLength : UUID_SEGMENTS) {
+      result.append(uuid, startOffset, startOffset + segmentLength);
+      if (result.length() < 36) {
+        result.append('-');
+      }
+      startOffset += segmentLength;
+    }
+    return result.toString();
+  }
+
+  /**
+   * Method converts a string Id to {@code UUID}. This Method specifically converts id's with less
+   * than 32 digit hex characters into UUID format (See RFC 4122: A Universally Unique IDentifier
+   * (UUID) URN Namespace) by left padding id with Zeroes and adding hyphens. It assumes
+   * that if the input id contains hyphens it is already an UUID. Please don't use this method to
+   * validate/guarantee your id as an UUID.
+   *
+   * @param id a string encoded in hex characters.
+   * @return a UUID string.
+   */
+  @Nullable
+  public static String convertToUuidString(@Nullable String id) {
+    if (id == null || id.contains("-")) {
+      return id;
+    }
+    return addHyphensToUuid(StringUtils.leftPad(id, 32, '0'));
+  }
+
+  /**
+   * Creates an iterator over values a comma-delimited string.
+   *
+   * @param inputString input string
+   * @return iterator
+   */
+  @Nonnull
+  public static List csvToList(@Nullable String inputString) {
+    return inputString == null
+        ? Collections.emptyList()
+        : Splitter.on(",").omitEmptyStrings().trimResults().splitToList(inputString);
+  }
+
+  /**
+   * Attempts to retrieve build.version from system build properties.
+   *
+   * @return build version as string
+   */
+  public static String getBuildVersion() {
+    try {
+      return buildProps.getString("build.version");
+    } catch (MissingResourceException ex) {
+      return "unknown";
+    }
+  }
+
+  /**
+   * Attempts to retrieve build.package from system build properties.
+   *
+   * @return package as string
+   */
+  public static String getPackage() {
+    try {
+      return buildProps.getString("build.package");
+    } catch (MissingResourceException ex) {
+      return "unknown";
+    }
+  }
+
+  /**
+   * Returns current java runtime version information (name/vendor/version) as a string.
+   *
+   * @return java runtime version as string
+   */
+  public static String getJavaVersion() {
+    return System.getProperty("java.runtime.name", "(unknown runtime)")
+        + " ("
+        + System.getProperty("java.vendor", "")
+        + ") "
+        + System.getProperty("java.version", "(unknown version)");
+  }
+
+  /**
+   * Makes a best-effort attempt to resolve the hostname for the local host.
+   *
+   * @return name for localhost
+   */
+  private static String detectedHostName;
+
+  public static String getLocalHostName() {
+    if (detectedHostName == null) {
+      detectedHostName = detectLocalHostName();
+    }
+    return detectedHostName;
+  }
+
+  public static String detectLocalHostName() {
+    for (String env : Arrays.asList("COMPUTERNAME", "HOSTNAME", "PROXY_HOSTNAME")) {
+      String hostname = System.getenv(env);
+      if (StringUtils.isNotBlank(hostname)) {
+        log.info("Hostname: '" + hostname + "' (detected using '" + env + "' env variable)");
+        return hostname;
+      }
+    }
+
+    try {
+      String hostname =
+          new BufferedReader(
+                  new InputStreamReader(Runtime.getRuntime().exec("hostname").getInputStream()))
+              .readLine();
+      if (StringUtils.isNotBlank(hostname)) {
+        log.info("Hostname: '" + hostname + "' (detected using 'hostname' command)");
+        return hostname;
+      }
+    } catch (IOException e) {
+      log.fine("Error running 'hostname' command. " + e.getMessage());
+    }
+
+    InetAddress localAddress = null;
+    try {
+      Enumeration nics = NetworkInterface.getNetworkInterfaces();
+      while (nics.hasMoreElements()) {
+        NetworkInterface network = nics.nextElement();
+        if (!network.isUp() || network.isLoopback()) {
+          continue;
+        }
+        for (Enumeration addresses = network.getInetAddresses();
+            addresses.hasMoreElements(); ) {
+          InetAddress address = addresses.nextElement();
+          if (address.isAnyLocalAddress()
+              || address.isLoopbackAddress()
+              || address.isMulticastAddress()) {
+            continue;
+          }
+          if (address instanceof Inet4Address) { // prefer ipv4
+            localAddress = address;
+            break;
+          }
+          if (localAddress == null) {
+            localAddress = address;
+          }
+        }
+      }
+    } catch (SocketException ex) {
+      // ignore
+    }
+    if (localAddress != null) {
+      String hostname = localAddress.getCanonicalHostName();
+      log.info("Hostname: '" + hostname + "' (detected using network interfaces)");
+      return hostname;
+    }
+
+    log.info("Hostname not detected, using 'localhost')");
+    return "localhost";
+  }
+
+  /**
+   * Check if the HTTP 407/408 response was actually received from Wavefront - if it's a JSON object
+   * containing "code" key, with value equal to the HTTP response code, it's most likely from us.
+   *
+   * @param response Response object.
+   * @return whether we consider it a Wavefront response
+   */
+  public static boolean isWavefrontResponse(@Nonnull Response response) {
+    try {
+      Status res = JSON_PARSER.readValue(response.readEntity(String.class), Status.class);
+      if (res.code == response.getStatus()) {
+        return true;
+      }
+    } catch (Exception ex) {
+      // ignore
+    }
+    return false;
+  }
+
+  /**
+   * Use this to throw checked exceptions from iterator methods that do not declare that they throw
+   * checked exceptions.
+   */
+  @SuppressWarnings({"unchecked", "TypeParameterUnusedInFormals"})
+  public static  E throwAny(Throwable t) throws E {
+    throw (E) t;
+  }
+
+  @JsonIgnoreProperties(ignoreUnknown = true)
+  private static class Status {
+    @JsonProperty String message;
+    @JsonProperty int code;
+  }
+}
diff --git a/proxy/src/main/java/datadog/agentpayload/AgentPayload.java b/proxy/src/main/java/datadog/agentpayload/AgentPayload.java
new file mode 100644
index 000000000..56adfface
--- /dev/null
+++ b/proxy/src/main/java/datadog/agentpayload/AgentPayload.java
@@ -0,0 +1,15354 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: github.com/DataDog/agent-payload/proto/metrics/agent_payload.proto
+
+package datadog.agentpayload;
+
+public final class AgentPayload {
+  private AgentPayload() {}
+
+  public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {}
+
+  public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
+    registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
+  }
+
+  public interface CommonMetadataOrBuilder
+      extends
+      // @@protoc_insertion_point(interface_extends:datadog.agentpayload.CommonMetadata)
+      com.google.protobuf.MessageOrBuilder {
+
+    /**
+     * string agent_version = 1;
+     *
+     * @return The agentVersion.
+     */
+    java.lang.String getAgentVersion();
+    /**
+     * string agent_version = 1;
+     *
+     * @return The bytes for agentVersion.
+     */
+    com.google.protobuf.ByteString getAgentVersionBytes();
+
+    /**
+     * string timezone = 2;
+     *
+     * @return The timezone.
+     */
+    java.lang.String getTimezone();
+    /**
+     * string timezone = 2;
+     *
+     * @return The bytes for timezone.
+     */
+    com.google.protobuf.ByteString getTimezoneBytes();
+
+    /**
+     * double current_epoch = 3;
+     *
+     * @return The currentEpoch.
+     */
+    double getCurrentEpoch();
+
+    /**
+     * string internal_ip = 4;
+     *
+     * @return The internalIp.
+     */
+    java.lang.String getInternalIp();
+    /**
+     * string internal_ip = 4;
+     *
+     * @return The bytes for internalIp.
+     */
+    com.google.protobuf.ByteString getInternalIpBytes();
+
+    /**
+     * string public_ip = 5;
+     *
+     * @return The publicIp.
+     */
+    java.lang.String getPublicIp();
+    /**
+     * string public_ip = 5;
+     *
+     * @return The bytes for publicIp.
+     */
+    com.google.protobuf.ByteString getPublicIpBytes();
+
+    /**
+     * string api_key = 6;
+     *
+     * @return The apiKey.
+     */
+    java.lang.String getApiKey();
+    /**
+     * string api_key = 6;
+     *
+     * @return The bytes for apiKey.
+     */
+    com.google.protobuf.ByteString getApiKeyBytes();
+  }
+  /** Protobuf type {@code datadog.agentpayload.CommonMetadata} */
+  public static final class CommonMetadata extends com.google.protobuf.GeneratedMessageV3
+      implements
+      // @@protoc_insertion_point(message_implements:datadog.agentpayload.CommonMetadata)
+      CommonMetadataOrBuilder {
+    private static final long serialVersionUID = 0L;
+    // Use CommonMetadata.newBuilder() to construct.
+    private CommonMetadata(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+      super(builder);
+    }
+
+    private CommonMetadata() {
+      agentVersion_ = "";
+      timezone_ = "";
+      internalIp_ = "";
+      publicIp_ = "";
+      apiKey_ = "";
+    }
+
+    @java.lang.Override
+    @SuppressWarnings({"unused"})
+    protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+      return new CommonMetadata();
+    }
+
+    public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+      return datadog.agentpayload.AgentPayload
+          .internal_static_datadog_agentpayload_CommonMetadata_descriptor;
+    }
+
+    @java.lang.Override
+    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return datadog.agentpayload.AgentPayload
+          .internal_static_datadog_agentpayload_CommonMetadata_fieldAccessorTable
+          .ensureFieldAccessorsInitialized(
+              datadog.agentpayload.AgentPayload.CommonMetadata.class,
+              datadog.agentpayload.AgentPayload.CommonMetadata.Builder.class);
+    }
+
+    public static final int AGENT_VERSION_FIELD_NUMBER = 1;
+
+    @SuppressWarnings("serial")
+    private volatile java.lang.Object agentVersion_ = "";
+    /**
+     * string agent_version = 1;
+     *
+     * @return The agentVersion.
+     */
+    @java.lang.Override
+    public java.lang.String getAgentVersion() {
+      java.lang.Object ref = agentVersion_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        agentVersion_ = s;
+        return s;
+      }
+    }
+    /**
+     * string agent_version = 1;
+     *
+     * @return The bytes for agentVersion.
+     */
+    @java.lang.Override
+    public com.google.protobuf.ByteString getAgentVersionBytes() {
+      java.lang.Object ref = agentVersion_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b =
+            com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+        agentVersion_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
+    public static final int TIMEZONE_FIELD_NUMBER = 2;
+
+    @SuppressWarnings("serial")
+    private volatile java.lang.Object timezone_ = "";
+    /**
+     * string timezone = 2;
+     *
+     * @return The timezone.
+     */
+    @java.lang.Override
+    public java.lang.String getTimezone() {
+      java.lang.Object ref = timezone_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        timezone_ = s;
+        return s;
+      }
+    }
+    /**
+     * string timezone = 2;
+     *
+     * @return The bytes for timezone.
+     */
+    @java.lang.Override
+    public com.google.protobuf.ByteString getTimezoneBytes() {
+      java.lang.Object ref = timezone_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b =
+            com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+        timezone_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
+    public static final int CURRENT_EPOCH_FIELD_NUMBER = 3;
+    private double currentEpoch_ = 0D;
+    /**
+     * double current_epoch = 3;
+     *
+     * @return The currentEpoch.
+     */
+    @java.lang.Override
+    public double getCurrentEpoch() {
+      return currentEpoch_;
+    }
+
+    public static final int INTERNAL_IP_FIELD_NUMBER = 4;
+
+    @SuppressWarnings("serial")
+    private volatile java.lang.Object internalIp_ = "";
+    /**
+     * string internal_ip = 4;
+     *
+     * @return The internalIp.
+     */
+    @java.lang.Override
+    public java.lang.String getInternalIp() {
+      java.lang.Object ref = internalIp_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        internalIp_ = s;
+        return s;
+      }
+    }
+    /**
+     * string internal_ip = 4;
+     *
+     * @return The bytes for internalIp.
+     */
+    @java.lang.Override
+    public com.google.protobuf.ByteString getInternalIpBytes() {
+      java.lang.Object ref = internalIp_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b =
+            com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+        internalIp_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
+    public static final int PUBLIC_IP_FIELD_NUMBER = 5;
+
+    @SuppressWarnings("serial")
+    private volatile java.lang.Object publicIp_ = "";
+    /**
+     * string public_ip = 5;
+     *
+     * @return The publicIp.
+     */
+    @java.lang.Override
+    public java.lang.String getPublicIp() {
+      java.lang.Object ref = publicIp_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        publicIp_ = s;
+        return s;
+      }
+    }
+    /**
+     * string public_ip = 5;
+     *
+     * @return The bytes for publicIp.
+     */
+    @java.lang.Override
+    public com.google.protobuf.ByteString getPublicIpBytes() {
+      java.lang.Object ref = publicIp_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b =
+            com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+        publicIp_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
+    public static final int API_KEY_FIELD_NUMBER = 6;
+
+    @SuppressWarnings("serial")
+    private volatile java.lang.Object apiKey_ = "";
+    /**
+     * string api_key = 6;
+     *
+     * @return The apiKey.
+     */
+    @java.lang.Override
+    public java.lang.String getApiKey() {
+      java.lang.Object ref = apiKey_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        apiKey_ = s;
+        return s;
+      }
+    }
+    /**
+     * string api_key = 6;
+     *
+     * @return The bytes for apiKey.
+     */
+    @java.lang.Override
+    public com.google.protobuf.ByteString getApiKeyBytes() {
+      java.lang.Object ref = apiKey_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b =
+            com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+        apiKey_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
+    private byte memoizedIsInitialized = -1;
+
+    @java.lang.Override
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized == 1) return true;
+      if (isInitialized == 0) return false;
+
+      memoizedIsInitialized = 1;
+      return true;
+    }
+
+    @java.lang.Override
+    public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(agentVersion_)) {
+        com.google.protobuf.GeneratedMessageV3.writeString(output, 1, agentVersion_);
+      }
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(timezone_)) {
+        com.google.protobuf.GeneratedMessageV3.writeString(output, 2, timezone_);
+      }
+      if (java.lang.Double.doubleToRawLongBits(currentEpoch_) != 0) {
+        output.writeDouble(3, currentEpoch_);
+      }
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(internalIp_)) {
+        com.google.protobuf.GeneratedMessageV3.writeString(output, 4, internalIp_);
+      }
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(publicIp_)) {
+        com.google.protobuf.GeneratedMessageV3.writeString(output, 5, publicIp_);
+      }
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(apiKey_)) {
+        com.google.protobuf.GeneratedMessageV3.writeString(output, 6, apiKey_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+
+    @java.lang.Override
+    public int getSerializedSize() {
+      int size = memoizedSize;
+      if (size != -1) return size;
+
+      size = 0;
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(agentVersion_)) {
+        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, agentVersion_);
+      }
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(timezone_)) {
+        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, timezone_);
+      }
+      if (java.lang.Double.doubleToRawLongBits(currentEpoch_) != 0) {
+        size += com.google.protobuf.CodedOutputStream.computeDoubleSize(3, currentEpoch_);
+      }
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(internalIp_)) {
+        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(4, internalIp_);
+      }
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(publicIp_)) {
+        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(5, publicIp_);
+      }
+      if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(apiKey_)) {
+        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(6, apiKey_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSize = size;
+      return size;
+    }
+
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof datadog.agentpayload.AgentPayload.CommonMetadata)) {
+        return super.equals(obj);
+      }
+      datadog.agentpayload.AgentPayload.CommonMetadata other =
+          (datadog.agentpayload.AgentPayload.CommonMetadata) obj;
+
+      if (!getAgentVersion().equals(other.getAgentVersion())) return false;
+      if (!getTimezone().equals(other.getTimezone())) return false;
+      if (java.lang.Double.doubleToLongBits(getCurrentEpoch())
+          != java.lang.Double.doubleToLongBits(other.getCurrentEpoch())) return false;
+      if (!getInternalIp().equals(other.getInternalIp())) return false;
+      if (!getPublicIp().equals(other.getPublicIp())) return false;
+      if (!getApiKey().equals(other.getApiKey())) return false;
+      if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+      return true;
+    }
+
+    @java.lang.Override
+    public int hashCode() {
+      if (memoizedHashCode != 0) {
+        return memoizedHashCode;
+      }
+      int hash = 41;
+      hash = (19 * hash) + getDescriptor().hashCode();
+      hash = (37 * hash) + AGENT_VERSION_FIELD_NUMBER;
+      hash = (53 * hash) + getAgentVersion().hashCode();
+      hash = (37 * hash) + TIMEZONE_FIELD_NUMBER;
+      hash = (53 * hash) + getTimezone().hashCode();
+      hash = (37 * hash) + CURRENT_EPOCH_FIELD_NUMBER;
+      hash =
+          (53 * hash)
+              + com.google.protobuf.Internal.hashLong(
+                  java.lang.Double.doubleToLongBits(getCurrentEpoch()));
+      hash = (37 * hash) + INTERNAL_IP_FIELD_NUMBER;
+      hash = (53 * hash) + getInternalIp().hashCode();
+      hash = (37 * hash) + PUBLIC_IP_FIELD_NUMBER;
+      hash = (53 * hash) + getPublicIp().hashCode();
+      hash = (37 * hash) + API_KEY_FIELD_NUMBER;
+      hash = (53 * hash) + getApiKey().hashCode();
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      memoizedHashCode = hash;
+      return hash;
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseFrom(
+        java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseFrom(
+        java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseFrom(
+        byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseFrom(
+        java.io.InputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseFrom(
+        java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseDelimitedFrom(
+        java.io.InputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseDelimitedFrom(
+        java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseFrom(
+        com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    @java.lang.Override
+    public Builder newBuilderForType() {
+      return newBuilder();
+    }
+
+    public static Builder newBuilder() {
+      return DEFAULT_INSTANCE.toBuilder();
+    }
+
+    public static Builder newBuilder(datadog.agentpayload.AgentPayload.CommonMetadata prototype) {
+      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+    }
+
+    @java.lang.Override
+    public Builder toBuilder() {
+      return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+    }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    /** Protobuf type {@code datadog.agentpayload.CommonMetadata} */
+    public static final class Builder
+        extends com.google.protobuf.GeneratedMessageV3.Builder
+        implements
+        // @@protoc_insertion_point(builder_implements:datadog.agentpayload.CommonMetadata)
+        datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_CommonMetadata_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_CommonMetadata_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                datadog.agentpayload.AgentPayload.CommonMetadata.class,
+                datadog.agentpayload.AgentPayload.CommonMetadata.Builder.class);
+      }
+
+      // Construct using datadog.agentpayload.AgentPayload.CommonMetadata.newBuilder()
+      private Builder() {}
+
+      private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        super(parent);
+      }
+
+      @java.lang.Override
+      public Builder clear() {
+        super.clear();
+        bitField0_ = 0;
+        agentVersion_ = "";
+        timezone_ = "";
+        currentEpoch_ = 0D;
+        internalIp_ = "";
+        publicIp_ = "";
+        apiKey_ = "";
+        return this;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_CommonMetadata_descriptor;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.CommonMetadata getDefaultInstanceForType() {
+        return datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance();
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.CommonMetadata build() {
+        datadog.agentpayload.AgentPayload.CommonMetadata result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.CommonMetadata buildPartial() {
+        datadog.agentpayload.AgentPayload.CommonMetadata result =
+            new datadog.agentpayload.AgentPayload.CommonMetadata(this);
+        if (bitField0_ != 0) {
+          buildPartial0(result);
+        }
+        onBuilt();
+        return result;
+      }
+
+      private void buildPartial0(datadog.agentpayload.AgentPayload.CommonMetadata result) {
+        int from_bitField0_ = bitField0_;
+        if (((from_bitField0_ & 0x00000001) != 0)) {
+          result.agentVersion_ = agentVersion_;
+        }
+        if (((from_bitField0_ & 0x00000002) != 0)) {
+          result.timezone_ = timezone_;
+        }
+        if (((from_bitField0_ & 0x00000004) != 0)) {
+          result.currentEpoch_ = currentEpoch_;
+        }
+        if (((from_bitField0_ & 0x00000008) != 0)) {
+          result.internalIp_ = internalIp_;
+        }
+        if (((from_bitField0_ & 0x00000010) != 0)) {
+          result.publicIp_ = publicIp_;
+        }
+        if (((from_bitField0_ & 0x00000020) != 0)) {
+          result.apiKey_ = apiKey_;
+        }
+      }
+
+      @java.lang.Override
+      public Builder clone() {
+        return super.clone();
+      }
+
+      @java.lang.Override
+      public Builder setField(
+          com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+        return super.setField(field, value);
+      }
+
+      @java.lang.Override
+      public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+        return super.clearField(field);
+      }
+
+      @java.lang.Override
+      public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+        return super.clearOneof(oneof);
+      }
+
+      @java.lang.Override
+      public Builder setRepeatedField(
+          com.google.protobuf.Descriptors.FieldDescriptor field,
+          int index,
+          java.lang.Object value) {
+        return super.setRepeatedField(field, index, value);
+      }
+
+      @java.lang.Override
+      public Builder addRepeatedField(
+          com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+        return super.addRepeatedField(field, value);
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof datadog.agentpayload.AgentPayload.CommonMetadata) {
+          return mergeFrom((datadog.agentpayload.AgentPayload.CommonMetadata) other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+
+      public Builder mergeFrom(datadog.agentpayload.AgentPayload.CommonMetadata other) {
+        if (other == datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance())
+          return this;
+        if (!other.getAgentVersion().isEmpty()) {
+          agentVersion_ = other.agentVersion_;
+          bitField0_ |= 0x00000001;
+          onChanged();
+        }
+        if (!other.getTimezone().isEmpty()) {
+          timezone_ = other.timezone_;
+          bitField0_ |= 0x00000002;
+          onChanged();
+        }
+        if (other.getCurrentEpoch() != 0D) {
+          setCurrentEpoch(other.getCurrentEpoch());
+        }
+        if (!other.getInternalIp().isEmpty()) {
+          internalIp_ = other.internalIp_;
+          bitField0_ |= 0x00000008;
+          onChanged();
+        }
+        if (!other.getPublicIp().isEmpty()) {
+          publicIp_ = other.publicIp_;
+          bitField0_ |= 0x00000010;
+          onChanged();
+        }
+        if (!other.getApiKey().isEmpty()) {
+          apiKey_ = other.apiKey_;
+          bitField0_ |= 0x00000020;
+          onChanged();
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        onChanged();
+        return this;
+      }
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        return true;
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        if (extensionRegistry == null) {
+          throw new java.lang.NullPointerException();
+        }
+        try {
+          boolean done = false;
+          while (!done) {
+            int tag = input.readTag();
+            switch (tag) {
+              case 0:
+                done = true;
+                break;
+              case 10:
+                {
+                  agentVersion_ = input.readStringRequireUtf8();
+                  bitField0_ |= 0x00000001;
+                  break;
+                } // case 10
+              case 18:
+                {
+                  timezone_ = input.readStringRequireUtf8();
+                  bitField0_ |= 0x00000002;
+                  break;
+                } // case 18
+              case 25:
+                {
+                  currentEpoch_ = input.readDouble();
+                  bitField0_ |= 0x00000004;
+                  break;
+                } // case 25
+              case 34:
+                {
+                  internalIp_ = input.readStringRequireUtf8();
+                  bitField0_ |= 0x00000008;
+                  break;
+                } // case 34
+              case 42:
+                {
+                  publicIp_ = input.readStringRequireUtf8();
+                  bitField0_ |= 0x00000010;
+                  break;
+                } // case 42
+              case 50:
+                {
+                  apiKey_ = input.readStringRequireUtf8();
+                  bitField0_ |= 0x00000020;
+                  break;
+                } // case 50
+              default:
+                {
+                  if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                    done = true; // was an endgroup tag
+                  }
+                  break;
+                } // default:
+            } // switch (tag)
+          } // while (!done)
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          throw e.unwrapIOException();
+        } finally {
+          onChanged();
+        } // finally
+        return this;
+      }
+
+      private int bitField0_;
+
+      private java.lang.Object agentVersion_ = "";
+      /**
+       * string agent_version = 1;
+       *
+       * @return The agentVersion.
+       */
+      public java.lang.String getAgentVersion() {
+        java.lang.Object ref = agentVersion_;
+        if (!(ref instanceof java.lang.String)) {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          agentVersion_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * string agent_version = 1;
+       *
+       * @return The bytes for agentVersion.
+       */
+      public com.google.protobuf.ByteString getAgentVersionBytes() {
+        java.lang.Object ref = agentVersion_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          agentVersion_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * string agent_version = 1;
+       *
+       * @param value The agentVersion to set.
+       * @return This builder for chaining.
+       */
+      public Builder setAgentVersion(java.lang.String value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        agentVersion_ = value;
+        bitField0_ |= 0x00000001;
+        onChanged();
+        return this;
+      }
+      /**
+       * string agent_version = 1;
+       *
+       * @return This builder for chaining.
+       */
+      public Builder clearAgentVersion() {
+        agentVersion_ = getDefaultInstance().getAgentVersion();
+        bitField0_ = (bitField0_ & ~0x00000001);
+        onChanged();
+        return this;
+      }
+      /**
+       * string agent_version = 1;
+       *
+       * @param value The bytes for agentVersion to set.
+       * @return This builder for chaining.
+       */
+      public Builder setAgentVersionBytes(com.google.protobuf.ByteString value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        checkByteStringIsUtf8(value);
+        agentVersion_ = value;
+        bitField0_ |= 0x00000001;
+        onChanged();
+        return this;
+      }
+
+      private java.lang.Object timezone_ = "";
+      /**
+       * string timezone = 2;
+       *
+       * @return The timezone.
+       */
+      public java.lang.String getTimezone() {
+        java.lang.Object ref = timezone_;
+        if (!(ref instanceof java.lang.String)) {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          timezone_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * string timezone = 2;
+       *
+       * @return The bytes for timezone.
+       */
+      public com.google.protobuf.ByteString getTimezoneBytes() {
+        java.lang.Object ref = timezone_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          timezone_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * string timezone = 2;
+       *
+       * @param value The timezone to set.
+       * @return This builder for chaining.
+       */
+      public Builder setTimezone(java.lang.String value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        timezone_ = value;
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return this;
+      }
+      /**
+       * string timezone = 2;
+       *
+       * @return This builder for chaining.
+       */
+      public Builder clearTimezone() {
+        timezone_ = getDefaultInstance().getTimezone();
+        bitField0_ = (bitField0_ & ~0x00000002);
+        onChanged();
+        return this;
+      }
+      /**
+       * string timezone = 2;
+       *
+       * @param value The bytes for timezone to set.
+       * @return This builder for chaining.
+       */
+      public Builder setTimezoneBytes(com.google.protobuf.ByteString value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        checkByteStringIsUtf8(value);
+        timezone_ = value;
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return this;
+      }
+
+      private double currentEpoch_;
+      /**
+       * double current_epoch = 3;
+       *
+       * @return The currentEpoch.
+       */
+      @java.lang.Override
+      public double getCurrentEpoch() {
+        return currentEpoch_;
+      }
+      /**
+       * double current_epoch = 3;
+       *
+       * @param value The currentEpoch to set.
+       * @return This builder for chaining.
+       */
+      public Builder setCurrentEpoch(double value) {
+
+        currentEpoch_ = value;
+        bitField0_ |= 0x00000004;
+        onChanged();
+        return this;
+      }
+      /**
+       * double current_epoch = 3;
+       *
+       * @return This builder for chaining.
+       */
+      public Builder clearCurrentEpoch() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        currentEpoch_ = 0D;
+        onChanged();
+        return this;
+      }
+
+      private java.lang.Object internalIp_ = "";
+      /**
+       * string internal_ip = 4;
+       *
+       * @return The internalIp.
+       */
+      public java.lang.String getInternalIp() {
+        java.lang.Object ref = internalIp_;
+        if (!(ref instanceof java.lang.String)) {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          internalIp_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * string internal_ip = 4;
+       *
+       * @return The bytes for internalIp.
+       */
+      public com.google.protobuf.ByteString getInternalIpBytes() {
+        java.lang.Object ref = internalIp_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          internalIp_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * string internal_ip = 4;
+       *
+       * @param value The internalIp to set.
+       * @return This builder for chaining.
+       */
+      public Builder setInternalIp(java.lang.String value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        internalIp_ = value;
+        bitField0_ |= 0x00000008;
+        onChanged();
+        return this;
+      }
+      /**
+       * string internal_ip = 4;
+       *
+       * @return This builder for chaining.
+       */
+      public Builder clearInternalIp() {
+        internalIp_ = getDefaultInstance().getInternalIp();
+        bitField0_ = (bitField0_ & ~0x00000008);
+        onChanged();
+        return this;
+      }
+      /**
+       * string internal_ip = 4;
+       *
+       * @param value The bytes for internalIp to set.
+       * @return This builder for chaining.
+       */
+      public Builder setInternalIpBytes(com.google.protobuf.ByteString value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        checkByteStringIsUtf8(value);
+        internalIp_ = value;
+        bitField0_ |= 0x00000008;
+        onChanged();
+        return this;
+      }
+
+      private java.lang.Object publicIp_ = "";
+      /**
+       * string public_ip = 5;
+       *
+       * @return The publicIp.
+       */
+      public java.lang.String getPublicIp() {
+        java.lang.Object ref = publicIp_;
+        if (!(ref instanceof java.lang.String)) {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          publicIp_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * string public_ip = 5;
+       *
+       * @return The bytes for publicIp.
+       */
+      public com.google.protobuf.ByteString getPublicIpBytes() {
+        java.lang.Object ref = publicIp_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          publicIp_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * string public_ip = 5;
+       *
+       * @param value The publicIp to set.
+       * @return This builder for chaining.
+       */
+      public Builder setPublicIp(java.lang.String value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        publicIp_ = value;
+        bitField0_ |= 0x00000010;
+        onChanged();
+        return this;
+      }
+      /**
+       * string public_ip = 5;
+       *
+       * @return This builder for chaining.
+       */
+      public Builder clearPublicIp() {
+        publicIp_ = getDefaultInstance().getPublicIp();
+        bitField0_ = (bitField0_ & ~0x00000010);
+        onChanged();
+        return this;
+      }
+      /**
+       * string public_ip = 5;
+       *
+       * @param value The bytes for publicIp to set.
+       * @return This builder for chaining.
+       */
+      public Builder setPublicIpBytes(com.google.protobuf.ByteString value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        checkByteStringIsUtf8(value);
+        publicIp_ = value;
+        bitField0_ |= 0x00000010;
+        onChanged();
+        return this;
+      }
+
+      private java.lang.Object apiKey_ = "";
+      /**
+       * string api_key = 6;
+       *
+       * @return The apiKey.
+       */
+      public java.lang.String getApiKey() {
+        java.lang.Object ref = apiKey_;
+        if (!(ref instanceof java.lang.String)) {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          apiKey_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * string api_key = 6;
+       *
+       * @return The bytes for apiKey.
+       */
+      public com.google.protobuf.ByteString getApiKeyBytes() {
+        java.lang.Object ref = apiKey_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          apiKey_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * string api_key = 6;
+       *
+       * @param value The apiKey to set.
+       * @return This builder for chaining.
+       */
+      public Builder setApiKey(java.lang.String value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        apiKey_ = value;
+        bitField0_ |= 0x00000020;
+        onChanged();
+        return this;
+      }
+      /**
+       * string api_key = 6;
+       *
+       * @return This builder for chaining.
+       */
+      public Builder clearApiKey() {
+        apiKey_ = getDefaultInstance().getApiKey();
+        bitField0_ = (bitField0_ & ~0x00000020);
+        onChanged();
+        return this;
+      }
+      /**
+       * string api_key = 6;
+       *
+       * @param value The bytes for apiKey to set.
+       * @return This builder for chaining.
+       */
+      public Builder setApiKeyBytes(com.google.protobuf.ByteString value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        checkByteStringIsUtf8(value);
+        apiKey_ = value;
+        bitField0_ |= 0x00000020;
+        onChanged();
+        return this;
+      }
+
+      @java.lang.Override
+      public final Builder setUnknownFields(
+          final com.google.protobuf.UnknownFieldSet unknownFields) {
+        return super.setUnknownFields(unknownFields);
+      }
+
+      @java.lang.Override
+      public final Builder mergeUnknownFields(
+          final com.google.protobuf.UnknownFieldSet unknownFields) {
+        return super.mergeUnknownFields(unknownFields);
+      }
+
+      // @@protoc_insertion_point(builder_scope:datadog.agentpayload.CommonMetadata)
+    }
+
+    // @@protoc_insertion_point(class_scope:datadog.agentpayload.CommonMetadata)
+    private static final datadog.agentpayload.AgentPayload.CommonMetadata DEFAULT_INSTANCE;
+
+    static {
+      DEFAULT_INSTANCE = new datadog.agentpayload.AgentPayload.CommonMetadata();
+    }
+
+    public static datadog.agentpayload.AgentPayload.CommonMetadata getDefaultInstance() {
+      return DEFAULT_INSTANCE;
+    }
+
+    private static final com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+          @java.lang.Override
+          public CommonMetadata parsePartialFrom(
+              com.google.protobuf.CodedInputStream input,
+              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+              throws com.google.protobuf.InvalidProtocolBufferException {
+            Builder builder = newBuilder();
+            try {
+              builder.mergeFrom(input, extensionRegistry);
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+              throw e.setUnfinishedMessage(builder.buildPartial());
+            } catch (com.google.protobuf.UninitializedMessageException e) {
+              throw e.asInvalidProtocolBufferException()
+                  .setUnfinishedMessage(builder.buildPartial());
+            } catch (java.io.IOException e) {
+              throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                  .setUnfinishedMessage(builder.buildPartial());
+            }
+            return builder.buildPartial();
+          }
+        };
+
+    public static com.google.protobuf.Parser parser() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.CommonMetadata getDefaultInstanceForType() {
+      return DEFAULT_INSTANCE;
+    }
+  }
+
+  public interface MetricPayloadOrBuilder
+      extends
+      // @@protoc_insertion_point(interface_extends:datadog.agentpayload.MetricPayload)
+      com.google.protobuf.MessageOrBuilder {
+
+    /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+    java.util.List getSeriesList();
+    /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+    datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries getSeries(int index);
+    /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+    int getSeriesCount();
+    /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+    java.util.List
+        getSeriesOrBuilderList();
+    /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+    datadog.agentpayload.AgentPayload.MetricPayload.MetricSeriesOrBuilder getSeriesOrBuilder(
+        int index);
+  }
+  /** Protobuf type {@code datadog.agentpayload.MetricPayload} */
+  public static final class MetricPayload extends com.google.protobuf.GeneratedMessageV3
+      implements
+      // @@protoc_insertion_point(message_implements:datadog.agentpayload.MetricPayload)
+      MetricPayloadOrBuilder {
+    private static final long serialVersionUID = 0L;
+    // Use MetricPayload.newBuilder() to construct.
+    private MetricPayload(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+      super(builder);
+    }
+
+    private MetricPayload() {
+      series_ = java.util.Collections.emptyList();
+    }
+
+    @java.lang.Override
+    @SuppressWarnings({"unused"})
+    protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+      return new MetricPayload();
+    }
+
+    public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+      return datadog.agentpayload.AgentPayload
+          .internal_static_datadog_agentpayload_MetricPayload_descriptor;
+    }
+
+    @java.lang.Override
+    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return datadog.agentpayload.AgentPayload
+          .internal_static_datadog_agentpayload_MetricPayload_fieldAccessorTable
+          .ensureFieldAccessorsInitialized(
+              datadog.agentpayload.AgentPayload.MetricPayload.class,
+              datadog.agentpayload.AgentPayload.MetricPayload.Builder.class);
+    }
+
+    /** Protobuf enum {@code datadog.agentpayload.MetricPayload.MetricType} */
+    public enum MetricType implements com.google.protobuf.ProtocolMessageEnum {
+      /** UNSPECIFIED = 0; */
+      UNSPECIFIED(0),
+      /** COUNT = 1; */
+      COUNT(1),
+      /** RATE = 2; */
+      RATE(2),
+      /** GAUGE = 3; */
+      GAUGE(3),
+      UNRECOGNIZED(-1),
+      ;
+
+      /** UNSPECIFIED = 0; */
+      public static final int UNSPECIFIED_VALUE = 0;
+      /** COUNT = 1; */
+      public static final int COUNT_VALUE = 1;
+      /** RATE = 2; */
+      public static final int RATE_VALUE = 2;
+      /** GAUGE = 3; */
+      public static final int GAUGE_VALUE = 3;
+
+      public final int getNumber() {
+        if (this == UNRECOGNIZED) {
+          throw new java.lang.IllegalArgumentException(
+              "Can't get the number of an unknown enum value.");
+        }
+        return value;
+      }
+
+      /**
+       * @param value The numeric wire value of the corresponding enum entry.
+       * @return The enum associated with the given numeric wire value.
+       * @deprecated Use {@link #forNumber(int)} instead.
+       */
+      @java.lang.Deprecated
+      public static MetricType valueOf(int value) {
+        return forNumber(value);
+      }
+
+      /**
+       * @param value The numeric wire value of the corresponding enum entry.
+       * @return The enum associated with the given numeric wire value.
+       */
+      public static MetricType forNumber(int value) {
+        switch (value) {
+          case 0:
+            return UNSPECIFIED;
+          case 1:
+            return COUNT;
+          case 2:
+            return RATE;
+          case 3:
+            return GAUGE;
+          default:
+            return null;
+        }
+      }
+
+      public static com.google.protobuf.Internal.EnumLiteMap internalGetValueMap() {
+        return internalValueMap;
+      }
+
+      private static final com.google.protobuf.Internal.EnumLiteMap internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap() {
+            public MetricType findValueByNumber(int number) {
+              return MetricType.forNumber(number);
+            }
+          };
+
+      public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() {
+        if (this == UNRECOGNIZED) {
+          throw new java.lang.IllegalStateException(
+              "Can't get the descriptor of an unrecognized enum value.");
+        }
+        return getDescriptor().getValues().get(ordinal());
+      }
+
+      public final com.google.protobuf.Descriptors.EnumDescriptor getDescriptorForType() {
+        return getDescriptor();
+      }
+
+      public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() {
+        return datadog.agentpayload.AgentPayload.MetricPayload.getDescriptor()
+            .getEnumTypes()
+            .get(0);
+      }
+
+      private static final MetricType[] VALUES = values();
+
+      public static MetricType valueOf(com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+        if (desc.getType() != getDescriptor()) {
+          throw new java.lang.IllegalArgumentException("EnumValueDescriptor is not for this type.");
+        }
+        if (desc.getIndex() == -1) {
+          return UNRECOGNIZED;
+        }
+        return VALUES[desc.getIndex()];
+      }
+
+      private final int value;
+
+      private MetricType(int value) {
+        this.value = value;
+      }
+
+      // @@protoc_insertion_point(enum_scope:datadog.agentpayload.MetricPayload.MetricType)
+    }
+
+    public interface MetricPointOrBuilder
+        extends
+        // @@protoc_insertion_point(interface_extends:datadog.agentpayload.MetricPayload.MetricPoint)
+        com.google.protobuf.MessageOrBuilder {
+
+      /**
+       *
+       *
+       * 
+       * metric value
+       * 

+       *
+       * double value = 1;
+       *
+       * @return The value.
+       */
+      double getValue();
+
+      /**
+       *
+       *
+       * 
+       * timestamp for this value in seconds since the UNIX epoch
+       * 

+       *
+       * int64 timestamp = 2;
+       *
+       * @return The timestamp.
+       */
+      long getTimestamp();
+    }
+    /** Protobuf type {@code datadog.agentpayload.MetricPayload.MetricPoint} */
+    public static final class MetricPoint extends com.google.protobuf.GeneratedMessageV3
+        implements
+        // @@protoc_insertion_point(message_implements:datadog.agentpayload.MetricPayload.MetricPoint)
+        MetricPointOrBuilder {
+      private static final long serialVersionUID = 0L;
+      // Use MetricPoint.newBuilder() to construct.
+      private MetricPoint(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+        super(builder);
+      }
+
+      private MetricPoint() {}
+
+      @java.lang.Override
+      @SuppressWarnings({"unused"})
+      protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+        return new MetricPoint();
+      }
+
+      public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_MetricPayload_MetricPoint_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_MetricPayload_MetricPoint_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.class,
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder.class);
+      }
+
+      public static final int VALUE_FIELD_NUMBER = 1;
+      private double value_ = 0D;
+      /**
+       *
+       *
+       * 
+       * metric value
+       * 

+       *
+       * double value = 1;
+       *
+       * @return The value.
+       */
+      @java.lang.Override
+      public double getValue() {
+        return value_;
+      }
+
+      public static final int TIMESTAMP_FIELD_NUMBER = 2;
+      private long timestamp_ = 0L;
+      /**
+       *
+       *
+       * 
+       * timestamp for this value in seconds since the UNIX epoch
+       * 

+       *
+       * int64 timestamp = 2;
+       *
+       * @return The timestamp.
+       */
+      @java.lang.Override
+      public long getTimestamp() {
+        return timestamp_;
+      }
+
+      private byte memoizedIsInitialized = -1;
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        byte isInitialized = memoizedIsInitialized;
+        if (isInitialized == 1) return true;
+        if (isInitialized == 0) return false;
+
+        memoizedIsInitialized = 1;
+        return true;
+      }
+
+      @java.lang.Override
+      public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+        if (java.lang.Double.doubleToRawLongBits(value_) != 0) {
+          output.writeDouble(1, value_);
+        }
+        if (timestamp_ != 0L) {
+          output.writeInt64(2, timestamp_);
+        }
+        getUnknownFields().writeTo(output);
+      }
+
+      @java.lang.Override
+      public int getSerializedSize() {
+        int size = memoizedSize;
+        if (size != -1) return size;
+
+        size = 0;
+        if (java.lang.Double.doubleToRawLongBits(value_) != 0) {
+          size += com.google.protobuf.CodedOutputStream.computeDoubleSize(1, value_);
+        }
+        if (timestamp_ != 0L) {
+          size += com.google.protobuf.CodedOutputStream.computeInt64Size(2, timestamp_);
+        }
+        size += getUnknownFields().getSerializedSize();
+        memoizedSize = size;
+        return size;
+      }
+
+      @java.lang.Override
+      public boolean equals(final java.lang.Object obj) {
+        if (obj == this) {
+          return true;
+        }
+        if (!(obj instanceof datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint)) {
+          return super.equals(obj);
+        }
+        datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint other =
+            (datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint) obj;
+
+        if (java.lang.Double.doubleToLongBits(getValue())
+            != java.lang.Double.doubleToLongBits(other.getValue())) return false;
+        if (getTimestamp() != other.getTimestamp()) return false;
+        if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+        return true;
+      }
+
+      @java.lang.Override
+      public int hashCode() {
+        if (memoizedHashCode != 0) {
+          return memoizedHashCode;
+        }
+        int hash = 41;
+        hash = (19 * hash) + getDescriptor().hashCode();
+        hash = (37 * hash) + VALUE_FIELD_NUMBER;
+        hash =
+            (53 * hash)
+                + com.google.protobuf.Internal.hashLong(
+                    java.lang.Double.doubleToLongBits(getValue()));
+        hash = (37 * hash) + TIMESTAMP_FIELD_NUMBER;
+        hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTimestamp());
+        hash = (29 * hash) + getUnknownFields().hashCode();
+        memoizedHashCode = hash;
+        return hash;
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseFrom(
+          java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseFrom(
+          java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseFrom(
+          com.google.protobuf.ByteString data)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseFrom(
+          com.google.protobuf.ByteString data,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseFrom(
+          byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseFrom(
+          byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseFrom(
+          java.io.InputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseFrom(
+          java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseDelimitedFrom(
+          java.io.InputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseDelimitedFrom(
+          java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseFrom(
+          com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint parseFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      @java.lang.Override
+      public Builder newBuilderForType() {
+        return newBuilder();
+      }
+
+      public static Builder newBuilder() {
+        return DEFAULT_INSTANCE.toBuilder();
+      }
+
+      public static Builder newBuilder(
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint prototype) {
+        return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+      }
+
+      @java.lang.Override
+      public Builder toBuilder() {
+        return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+      }
+
+      @java.lang.Override
+      protected Builder newBuilderForType(
+          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        Builder builder = new Builder(parent);
+        return builder;
+      }
+      /** Protobuf type {@code datadog.agentpayload.MetricPayload.MetricPoint} */
+      public static final class Builder
+          extends com.google.protobuf.GeneratedMessageV3.Builder
+          implements
+          // @@protoc_insertion_point(builder_implements:datadog.agentpayload.MetricPayload.MetricPoint)
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricPointOrBuilder {
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_MetricPayload_MetricPoint_descriptor;
+        }
+
+        @java.lang.Override
+        protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+            internalGetFieldAccessorTable() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_MetricPayload_MetricPoint_fieldAccessorTable
+              .ensureFieldAccessorsInitialized(
+                  datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.class,
+                  datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder.class);
+        }
+
+        // Construct using datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.newBuilder()
+        private Builder() {}
+
+        private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          super(parent);
+        }
+
+        @java.lang.Override
+        public Builder clear() {
+          super.clear();
+          bitField0_ = 0;
+          value_ = 0D;
+          timestamp_ = 0L;
+          return this;
+        }
+
+        @java.lang.Override
+        public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_MetricPayload_MetricPoint_descriptor;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint
+            getDefaultInstanceForType() {
+          return datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.getDefaultInstance();
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint build() {
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint result = buildPartial();
+          if (!result.isInitialized()) {
+            throw newUninitializedMessageException(result);
+          }
+          return result;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint buildPartial() {
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint result =
+              new datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint(this);
+          if (bitField0_ != 0) {
+            buildPartial0(result);
+          }
+          onBuilt();
+          return result;
+        }
+
+        private void buildPartial0(
+            datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint result) {
+          int from_bitField0_ = bitField0_;
+          if (((from_bitField0_ & 0x00000001) != 0)) {
+            result.value_ = value_;
+          }
+          if (((from_bitField0_ & 0x00000002) != 0)) {
+            result.timestamp_ = timestamp_;
+          }
+        }
+
+        @java.lang.Override
+        public Builder clone() {
+          return super.clone();
+        }
+
+        @java.lang.Override
+        public Builder setField(
+            com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+          return super.setField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+          return super.clearField(field);
+        }
+
+        @java.lang.Override
+        public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+          return super.clearOneof(oneof);
+        }
+
+        @java.lang.Override
+        public Builder setRepeatedField(
+            com.google.protobuf.Descriptors.FieldDescriptor field,
+            int index,
+            java.lang.Object value) {
+          return super.setRepeatedField(field, index, value);
+        }
+
+        @java.lang.Override
+        public Builder addRepeatedField(
+            com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+          return super.addRepeatedField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(com.google.protobuf.Message other) {
+          if (other instanceof datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint) {
+            return mergeFrom((datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint) other);
+          } else {
+            super.mergeFrom(other);
+            return this;
+          }
+        }
+
+        public Builder mergeFrom(
+            datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint other) {
+          if (other
+              == datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.getDefaultInstance())
+            return this;
+          if (other.getValue() != 0D) {
+            setValue(other.getValue());
+          }
+          if (other.getTimestamp() != 0L) {
+            setTimestamp(other.getTimestamp());
+          }
+          this.mergeUnknownFields(other.getUnknownFields());
+          onChanged();
+          return this;
+        }
+
+        @java.lang.Override
+        public final boolean isInitialized() {
+          return true;
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+          if (extensionRegistry == null) {
+            throw new java.lang.NullPointerException();
+          }
+          try {
+            boolean done = false;
+            while (!done) {
+              int tag = input.readTag();
+              switch (tag) {
+                case 0:
+                  done = true;
+                  break;
+                case 9:
+                  {
+                    value_ = input.readDouble();
+                    bitField0_ |= 0x00000001;
+                    break;
+                  } // case 9
+                case 16:
+                  {
+                    timestamp_ = input.readInt64();
+                    bitField0_ |= 0x00000002;
+                    break;
+                  } // case 16
+                default:
+                  {
+                    if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                      done = true; // was an endgroup tag
+                    }
+                    break;
+                  } // default:
+              } // switch (tag)
+            } // while (!done)
+          } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+            throw e.unwrapIOException();
+          } finally {
+            onChanged();
+          } // finally
+          return this;
+        }
+
+        private int bitField0_;
+
+        private double value_;
+        /**
+         *
+         *
+         * 
+         * metric value
+         * 

+         *
+         * double value = 1;
+         *
+         * @return The value.
+         */
+        @java.lang.Override
+        public double getValue() {
+          return value_;
+        }
+        /**
+         *
+         *
+         * 
+         * metric value
+         * 

+         *
+         * double value = 1;
+         *
+         * @param value The value to set.
+         * @return This builder for chaining.
+         */
+        public Builder setValue(double value) {
+
+          value_ = value;
+          bitField0_ |= 0x00000001;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * metric value
+         * 

+         *
+         * double value = 1;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearValue() {
+          bitField0_ = (bitField0_ & ~0x00000001);
+          value_ = 0D;
+          onChanged();
+          return this;
+        }
+
+        private long timestamp_;
+        /**
+         *
+         *
+         * 
+         * timestamp for this value in seconds since the UNIX epoch
+         * 

+         *
+         * int64 timestamp = 2;
+         *
+         * @return The timestamp.
+         */
+        @java.lang.Override
+        public long getTimestamp() {
+          return timestamp_;
+        }
+        /**
+         *
+         *
+         * 
+         * timestamp for this value in seconds since the UNIX epoch
+         * 

+         *
+         * int64 timestamp = 2;
+         *
+         * @param value The timestamp to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTimestamp(long value) {
+
+          timestamp_ = value;
+          bitField0_ |= 0x00000002;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * timestamp for this value in seconds since the UNIX epoch
+         * 

+         *
+         * int64 timestamp = 2;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearTimestamp() {
+          bitField0_ = (bitField0_ & ~0x00000002);
+          timestamp_ = 0L;
+          onChanged();
+          return this;
+        }
+
+        @java.lang.Override
+        public final Builder setUnknownFields(
+            final com.google.protobuf.UnknownFieldSet unknownFields) {
+          return super.setUnknownFields(unknownFields);
+        }
+
+        @java.lang.Override
+        public final Builder mergeUnknownFields(
+            final com.google.protobuf.UnknownFieldSet unknownFields) {
+          return super.mergeUnknownFields(unknownFields);
+        }
+
+        // @@protoc_insertion_point(builder_scope:datadog.agentpayload.MetricPayload.MetricPoint)
+      }
+
+      // @@protoc_insertion_point(class_scope:datadog.agentpayload.MetricPayload.MetricPoint)
+      private static final datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint
+          DEFAULT_INSTANCE;
+
+      static {
+        DEFAULT_INSTANCE = new datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint();
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint
+          getDefaultInstance() {
+        return DEFAULT_INSTANCE;
+      }
+
+      private static final com.google.protobuf.Parser PARSER =
+          new com.google.protobuf.AbstractParser() {
+            @java.lang.Override
+            public MetricPoint parsePartialFrom(
+                com.google.protobuf.CodedInputStream input,
+                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                throws com.google.protobuf.InvalidProtocolBufferException {
+              Builder builder = newBuilder();
+              try {
+                builder.mergeFrom(input, extensionRegistry);
+              } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                throw e.setUnfinishedMessage(builder.buildPartial());
+              } catch (com.google.protobuf.UninitializedMessageException e) {
+                throw e.asInvalidProtocolBufferException()
+                    .setUnfinishedMessage(builder.buildPartial());
+              } catch (java.io.IOException e) {
+                throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                    .setUnfinishedMessage(builder.buildPartial());
+              }
+              return builder.buildPartial();
+            }
+          };
+
+      public static com.google.protobuf.Parser parser() {
+        return PARSER;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Parser getParserForType() {
+        return PARSER;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint
+          getDefaultInstanceForType() {
+        return DEFAULT_INSTANCE;
+      }
+    }
+
+    public interface ResourceOrBuilder
+        extends
+        // @@protoc_insertion_point(interface_extends:datadog.agentpayload.MetricPayload.Resource)
+        com.google.protobuf.MessageOrBuilder {
+
+      /**
+       * string type = 1;
+       *
+       * @return The type.
+       */
+      java.lang.String getType();
+      /**
+       * string type = 1;
+       *
+       * @return The bytes for type.
+       */
+      com.google.protobuf.ByteString getTypeBytes();
+
+      /**
+       * string name = 2;
+       *
+       * @return The name.
+       */
+      java.lang.String getName();
+      /**
+       * string name = 2;
+       *
+       * @return The bytes for name.
+       */
+      com.google.protobuf.ByteString getNameBytes();
+    }
+    /** Protobuf type {@code datadog.agentpayload.MetricPayload.Resource} */
+    public static final class Resource extends com.google.protobuf.GeneratedMessageV3
+        implements
+        // @@protoc_insertion_point(message_implements:datadog.agentpayload.MetricPayload.Resource)
+        ResourceOrBuilder {
+      private static final long serialVersionUID = 0L;
+      // Use Resource.newBuilder() to construct.
+      private Resource(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+        super(builder);
+      }
+
+      private Resource() {
+        type_ = "";
+        name_ = "";
+      }
+
+      @java.lang.Override
+      @SuppressWarnings({"unused"})
+      protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+        return new Resource();
+      }
+
+      public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_MetricPayload_Resource_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_MetricPayload_Resource_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                datadog.agentpayload.AgentPayload.MetricPayload.Resource.class,
+                datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder.class);
+      }
+
+      public static final int TYPE_FIELD_NUMBER = 1;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object type_ = "";
+      /**
+       * string type = 1;
+       *
+       * @return The type.
+       */
+      @java.lang.Override
+      public java.lang.String getType() {
+        java.lang.Object ref = type_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          type_ = s;
+          return s;
+        }
+      }
+      /**
+       * string type = 1;
+       *
+       * @return The bytes for type.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getTypeBytes() {
+        java.lang.Object ref = type_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          type_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int NAME_FIELD_NUMBER = 2;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object name_ = "";
+      /**
+       * string name = 2;
+       *
+       * @return The name.
+       */
+      @java.lang.Override
+      public java.lang.String getName() {
+        java.lang.Object ref = name_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          name_ = s;
+          return s;
+        }
+      }
+      /**
+       * string name = 2;
+       *
+       * @return The bytes for name.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getNameBytes() {
+        java.lang.Object ref = name_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          name_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      private byte memoizedIsInitialized = -1;
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        byte isInitialized = memoizedIsInitialized;
+        if (isInitialized == 1) return true;
+        if (isInitialized == 0) return false;
+
+        memoizedIsInitialized = 1;
+        return true;
+      }
+
+      @java.lang.Override
+      public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(type_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 1, type_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(name_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 2, name_);
+        }
+        getUnknownFields().writeTo(output);
+      }
+
+      @java.lang.Override
+      public int getSerializedSize() {
+        int size = memoizedSize;
+        if (size != -1) return size;
+
+        size = 0;
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(type_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, type_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(name_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, name_);
+        }
+        size += getUnknownFields().getSerializedSize();
+        memoizedSize = size;
+        return size;
+      }
+
+      @java.lang.Override
+      public boolean equals(final java.lang.Object obj) {
+        if (obj == this) {
+          return true;
+        }
+        if (!(obj instanceof datadog.agentpayload.AgentPayload.MetricPayload.Resource)) {
+          return super.equals(obj);
+        }
+        datadog.agentpayload.AgentPayload.MetricPayload.Resource other =
+            (datadog.agentpayload.AgentPayload.MetricPayload.Resource) obj;
+
+        if (!getType().equals(other.getType())) return false;
+        if (!getName().equals(other.getName())) return false;
+        if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+        return true;
+      }
+
+      @java.lang.Override
+      public int hashCode() {
+        if (memoizedHashCode != 0) {
+          return memoizedHashCode;
+        }
+        int hash = 41;
+        hash = (19 * hash) + getDescriptor().hashCode();
+        hash = (37 * hash) + TYPE_FIELD_NUMBER;
+        hash = (53 * hash) + getType().hashCode();
+        hash = (37 * hash) + NAME_FIELD_NUMBER;
+        hash = (53 * hash) + getName().hashCode();
+        hash = (29 * hash) + getUnknownFields().hashCode();
+        memoizedHashCode = hash;
+        return hash;
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseFrom(
+          java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseFrom(
+          java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseFrom(
+          com.google.protobuf.ByteString data)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseFrom(
+          com.google.protobuf.ByteString data,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseFrom(byte[] data)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseFrom(
+          byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseFrom(
+          java.io.InputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseFrom(
+          java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseDelimitedFrom(
+          java.io.InputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseDelimitedFrom(
+          java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseFrom(
+          com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource parseFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      @java.lang.Override
+      public Builder newBuilderForType() {
+        return newBuilder();
+      }
+
+      public static Builder newBuilder() {
+        return DEFAULT_INSTANCE.toBuilder();
+      }
+
+      public static Builder newBuilder(
+          datadog.agentpayload.AgentPayload.MetricPayload.Resource prototype) {
+        return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+      }
+
+      @java.lang.Override
+      public Builder toBuilder() {
+        return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+      }
+
+      @java.lang.Override
+      protected Builder newBuilderForType(
+          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        Builder builder = new Builder(parent);
+        return builder;
+      }
+      /** Protobuf type {@code datadog.agentpayload.MetricPayload.Resource} */
+      public static final class Builder
+          extends com.google.protobuf.GeneratedMessageV3.Builder
+          implements
+          // @@protoc_insertion_point(builder_implements:datadog.agentpayload.MetricPayload.Resource)
+          datadog.agentpayload.AgentPayload.MetricPayload.ResourceOrBuilder {
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_MetricPayload_Resource_descriptor;
+        }
+
+        @java.lang.Override
+        protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+            internalGetFieldAccessorTable() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_MetricPayload_Resource_fieldAccessorTable
+              .ensureFieldAccessorsInitialized(
+                  datadog.agentpayload.AgentPayload.MetricPayload.Resource.class,
+                  datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder.class);
+        }
+
+        // Construct using datadog.agentpayload.AgentPayload.MetricPayload.Resource.newBuilder()
+        private Builder() {}
+
+        private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          super(parent);
+        }
+
+        @java.lang.Override
+        public Builder clear() {
+          super.clear();
+          bitField0_ = 0;
+          type_ = "";
+          name_ = "";
+          return this;
+        }
+
+        @java.lang.Override
+        public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_MetricPayload_Resource_descriptor;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.MetricPayload.Resource
+            getDefaultInstanceForType() {
+          return datadog.agentpayload.AgentPayload.MetricPayload.Resource.getDefaultInstance();
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.MetricPayload.Resource build() {
+          datadog.agentpayload.AgentPayload.MetricPayload.Resource result = buildPartial();
+          if (!result.isInitialized()) {
+            throw newUninitializedMessageException(result);
+          }
+          return result;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.MetricPayload.Resource buildPartial() {
+          datadog.agentpayload.AgentPayload.MetricPayload.Resource result =
+              new datadog.agentpayload.AgentPayload.MetricPayload.Resource(this);
+          if (bitField0_ != 0) {
+            buildPartial0(result);
+          }
+          onBuilt();
+          return result;
+        }
+
+        private void buildPartial0(
+            datadog.agentpayload.AgentPayload.MetricPayload.Resource result) {
+          int from_bitField0_ = bitField0_;
+          if (((from_bitField0_ & 0x00000001) != 0)) {
+            result.type_ = type_;
+          }
+          if (((from_bitField0_ & 0x00000002) != 0)) {
+            result.name_ = name_;
+          }
+        }
+
+        @java.lang.Override
+        public Builder clone() {
+          return super.clone();
+        }
+
+        @java.lang.Override
+        public Builder setField(
+            com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+          return super.setField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+          return super.clearField(field);
+        }
+
+        @java.lang.Override
+        public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+          return super.clearOneof(oneof);
+        }
+
+        @java.lang.Override
+        public Builder setRepeatedField(
+            com.google.protobuf.Descriptors.FieldDescriptor field,
+            int index,
+            java.lang.Object value) {
+          return super.setRepeatedField(field, index, value);
+        }
+
+        @java.lang.Override
+        public Builder addRepeatedField(
+            com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+          return super.addRepeatedField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(com.google.protobuf.Message other) {
+          if (other instanceof datadog.agentpayload.AgentPayload.MetricPayload.Resource) {
+            return mergeFrom((datadog.agentpayload.AgentPayload.MetricPayload.Resource) other);
+          } else {
+            super.mergeFrom(other);
+            return this;
+          }
+        }
+
+        public Builder mergeFrom(datadog.agentpayload.AgentPayload.MetricPayload.Resource other) {
+          if (other
+              == datadog.agentpayload.AgentPayload.MetricPayload.Resource.getDefaultInstance())
+            return this;
+          if (!other.getType().isEmpty()) {
+            type_ = other.type_;
+            bitField0_ |= 0x00000001;
+            onChanged();
+          }
+          if (!other.getName().isEmpty()) {
+            name_ = other.name_;
+            bitField0_ |= 0x00000002;
+            onChanged();
+          }
+          this.mergeUnknownFields(other.getUnknownFields());
+          onChanged();
+          return this;
+        }
+
+        @java.lang.Override
+        public final boolean isInitialized() {
+          return true;
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+          if (extensionRegistry == null) {
+            throw new java.lang.NullPointerException();
+          }
+          try {
+            boolean done = false;
+            while (!done) {
+              int tag = input.readTag();
+              switch (tag) {
+                case 0:
+                  done = true;
+                  break;
+                case 10:
+                  {
+                    type_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000001;
+                    break;
+                  } // case 10
+                case 18:
+                  {
+                    name_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000002;
+                    break;
+                  } // case 18
+                default:
+                  {
+                    if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                      done = true; // was an endgroup tag
+                    }
+                    break;
+                  } // default:
+              } // switch (tag)
+            } // while (!done)
+          } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+            throw e.unwrapIOException();
+          } finally {
+            onChanged();
+          } // finally
+          return this;
+        }
+
+        private int bitField0_;
+
+        private java.lang.Object type_ = "";
+        /**
+         * string type = 1;
+         *
+         * @return The type.
+         */
+        public java.lang.String getType() {
+          java.lang.Object ref = type_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            type_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string type = 1;
+         *
+         * @return The bytes for type.
+         */
+        public com.google.protobuf.ByteString getTypeBytes() {
+          java.lang.Object ref = type_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            type_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string type = 1;
+         *
+         * @param value The type to set.
+         * @return This builder for chaining.
+         */
+        public Builder setType(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          type_ = value;
+          bitField0_ |= 0x00000001;
+          onChanged();
+          return this;
+        }
+        /**
+         * string type = 1;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearType() {
+          type_ = getDefaultInstance().getType();
+          bitField0_ = (bitField0_ & ~0x00000001);
+          onChanged();
+          return this;
+        }
+        /**
+         * string type = 1;
+         *
+         * @param value The bytes for type to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTypeBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          type_ = value;
+          bitField0_ |= 0x00000001;
+          onChanged();
+          return this;
+        }
+
+        private java.lang.Object name_ = "";
+        /**
+         * string name = 2;
+         *
+         * @return The name.
+         */
+        public java.lang.String getName() {
+          java.lang.Object ref = name_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            name_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string name = 2;
+         *
+         * @return The bytes for name.
+         */
+        public com.google.protobuf.ByteString getNameBytes() {
+          java.lang.Object ref = name_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            name_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string name = 2;
+         *
+         * @param value The name to set.
+         * @return This builder for chaining.
+         */
+        public Builder setName(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          name_ = value;
+          bitField0_ |= 0x00000002;
+          onChanged();
+          return this;
+        }
+        /**
+         * string name = 2;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearName() {
+          name_ = getDefaultInstance().getName();
+          bitField0_ = (bitField0_ & ~0x00000002);
+          onChanged();
+          return this;
+        }
+        /**
+         * string name = 2;
+         *
+         * @param value The bytes for name to set.
+         * @return This builder for chaining.
+         */
+        public Builder setNameBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          name_ = value;
+          bitField0_ |= 0x00000002;
+          onChanged();
+          return this;
+        }
+
+        @java.lang.Override
+        public final Builder setUnknownFields(
+            final com.google.protobuf.UnknownFieldSet unknownFields) {
+          return super.setUnknownFields(unknownFields);
+        }
+
+        @java.lang.Override
+        public final Builder mergeUnknownFields(
+            final com.google.protobuf.UnknownFieldSet unknownFields) {
+          return super.mergeUnknownFields(unknownFields);
+        }
+
+        // @@protoc_insertion_point(builder_scope:datadog.agentpayload.MetricPayload.Resource)
+      }
+
+      // @@protoc_insertion_point(class_scope:datadog.agentpayload.MetricPayload.Resource)
+      private static final datadog.agentpayload.AgentPayload.MetricPayload.Resource
+          DEFAULT_INSTANCE;
+
+      static {
+        DEFAULT_INSTANCE = new datadog.agentpayload.AgentPayload.MetricPayload.Resource();
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.Resource getDefaultInstance() {
+        return DEFAULT_INSTANCE;
+      }
+
+      private static final com.google.protobuf.Parser PARSER =
+          new com.google.protobuf.AbstractParser() {
+            @java.lang.Override
+            public Resource parsePartialFrom(
+                com.google.protobuf.CodedInputStream input,
+                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                throws com.google.protobuf.InvalidProtocolBufferException {
+              Builder builder = newBuilder();
+              try {
+                builder.mergeFrom(input, extensionRegistry);
+              } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                throw e.setUnfinishedMessage(builder.buildPartial());
+              } catch (com.google.protobuf.UninitializedMessageException e) {
+                throw e.asInvalidProtocolBufferException()
+                    .setUnfinishedMessage(builder.buildPartial());
+              } catch (java.io.IOException e) {
+                throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                    .setUnfinishedMessage(builder.buildPartial());
+              }
+              return builder.buildPartial();
+            }
+          };
+
+      public static com.google.protobuf.Parser parser() {
+        return PARSER;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Parser getParserForType() {
+        return PARSER;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload.Resource getDefaultInstanceForType() {
+        return DEFAULT_INSTANCE;
+      }
+    }
+
+    public interface MetricSeriesOrBuilder
+        extends
+        // @@protoc_insertion_point(interface_extends:datadog.agentpayload.MetricPayload.MetricSeries)
+        com.google.protobuf.MessageOrBuilder {
+
+      /**
+       *
+       *
+       * 
+       * Resources this series applies to; include at least
+       * { type="host", name=<hostname> }
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+       */
+      java.util.List getResourcesList();
+      /**
+       *
+       *
+       * 
+       * Resources this series applies to; include at least
+       * { type="host", name=<hostname> }
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+       */
+      datadog.agentpayload.AgentPayload.MetricPayload.Resource getResources(int index);
+      /**
+       *
+       *
+       * 
+       * Resources this series applies to; include at least
+       * { type="host", name=<hostname> }
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+       */
+      int getResourcesCount();
+      /**
+       *
+       *
+       * 
+       * Resources this series applies to; include at least
+       * { type="host", name=<hostname> }
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+       */
+      java.util.List
+          getResourcesOrBuilderList();
+      /**
+       *
+       *
+       * 
+       * Resources this series applies to; include at least
+       * { type="host", name=<hostname> }
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+       */
+      datadog.agentpayload.AgentPayload.MetricPayload.ResourceOrBuilder getResourcesOrBuilder(
+          int index);
+
+      /**
+       *
+       *
+       * 
+       * metric name
+       * 

+       *
+       * string metric = 2;
+       *
+       * @return The metric.
+       */
+      java.lang.String getMetric();
+      /**
+       *
+       *
+       * 
+       * metric name
+       * 

+       *
+       * string metric = 2;
+       *
+       * @return The bytes for metric.
+       */
+      com.google.protobuf.ByteString getMetricBytes();
+
+      /**
+       *
+       *
+       * 
+       * tags for this metric
+       * 

+       *
+       * repeated string tags = 3;
+       *
+       * @return A list containing the tags.
+       */
+      java.util.List getTagsList();
+      /**
+       *
+       *
+       * 
+       * tags for this metric
+       * 

+       *
+       * repeated string tags = 3;
+       *
+       * @return The count of tags.
+       */
+      int getTagsCount();
+      /**
+       *
+       *
+       * 
+       * tags for this metric
+       * 

+       *
+       * repeated string tags = 3;
+       *
+       * @param index The index of the element to return.
+       * @return The tags at the given index.
+       */
+      java.lang.String getTags(int index);
+      /**
+       *
+       *
+       * 
+       * tags for this metric
+       * 

+       *
+       * repeated string tags = 3;
+       *
+       * @param index The index of the value to return.
+       * @return The bytes of the tags at the given index.
+       */
+      com.google.protobuf.ByteString getTagsBytes(int index);
+
+      /**
+       *
+       *
+       * 
+       * data points for this metric
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+       */
+      java.util.List getPointsList();
+      /**
+       *
+       *
+       * 
+       * data points for this metric
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+       */
+      datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint getPoints(int index);
+      /**
+       *
+       *
+       * 
+       * data points for this metric
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+       */
+      int getPointsCount();
+      /**
+       *
+       *
+       * 
+       * data points for this metric
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+       */
+      java.util.List
+          getPointsOrBuilderList();
+      /**
+       *
+       *
+       * 
+       * data points for this metric
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+       */
+      datadog.agentpayload.AgentPayload.MetricPayload.MetricPointOrBuilder getPointsOrBuilder(
+          int index);
+
+      /**
+       *
+       *
+       * 
+       * type of metric
+       * 

+       *
+       * .datadog.agentpayload.MetricPayload.MetricType type = 5;
+       *
+       * @return The enum numeric value on the wire for type.
+       */
+      int getTypeValue();
+      /**
+       *
+       *
+       * 
+       * type of metric
+       * 

+       *
+       * .datadog.agentpayload.MetricPayload.MetricType type = 5;
+       *
+       * @return The type.
+       */
+      datadog.agentpayload.AgentPayload.MetricPayload.MetricType getType();
+
+      /**
+       *
+       *
+       * 
+       * metric unit name
+       * 

+       *
+       * string unit = 6;
+       *
+       * @return The unit.
+       */
+      java.lang.String getUnit();
+      /**
+       *
+       *
+       * 
+       * metric unit name
+       * 

+       *
+       * string unit = 6;
+       *
+       * @return The bytes for unit.
+       */
+      com.google.protobuf.ByteString getUnitBytes();
+
+      /**
+       *
+       *
+       * 
+       * source of this metric (check name, etc.)
+       * 

+       *
+       * string source_type_name = 7;
+       *
+       * @return The sourceTypeName.
+       */
+      java.lang.String getSourceTypeName();
+      /**
+       *
+       *
+       * 
+       * source of this metric (check name, etc.)
+       * 

+       *
+       * string source_type_name = 7;
+       *
+       * @return The bytes for sourceTypeName.
+       */
+      com.google.protobuf.ByteString getSourceTypeNameBytes();
+
+      /**
+       *
+       *
+       * 
+       * interval, in seconds, between samples of this metric
+       * 

+       *
+       * int64 interval = 8;
+       *
+       * @return The interval.
+       */
+      long getInterval();
+    }
+    /** Protobuf type {@code datadog.agentpayload.MetricPayload.MetricSeries} */
+    public static final class MetricSeries extends com.google.protobuf.GeneratedMessageV3
+        implements
+        // @@protoc_insertion_point(message_implements:datadog.agentpayload.MetricPayload.MetricSeries)
+        MetricSeriesOrBuilder {
+      private static final long serialVersionUID = 0L;
+      // Use MetricSeries.newBuilder() to construct.
+      private MetricSeries(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+        super(builder);
+      }
+
+      private MetricSeries() {
+        resources_ = java.util.Collections.emptyList();
+        metric_ = "";
+        tags_ = com.google.protobuf.LazyStringArrayList.emptyList();
+        points_ = java.util.Collections.emptyList();
+        type_ = 0;
+        unit_ = "";
+        sourceTypeName_ = "";
+      }
+
+      @java.lang.Override
+      @SuppressWarnings({"unused"})
+      protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+        return new MetricSeries();
+      }
+
+      public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_MetricPayload_MetricSeries_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_MetricPayload_MetricSeries_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.class,
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder.class);
+      }
+
+      public static final int RESOURCES_FIELD_NUMBER = 1;
+
+      @SuppressWarnings("serial")
+      private java.util.List resources_;
+      /**
+       *
+       *
+       * 
+       * Resources this series applies to; include at least
+       * { type="host", name=<hostname> }
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+       */
+      @java.lang.Override
+      public java.util.List
+          getResourcesList() {
+        return resources_;
+      }
+      /**
+       *
+       *
+       * 
+       * Resources this series applies to; include at least
+       * { type="host", name=<hostname> }
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+       */
+      @java.lang.Override
+      public java.util.List<
+              ? extends datadog.agentpayload.AgentPayload.MetricPayload.ResourceOrBuilder>
+          getResourcesOrBuilderList() {
+        return resources_;
+      }
+      /**
+       *
+       *
+       * 
+       * Resources this series applies to; include at least
+       * { type="host", name=<hostname> }
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+       */
+      @java.lang.Override
+      public int getResourcesCount() {
+        return resources_.size();
+      }
+      /**
+       *
+       *
+       * 
+       * Resources this series applies to; include at least
+       * { type="host", name=<hostname> }
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+       */
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload.Resource getResources(int index) {
+        return resources_.get(index);
+      }
+      /**
+       *
+       *
+       * 
+       * Resources this series applies to; include at least
+       * { type="host", name=<hostname> }
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+       */
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload.ResourceOrBuilder
+          getResourcesOrBuilder(int index) {
+        return resources_.get(index);
+      }
+
+      public static final int METRIC_FIELD_NUMBER = 2;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object metric_ = "";
+      /**
+       *
+       *
+       * 
+       * metric name
+       * 

+       *
+       * string metric = 2;
+       *
+       * @return The metric.
+       */
+      @java.lang.Override
+      public java.lang.String getMetric() {
+        java.lang.Object ref = metric_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          metric_ = s;
+          return s;
+        }
+      }
+      /**
+       *
+       *
+       * 
+       * metric name
+       * 

+       *
+       * string metric = 2;
+       *
+       * @return The bytes for metric.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getMetricBytes() {
+        java.lang.Object ref = metric_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          metric_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int TAGS_FIELD_NUMBER = 3;
+
+      @SuppressWarnings("serial")
+      private com.google.protobuf.LazyStringArrayList tags_ =
+          com.google.protobuf.LazyStringArrayList.emptyList();
+      /**
+       *
+       *
+       * 
+       * tags for this metric
+       * 

+       *
+       * repeated string tags = 3;
+       *
+       * @return A list containing the tags.
+       */
+      public com.google.protobuf.ProtocolStringList getTagsList() {
+        return tags_;
+      }
+      /**
+       *
+       *
+       * 
+       * tags for this metric
+       * 

+       *
+       * repeated string tags = 3;
+       *
+       * @return The count of tags.
+       */
+      public int getTagsCount() {
+        return tags_.size();
+      }
+      /**
+       *
+       *
+       * 
+       * tags for this metric
+       * 

+       *
+       * repeated string tags = 3;
+       *
+       * @param index The index of the element to return.
+       * @return The tags at the given index.
+       */
+      public java.lang.String getTags(int index) {
+        return tags_.get(index);
+      }
+      /**
+       *
+       *
+       * 
+       * tags for this metric
+       * 

+       *
+       * repeated string tags = 3;
+       *
+       * @param index The index of the value to return.
+       * @return The bytes of the tags at the given index.
+       */
+      public com.google.protobuf.ByteString getTagsBytes(int index) {
+        return tags_.getByteString(index);
+      }
+
+      public static final int POINTS_FIELD_NUMBER = 4;
+
+      @SuppressWarnings("serial")
+      private java.util.List points_;
+      /**
+       *
+       *
+       * 
+       * data points for this metric
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+       */
+      @java.lang.Override
+      public java.util.List
+          getPointsList() {
+        return points_;
+      }
+      /**
+       *
+       *
+       * 
+       * data points for this metric
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+       */
+      @java.lang.Override
+      public java.util.List<
+              ? extends datadog.agentpayload.AgentPayload.MetricPayload.MetricPointOrBuilder>
+          getPointsOrBuilderList() {
+        return points_;
+      }
+      /**
+       *
+       *
+       * 
+       * data points for this metric
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+       */
+      @java.lang.Override
+      public int getPointsCount() {
+        return points_.size();
+      }
+      /**
+       *
+       *
+       * 
+       * data points for this metric
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+       */
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint getPoints(int index) {
+        return points_.get(index);
+      }
+      /**
+       *
+       *
+       * 
+       * data points for this metric
+       * 

+       *
+       * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+       */
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload.MetricPointOrBuilder
+          getPointsOrBuilder(int index) {
+        return points_.get(index);
+      }
+
+      public static final int TYPE_FIELD_NUMBER = 5;
+      private int type_ = 0;
+      /**
+       *
+       *
+       * 
+       * type of metric
+       * 

+       *
+       * .datadog.agentpayload.MetricPayload.MetricType type = 5;
+       *
+       * @return The enum numeric value on the wire for type.
+       */
+      @java.lang.Override
+      public int getTypeValue() {
+        return type_;
+      }
+      /**
+       *
+       *
+       * 
+       * type of metric
+       * 

+       *
+       * .datadog.agentpayload.MetricPayload.MetricType type = 5;
+       *
+       * @return The type.
+       */
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload.MetricType getType() {
+        datadog.agentpayload.AgentPayload.MetricPayload.MetricType result =
+            datadog.agentpayload.AgentPayload.MetricPayload.MetricType.forNumber(type_);
+        return result == null
+            ? datadog.agentpayload.AgentPayload.MetricPayload.MetricType.UNRECOGNIZED
+            : result;
+      }
+
+      public static final int UNIT_FIELD_NUMBER = 6;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object unit_ = "";
+      /**
+       *
+       *
+       * 
+       * metric unit name
+       * 

+       *
+       * string unit = 6;
+       *
+       * @return The unit.
+       */
+      @java.lang.Override
+      public java.lang.String getUnit() {
+        java.lang.Object ref = unit_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          unit_ = s;
+          return s;
+        }
+      }
+      /**
+       *
+       *
+       * 
+       * metric unit name
+       * 

+       *
+       * string unit = 6;
+       *
+       * @return The bytes for unit.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getUnitBytes() {
+        java.lang.Object ref = unit_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          unit_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int SOURCE_TYPE_NAME_FIELD_NUMBER = 7;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object sourceTypeName_ = "";
+      /**
+       *
+       *
+       * 
+       * source of this metric (check name, etc.)
+       * 

+       *
+       * string source_type_name = 7;
+       *
+       * @return The sourceTypeName.
+       */
+      @java.lang.Override
+      public java.lang.String getSourceTypeName() {
+        java.lang.Object ref = sourceTypeName_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          sourceTypeName_ = s;
+          return s;
+        }
+      }
+      /**
+       *
+       *
+       * 
+       * source of this metric (check name, etc.)
+       * 

+       *
+       * string source_type_name = 7;
+       *
+       * @return The bytes for sourceTypeName.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getSourceTypeNameBytes() {
+        java.lang.Object ref = sourceTypeName_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          sourceTypeName_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int INTERVAL_FIELD_NUMBER = 8;
+      private long interval_ = 0L;
+      /**
+       *
+       *
+       * 
+       * interval, in seconds, between samples of this metric
+       * 

+       *
+       * int64 interval = 8;
+       *
+       * @return The interval.
+       */
+      @java.lang.Override
+      public long getInterval() {
+        return interval_;
+      }
+
+      private byte memoizedIsInitialized = -1;
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        byte isInitialized = memoizedIsInitialized;
+        if (isInitialized == 1) return true;
+        if (isInitialized == 0) return false;
+
+        memoizedIsInitialized = 1;
+        return true;
+      }
+
+      @java.lang.Override
+      public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+        for (int i = 0; i < resources_.size(); i++) {
+          output.writeMessage(1, resources_.get(i));
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(metric_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 2, metric_);
+        }
+        for (int i = 0; i < tags_.size(); i++) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 3, tags_.getRaw(i));
+        }
+        for (int i = 0; i < points_.size(); i++) {
+          output.writeMessage(4, points_.get(i));
+        }
+        if (type_
+            != datadog.agentpayload.AgentPayload.MetricPayload.MetricType.UNSPECIFIED.getNumber()) {
+          output.writeEnum(5, type_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(unit_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 6, unit_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(sourceTypeName_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 7, sourceTypeName_);
+        }
+        if (interval_ != 0L) {
+          output.writeInt64(8, interval_);
+        }
+        getUnknownFields().writeTo(output);
+      }
+
+      @java.lang.Override
+      public int getSerializedSize() {
+        int size = memoizedSize;
+        if (size != -1) return size;
+
+        size = 0;
+        for (int i = 0; i < resources_.size(); i++) {
+          size += com.google.protobuf.CodedOutputStream.computeMessageSize(1, resources_.get(i));
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(metric_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, metric_);
+        }
+        {
+          int dataSize = 0;
+          for (int i = 0; i < tags_.size(); i++) {
+            dataSize += computeStringSizeNoTag(tags_.getRaw(i));
+          }
+          size += dataSize;
+          size += 1 * getTagsList().size();
+        }
+        for (int i = 0; i < points_.size(); i++) {
+          size += com.google.protobuf.CodedOutputStream.computeMessageSize(4, points_.get(i));
+        }
+        if (type_
+            != datadog.agentpayload.AgentPayload.MetricPayload.MetricType.UNSPECIFIED.getNumber()) {
+          size += com.google.protobuf.CodedOutputStream.computeEnumSize(5, type_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(unit_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(6, unit_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(sourceTypeName_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(7, sourceTypeName_);
+        }
+        if (interval_ != 0L) {
+          size += com.google.protobuf.CodedOutputStream.computeInt64Size(8, interval_);
+        }
+        size += getUnknownFields().getSerializedSize();
+        memoizedSize = size;
+        return size;
+      }
+
+      @java.lang.Override
+      public boolean equals(final java.lang.Object obj) {
+        if (obj == this) {
+          return true;
+        }
+        if (!(obj instanceof datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries)) {
+          return super.equals(obj);
+        }
+        datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries other =
+            (datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries) obj;
+
+        if (!getResourcesList().equals(other.getResourcesList())) return false;
+        if (!getMetric().equals(other.getMetric())) return false;
+        if (!getTagsList().equals(other.getTagsList())) return false;
+        if (!getPointsList().equals(other.getPointsList())) return false;
+        if (type_ != other.type_) return false;
+        if (!getUnit().equals(other.getUnit())) return false;
+        if (!getSourceTypeName().equals(other.getSourceTypeName())) return false;
+        if (getInterval() != other.getInterval()) return false;
+        if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+        return true;
+      }
+
+      @java.lang.Override
+      public int hashCode() {
+        if (memoizedHashCode != 0) {
+          return memoizedHashCode;
+        }
+        int hash = 41;
+        hash = (19 * hash) + getDescriptor().hashCode();
+        if (getResourcesCount() > 0) {
+          hash = (37 * hash) + RESOURCES_FIELD_NUMBER;
+          hash = (53 * hash) + getResourcesList().hashCode();
+        }
+        hash = (37 * hash) + METRIC_FIELD_NUMBER;
+        hash = (53 * hash) + getMetric().hashCode();
+        if (getTagsCount() > 0) {
+          hash = (37 * hash) + TAGS_FIELD_NUMBER;
+          hash = (53 * hash) + getTagsList().hashCode();
+        }
+        if (getPointsCount() > 0) {
+          hash = (37 * hash) + POINTS_FIELD_NUMBER;
+          hash = (53 * hash) + getPointsList().hashCode();
+        }
+        hash = (37 * hash) + TYPE_FIELD_NUMBER;
+        hash = (53 * hash) + type_;
+        hash = (37 * hash) + UNIT_FIELD_NUMBER;
+        hash = (53 * hash) + getUnit().hashCode();
+        hash = (37 * hash) + SOURCE_TYPE_NAME_FIELD_NUMBER;
+        hash = (53 * hash) + getSourceTypeName().hashCode();
+        hash = (37 * hash) + INTERVAL_FIELD_NUMBER;
+        hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getInterval());
+        hash = (29 * hash) + getUnknownFields().hashCode();
+        memoizedHashCode = hash;
+        return hash;
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseFrom(
+          java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseFrom(
+          java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseFrom(
+          com.google.protobuf.ByteString data)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseFrom(
+          com.google.protobuf.ByteString data,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseFrom(
+          byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseFrom(
+          byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseFrom(
+          java.io.InputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseFrom(
+          java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseDelimitedFrom(
+          java.io.InputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseDelimitedFrom(
+          java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseFrom(
+          com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries parseFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      @java.lang.Override
+      public Builder newBuilderForType() {
+        return newBuilder();
+      }
+
+      public static Builder newBuilder() {
+        return DEFAULT_INSTANCE.toBuilder();
+      }
+
+      public static Builder newBuilder(
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries prototype) {
+        return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+      }
+
+      @java.lang.Override
+      public Builder toBuilder() {
+        return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+      }
+
+      @java.lang.Override
+      protected Builder newBuilderForType(
+          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        Builder builder = new Builder(parent);
+        return builder;
+      }
+      /** Protobuf type {@code datadog.agentpayload.MetricPayload.MetricSeries} */
+      public static final class Builder
+          extends com.google.protobuf.GeneratedMessageV3.Builder
+          implements
+          // @@protoc_insertion_point(builder_implements:datadog.agentpayload.MetricPayload.MetricSeries)
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricSeriesOrBuilder {
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_MetricPayload_MetricSeries_descriptor;
+        }
+
+        @java.lang.Override
+        protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+            internalGetFieldAccessorTable() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_MetricPayload_MetricSeries_fieldAccessorTable
+              .ensureFieldAccessorsInitialized(
+                  datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.class,
+                  datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder.class);
+        }
+
+        // Construct using datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.newBuilder()
+        private Builder() {}
+
+        private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          super(parent);
+        }
+
+        @java.lang.Override
+        public Builder clear() {
+          super.clear();
+          bitField0_ = 0;
+          if (resourcesBuilder_ == null) {
+            resources_ = java.util.Collections.emptyList();
+          } else {
+            resources_ = null;
+            resourcesBuilder_.clear();
+          }
+          bitField0_ = (bitField0_ & ~0x00000001);
+          metric_ = "";
+          tags_ = com.google.protobuf.LazyStringArrayList.emptyList();
+          if (pointsBuilder_ == null) {
+            points_ = java.util.Collections.emptyList();
+          } else {
+            points_ = null;
+            pointsBuilder_.clear();
+          }
+          bitField0_ = (bitField0_ & ~0x00000008);
+          type_ = 0;
+          unit_ = "";
+          sourceTypeName_ = "";
+          interval_ = 0L;
+          return this;
+        }
+
+        @java.lang.Override
+        public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_MetricPayload_MetricSeries_descriptor;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries
+            getDefaultInstanceForType() {
+          return datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.getDefaultInstance();
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries build() {
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries result = buildPartial();
+          if (!result.isInitialized()) {
+            throw newUninitializedMessageException(result);
+          }
+          return result;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries buildPartial() {
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries result =
+              new datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries(this);
+          buildPartialRepeatedFields(result);
+          if (bitField0_ != 0) {
+            buildPartial0(result);
+          }
+          onBuilt();
+          return result;
+        }
+
+        private void buildPartialRepeatedFields(
+            datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries result) {
+          if (resourcesBuilder_ == null) {
+            if (((bitField0_ & 0x00000001) != 0)) {
+              resources_ = java.util.Collections.unmodifiableList(resources_);
+              bitField0_ = (bitField0_ & ~0x00000001);
+            }
+            result.resources_ = resources_;
+          } else {
+            result.resources_ = resourcesBuilder_.build();
+          }
+          if (pointsBuilder_ == null) {
+            if (((bitField0_ & 0x00000008) != 0)) {
+              points_ = java.util.Collections.unmodifiableList(points_);
+              bitField0_ = (bitField0_ & ~0x00000008);
+            }
+            result.points_ = points_;
+          } else {
+            result.points_ = pointsBuilder_.build();
+          }
+        }
+
+        private void buildPartial0(
+            datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries result) {
+          int from_bitField0_ = bitField0_;
+          if (((from_bitField0_ & 0x00000002) != 0)) {
+            result.metric_ = metric_;
+          }
+          if (((from_bitField0_ & 0x00000004) != 0)) {
+            tags_.makeImmutable();
+            result.tags_ = tags_;
+          }
+          if (((from_bitField0_ & 0x00000010) != 0)) {
+            result.type_ = type_;
+          }
+          if (((from_bitField0_ & 0x00000020) != 0)) {
+            result.unit_ = unit_;
+          }
+          if (((from_bitField0_ & 0x00000040) != 0)) {
+            result.sourceTypeName_ = sourceTypeName_;
+          }
+          if (((from_bitField0_ & 0x00000080) != 0)) {
+            result.interval_ = interval_;
+          }
+        }
+
+        @java.lang.Override
+        public Builder clone() {
+          return super.clone();
+        }
+
+        @java.lang.Override
+        public Builder setField(
+            com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+          return super.setField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+          return super.clearField(field);
+        }
+
+        @java.lang.Override
+        public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+          return super.clearOneof(oneof);
+        }
+
+        @java.lang.Override
+        public Builder setRepeatedField(
+            com.google.protobuf.Descriptors.FieldDescriptor field,
+            int index,
+            java.lang.Object value) {
+          return super.setRepeatedField(field, index, value);
+        }
+
+        @java.lang.Override
+        public Builder addRepeatedField(
+            com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+          return super.addRepeatedField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(com.google.protobuf.Message other) {
+          if (other instanceof datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries) {
+            return mergeFrom((datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries) other);
+          } else {
+            super.mergeFrom(other);
+            return this;
+          }
+        }
+
+        public Builder mergeFrom(
+            datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries other) {
+          if (other
+              == datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.getDefaultInstance())
+            return this;
+          if (resourcesBuilder_ == null) {
+            if (!other.resources_.isEmpty()) {
+              if (resources_.isEmpty()) {
+                resources_ = other.resources_;
+                bitField0_ = (bitField0_ & ~0x00000001);
+              } else {
+                ensureResourcesIsMutable();
+                resources_.addAll(other.resources_);
+              }
+              onChanged();
+            }
+          } else {
+            if (!other.resources_.isEmpty()) {
+              if (resourcesBuilder_.isEmpty()) {
+                resourcesBuilder_.dispose();
+                resourcesBuilder_ = null;
+                resources_ = other.resources_;
+                bitField0_ = (bitField0_ & ~0x00000001);
+                resourcesBuilder_ =
+                    com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders
+                        ? getResourcesFieldBuilder()
+                        : null;
+              } else {
+                resourcesBuilder_.addAllMessages(other.resources_);
+              }
+            }
+          }
+          if (!other.getMetric().isEmpty()) {
+            metric_ = other.metric_;
+            bitField0_ |= 0x00000002;
+            onChanged();
+          }
+          if (!other.tags_.isEmpty()) {
+            if (tags_.isEmpty()) {
+              tags_ = other.tags_;
+              bitField0_ |= 0x00000004;
+            } else {
+              ensureTagsIsMutable();
+              tags_.addAll(other.tags_);
+            }
+            onChanged();
+          }
+          if (pointsBuilder_ == null) {
+            if (!other.points_.isEmpty()) {
+              if (points_.isEmpty()) {
+                points_ = other.points_;
+                bitField0_ = (bitField0_ & ~0x00000008);
+              } else {
+                ensurePointsIsMutable();
+                points_.addAll(other.points_);
+              }
+              onChanged();
+            }
+          } else {
+            if (!other.points_.isEmpty()) {
+              if (pointsBuilder_.isEmpty()) {
+                pointsBuilder_.dispose();
+                pointsBuilder_ = null;
+                points_ = other.points_;
+                bitField0_ = (bitField0_ & ~0x00000008);
+                pointsBuilder_ =
+                    com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders
+                        ? getPointsFieldBuilder()
+                        : null;
+              } else {
+                pointsBuilder_.addAllMessages(other.points_);
+              }
+            }
+          }
+          if (other.type_ != 0) {
+            setTypeValue(other.getTypeValue());
+          }
+          if (!other.getUnit().isEmpty()) {
+            unit_ = other.unit_;
+            bitField0_ |= 0x00000020;
+            onChanged();
+          }
+          if (!other.getSourceTypeName().isEmpty()) {
+            sourceTypeName_ = other.sourceTypeName_;
+            bitField0_ |= 0x00000040;
+            onChanged();
+          }
+          if (other.getInterval() != 0L) {
+            setInterval(other.getInterval());
+          }
+          this.mergeUnknownFields(other.getUnknownFields());
+          onChanged();
+          return this;
+        }
+
+        @java.lang.Override
+        public final boolean isInitialized() {
+          return true;
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+          if (extensionRegistry == null) {
+            throw new java.lang.NullPointerException();
+          }
+          try {
+            boolean done = false;
+            while (!done) {
+              int tag = input.readTag();
+              switch (tag) {
+                case 0:
+                  done = true;
+                  break;
+                case 10:
+                  {
+                    datadog.agentpayload.AgentPayload.MetricPayload.Resource m =
+                        input.readMessage(
+                            datadog.agentpayload.AgentPayload.MetricPayload.Resource.parser(),
+                            extensionRegistry);
+                    if (resourcesBuilder_ == null) {
+                      ensureResourcesIsMutable();
+                      resources_.add(m);
+                    } else {
+                      resourcesBuilder_.addMessage(m);
+                    }
+                    break;
+                  } // case 10
+                case 18:
+                  {
+                    metric_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000002;
+                    break;
+                  } // case 18
+                case 26:
+                  {
+                    java.lang.String s = input.readStringRequireUtf8();
+                    ensureTagsIsMutable();
+                    tags_.add(s);
+                    break;
+                  } // case 26
+                case 34:
+                  {
+                    datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint m =
+                        input.readMessage(
+                            datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.parser(),
+                            extensionRegistry);
+                    if (pointsBuilder_ == null) {
+                      ensurePointsIsMutable();
+                      points_.add(m);
+                    } else {
+                      pointsBuilder_.addMessage(m);
+                    }
+                    break;
+                  } // case 34
+                case 40:
+                  {
+                    type_ = input.readEnum();
+                    bitField0_ |= 0x00000010;
+                    break;
+                  } // case 40
+                case 50:
+                  {
+                    unit_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000020;
+                    break;
+                  } // case 50
+                case 58:
+                  {
+                    sourceTypeName_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000040;
+                    break;
+                  } // case 58
+                case 64:
+                  {
+                    interval_ = input.readInt64();
+                    bitField0_ |= 0x00000080;
+                    break;
+                  } // case 64
+                default:
+                  {
+                    if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                      done = true; // was an endgroup tag
+                    }
+                    break;
+                  } // default:
+              } // switch (tag)
+            } // while (!done)
+          } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+            throw e.unwrapIOException();
+          } finally {
+            onChanged();
+          } // finally
+          return this;
+        }
+
+        private int bitField0_;
+
+        private java.util.List
+            resources_ = java.util.Collections.emptyList();
+
+        private void ensureResourcesIsMutable() {
+          if (!((bitField0_ & 0x00000001) != 0)) {
+            resources_ =
+                new java.util.ArrayList(
+                    resources_);
+            bitField0_ |= 0x00000001;
+          }
+        }
+
+        private com.google.protobuf.RepeatedFieldBuilderV3<
+                datadog.agentpayload.AgentPayload.MetricPayload.Resource,
+                datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder,
+                datadog.agentpayload.AgentPayload.MetricPayload.ResourceOrBuilder>
+            resourcesBuilder_;
+
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public java.util.List
+            getResourcesList() {
+          if (resourcesBuilder_ == null) {
+            return java.util.Collections.unmodifiableList(resources_);
+          } else {
+            return resourcesBuilder_.getMessageList();
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public int getResourcesCount() {
+          if (resourcesBuilder_ == null) {
+            return resources_.size();
+          } else {
+            return resourcesBuilder_.getCount();
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public datadog.agentpayload.AgentPayload.MetricPayload.Resource getResources(int index) {
+          if (resourcesBuilder_ == null) {
+            return resources_.get(index);
+          } else {
+            return resourcesBuilder_.getMessage(index);
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public Builder setResources(
+            int index, datadog.agentpayload.AgentPayload.MetricPayload.Resource value) {
+          if (resourcesBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensureResourcesIsMutable();
+            resources_.set(index, value);
+            onChanged();
+          } else {
+            resourcesBuilder_.setMessage(index, value);
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public Builder setResources(
+            int index,
+            datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder builderForValue) {
+          if (resourcesBuilder_ == null) {
+            ensureResourcesIsMutable();
+            resources_.set(index, builderForValue.build());
+            onChanged();
+          } else {
+            resourcesBuilder_.setMessage(index, builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public Builder addResources(
+            datadog.agentpayload.AgentPayload.MetricPayload.Resource value) {
+          if (resourcesBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensureResourcesIsMutable();
+            resources_.add(value);
+            onChanged();
+          } else {
+            resourcesBuilder_.addMessage(value);
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public Builder addResources(
+            int index, datadog.agentpayload.AgentPayload.MetricPayload.Resource value) {
+          if (resourcesBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensureResourcesIsMutable();
+            resources_.add(index, value);
+            onChanged();
+          } else {
+            resourcesBuilder_.addMessage(index, value);
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public Builder addResources(
+            datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder builderForValue) {
+          if (resourcesBuilder_ == null) {
+            ensureResourcesIsMutable();
+            resources_.add(builderForValue.build());
+            onChanged();
+          } else {
+            resourcesBuilder_.addMessage(builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public Builder addResources(
+            int index,
+            datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder builderForValue) {
+          if (resourcesBuilder_ == null) {
+            ensureResourcesIsMutable();
+            resources_.add(index, builderForValue.build());
+            onChanged();
+          } else {
+            resourcesBuilder_.addMessage(index, builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public Builder addAllResources(
+            java.lang.Iterable
+                values) {
+          if (resourcesBuilder_ == null) {
+            ensureResourcesIsMutable();
+            com.google.protobuf.AbstractMessageLite.Builder.addAll(values, resources_);
+            onChanged();
+          } else {
+            resourcesBuilder_.addAllMessages(values);
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public Builder clearResources() {
+          if (resourcesBuilder_ == null) {
+            resources_ = java.util.Collections.emptyList();
+            bitField0_ = (bitField0_ & ~0x00000001);
+            onChanged();
+          } else {
+            resourcesBuilder_.clear();
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public Builder removeResources(int index) {
+          if (resourcesBuilder_ == null) {
+            ensureResourcesIsMutable();
+            resources_.remove(index);
+            onChanged();
+          } else {
+            resourcesBuilder_.remove(index);
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder getResourcesBuilder(
+            int index) {
+          return getResourcesFieldBuilder().getBuilder(index);
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public datadog.agentpayload.AgentPayload.MetricPayload.ResourceOrBuilder
+            getResourcesOrBuilder(int index) {
+          if (resourcesBuilder_ == null) {
+            return resources_.get(index);
+          } else {
+            return resourcesBuilder_.getMessageOrBuilder(index);
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public java.util.List<
+                ? extends datadog.agentpayload.AgentPayload.MetricPayload.ResourceOrBuilder>
+            getResourcesOrBuilderList() {
+          if (resourcesBuilder_ != null) {
+            return resourcesBuilder_.getMessageOrBuilderList();
+          } else {
+            return java.util.Collections.unmodifiableList(resources_);
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder
+            addResourcesBuilder() {
+          return getResourcesFieldBuilder()
+              .addBuilder(
+                  datadog.agentpayload.AgentPayload.MetricPayload.Resource.getDefaultInstance());
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder addResourcesBuilder(
+            int index) {
+          return getResourcesFieldBuilder()
+              .addBuilder(
+                  index,
+                  datadog.agentpayload.AgentPayload.MetricPayload.Resource.getDefaultInstance());
+        }
+        /**
+         *
+         *
+         * 
+         * Resources this series applies to; include at least
+         * { type="host", name=<hostname> }
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.Resource resources = 1;
+         */
+        public java.util.List
+            getResourcesBuilderList() {
+          return getResourcesFieldBuilder().getBuilderList();
+        }
+
+        private com.google.protobuf.RepeatedFieldBuilderV3<
+                datadog.agentpayload.AgentPayload.MetricPayload.Resource,
+                datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder,
+                datadog.agentpayload.AgentPayload.MetricPayload.ResourceOrBuilder>
+            getResourcesFieldBuilder() {
+          if (resourcesBuilder_ == null) {
+            resourcesBuilder_ =
+                new com.google.protobuf.RepeatedFieldBuilderV3<
+                    datadog.agentpayload.AgentPayload.MetricPayload.Resource,
+                    datadog.agentpayload.AgentPayload.MetricPayload.Resource.Builder,
+                    datadog.agentpayload.AgentPayload.MetricPayload.ResourceOrBuilder>(
+                    resources_,
+                    ((bitField0_ & 0x00000001) != 0),
+                    getParentForChildren(),
+                    isClean());
+            resources_ = null;
+          }
+          return resourcesBuilder_;
+        }
+
+        private java.lang.Object metric_ = "";
+        /**
+         *
+         *
+         * 
+         * metric name
+         * 

+         *
+         * string metric = 2;
+         *
+         * @return The metric.
+         */
+        public java.lang.String getMetric() {
+          java.lang.Object ref = metric_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            metric_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * metric name
+         * 

+         *
+         * string metric = 2;
+         *
+         * @return The bytes for metric.
+         */
+        public com.google.protobuf.ByteString getMetricBytes() {
+          java.lang.Object ref = metric_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            metric_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * metric name
+         * 

+         *
+         * string metric = 2;
+         *
+         * @param value The metric to set.
+         * @return This builder for chaining.
+         */
+        public Builder setMetric(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          metric_ = value;
+          bitField0_ |= 0x00000002;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * metric name
+         * 

+         *
+         * string metric = 2;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearMetric() {
+          metric_ = getDefaultInstance().getMetric();
+          bitField0_ = (bitField0_ & ~0x00000002);
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * metric name
+         * 

+         *
+         * string metric = 2;
+         *
+         * @param value The bytes for metric to set.
+         * @return This builder for chaining.
+         */
+        public Builder setMetricBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          metric_ = value;
+          bitField0_ |= 0x00000002;
+          onChanged();
+          return this;
+        }
+
+        private com.google.protobuf.LazyStringArrayList tags_ =
+            com.google.protobuf.LazyStringArrayList.emptyList();
+
+        private void ensureTagsIsMutable() {
+          if (!tags_.isModifiable()) {
+            tags_ = new com.google.protobuf.LazyStringArrayList(tags_);
+          }
+          bitField0_ |= 0x00000004;
+        }
+        /**
+         *
+         *
+         * 
+         * tags for this metric
+         * 

+         *
+         * repeated string tags = 3;
+         *
+         * @return A list containing the tags.
+         */
+        public com.google.protobuf.ProtocolStringList getTagsList() {
+          tags_.makeImmutable();
+          return tags_;
+        }
+        /**
+         *
+         *
+         * 
+         * tags for this metric
+         * 

+         *
+         * repeated string tags = 3;
+         *
+         * @return The count of tags.
+         */
+        public int getTagsCount() {
+          return tags_.size();
+        }
+        /**
+         *
+         *
+         * 
+         * tags for this metric
+         * 

+         *
+         * repeated string tags = 3;
+         *
+         * @param index The index of the element to return.
+         * @return The tags at the given index.
+         */
+        public java.lang.String getTags(int index) {
+          return tags_.get(index);
+        }
+        /**
+         *
+         *
+         * 
+         * tags for this metric
+         * 

+         *
+         * repeated string tags = 3;
+         *
+         * @param index The index of the value to return.
+         * @return The bytes of the tags at the given index.
+         */
+        public com.google.protobuf.ByteString getTagsBytes(int index) {
+          return tags_.getByteString(index);
+        }
+        /**
+         *
+         *
+         * 
+         * tags for this metric
+         * 

+         *
+         * repeated string tags = 3;
+         *
+         * @param index The index to set the value at.
+         * @param value The tags to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTags(int index, java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTagsIsMutable();
+          tags_.set(index, value);
+          bitField0_ |= 0x00000004;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * tags for this metric
+         * 

+         *
+         * repeated string tags = 3;
+         *
+         * @param value The tags to add.
+         * @return This builder for chaining.
+         */
+        public Builder addTags(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTagsIsMutable();
+          tags_.add(value);
+          bitField0_ |= 0x00000004;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * tags for this metric
+         * 

+         *
+         * repeated string tags = 3;
+         *
+         * @param values The tags to add.
+         * @return This builder for chaining.
+         */
+        public Builder addAllTags(java.lang.Iterable values) {
+          ensureTagsIsMutable();
+          com.google.protobuf.AbstractMessageLite.Builder.addAll(values, tags_);
+          bitField0_ |= 0x00000004;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * tags for this metric
+         * 

+         *
+         * repeated string tags = 3;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearTags() {
+          tags_ = com.google.protobuf.LazyStringArrayList.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000004);
+          ;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * tags for this metric
+         * 

+         *
+         * repeated string tags = 3;
+         *
+         * @param value The bytes of the tags to add.
+         * @return This builder for chaining.
+         */
+        public Builder addTagsBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          ensureTagsIsMutable();
+          tags_.add(value);
+          bitField0_ |= 0x00000004;
+          onChanged();
+          return this;
+        }
+
+        private java.util.List
+            points_ = java.util.Collections.emptyList();
+
+        private void ensurePointsIsMutable() {
+          if (!((bitField0_ & 0x00000008) != 0)) {
+            points_ =
+                new java.util.ArrayList<
+                    datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint>(points_);
+            bitField0_ |= 0x00000008;
+          }
+        }
+
+        private com.google.protobuf.RepeatedFieldBuilderV3<
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint,
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder,
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricPointOrBuilder>
+            pointsBuilder_;
+
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public java.util.List
+            getPointsList() {
+          if (pointsBuilder_ == null) {
+            return java.util.Collections.unmodifiableList(points_);
+          } else {
+            return pointsBuilder_.getMessageList();
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public int getPointsCount() {
+          if (pointsBuilder_ == null) {
+            return points_.size();
+          } else {
+            return pointsBuilder_.getCount();
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint getPoints(int index) {
+          if (pointsBuilder_ == null) {
+            return points_.get(index);
+          } else {
+            return pointsBuilder_.getMessage(index);
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public Builder setPoints(
+            int index, datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint value) {
+          if (pointsBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensurePointsIsMutable();
+            points_.set(index, value);
+            onChanged();
+          } else {
+            pointsBuilder_.setMessage(index, value);
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public Builder setPoints(
+            int index,
+            datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder builderForValue) {
+          if (pointsBuilder_ == null) {
+            ensurePointsIsMutable();
+            points_.set(index, builderForValue.build());
+            onChanged();
+          } else {
+            pointsBuilder_.setMessage(index, builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public Builder addPoints(
+            datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint value) {
+          if (pointsBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensurePointsIsMutable();
+            points_.add(value);
+            onChanged();
+          } else {
+            pointsBuilder_.addMessage(value);
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public Builder addPoints(
+            int index, datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint value) {
+          if (pointsBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensurePointsIsMutable();
+            points_.add(index, value);
+            onChanged();
+          } else {
+            pointsBuilder_.addMessage(index, value);
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public Builder addPoints(
+            datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder builderForValue) {
+          if (pointsBuilder_ == null) {
+            ensurePointsIsMutable();
+            points_.add(builderForValue.build());
+            onChanged();
+          } else {
+            pointsBuilder_.addMessage(builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public Builder addPoints(
+            int index,
+            datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder builderForValue) {
+          if (pointsBuilder_ == null) {
+            ensurePointsIsMutable();
+            points_.add(index, builderForValue.build());
+            onChanged();
+          } else {
+            pointsBuilder_.addMessage(index, builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public Builder addAllPoints(
+            java.lang.Iterable<
+                    ? extends datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint>
+                values) {
+          if (pointsBuilder_ == null) {
+            ensurePointsIsMutable();
+            com.google.protobuf.AbstractMessageLite.Builder.addAll(values, points_);
+            onChanged();
+          } else {
+            pointsBuilder_.addAllMessages(values);
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public Builder clearPoints() {
+          if (pointsBuilder_ == null) {
+            points_ = java.util.Collections.emptyList();
+            bitField0_ = (bitField0_ & ~0x00000008);
+            onChanged();
+          } else {
+            pointsBuilder_.clear();
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public Builder removePoints(int index) {
+          if (pointsBuilder_ == null) {
+            ensurePointsIsMutable();
+            points_.remove(index);
+            onChanged();
+          } else {
+            pointsBuilder_.remove(index);
+          }
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder getPointsBuilder(
+            int index) {
+          return getPointsFieldBuilder().getBuilder(index);
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricPointOrBuilder
+            getPointsOrBuilder(int index) {
+          if (pointsBuilder_ == null) {
+            return points_.get(index);
+          } else {
+            return pointsBuilder_.getMessageOrBuilder(index);
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public java.util.List<
+                ? extends datadog.agentpayload.AgentPayload.MetricPayload.MetricPointOrBuilder>
+            getPointsOrBuilderList() {
+          if (pointsBuilder_ != null) {
+            return pointsBuilder_.getMessageOrBuilderList();
+          } else {
+            return java.util.Collections.unmodifiableList(points_);
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder
+            addPointsBuilder() {
+          return getPointsFieldBuilder()
+              .addBuilder(
+                  datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.getDefaultInstance());
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder addPointsBuilder(
+            int index) {
+          return getPointsFieldBuilder()
+              .addBuilder(
+                  index,
+                  datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.getDefaultInstance());
+        }
+        /**
+         *
+         *
+         * 
+         * data points for this metric
+         * 

+         *
+         * repeated .datadog.agentpayload.MetricPayload.MetricPoint points = 4;
+         */
+        public java.util.List
+            getPointsBuilderList() {
+          return getPointsFieldBuilder().getBuilderList();
+        }
+
+        private com.google.protobuf.RepeatedFieldBuilderV3<
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint,
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder,
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricPointOrBuilder>
+            getPointsFieldBuilder() {
+          if (pointsBuilder_ == null) {
+            pointsBuilder_ =
+                new com.google.protobuf.RepeatedFieldBuilderV3<
+                    datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint,
+                    datadog.agentpayload.AgentPayload.MetricPayload.MetricPoint.Builder,
+                    datadog.agentpayload.AgentPayload.MetricPayload.MetricPointOrBuilder>(
+                    points_, ((bitField0_ & 0x00000008) != 0), getParentForChildren(), isClean());
+            points_ = null;
+          }
+          return pointsBuilder_;
+        }
+
+        private int type_ = 0;
+        /**
+         *
+         *
+         * 
+         * type of metric
+         * 

+         *
+         * .datadog.agentpayload.MetricPayload.MetricType type = 5;
+         *
+         * @return The enum numeric value on the wire for type.
+         */
+        @java.lang.Override
+        public int getTypeValue() {
+          return type_;
+        }
+        /**
+         *
+         *
+         * 
+         * type of metric
+         * 

+         *
+         * .datadog.agentpayload.MetricPayload.MetricType type = 5;
+         *
+         * @param value The enum numeric value on the wire for type to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTypeValue(int value) {
+          type_ = value;
+          bitField0_ |= 0x00000010;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * type of metric
+         * 

+         *
+         * .datadog.agentpayload.MetricPayload.MetricType type = 5;
+         *
+         * @return The type.
+         */
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.MetricPayload.MetricType getType() {
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricType result =
+              datadog.agentpayload.AgentPayload.MetricPayload.MetricType.forNumber(type_);
+          return result == null
+              ? datadog.agentpayload.AgentPayload.MetricPayload.MetricType.UNRECOGNIZED
+              : result;
+        }
+        /**
+         *
+         *
+         * 
+         * type of metric
+         * 

+         *
+         * .datadog.agentpayload.MetricPayload.MetricType type = 5;
+         *
+         * @param value The type to set.
+         * @return This builder for chaining.
+         */
+        public Builder setType(datadog.agentpayload.AgentPayload.MetricPayload.MetricType value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          bitField0_ |= 0x00000010;
+          type_ = value.getNumber();
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * type of metric
+         * 

+         *
+         * .datadog.agentpayload.MetricPayload.MetricType type = 5;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearType() {
+          bitField0_ = (bitField0_ & ~0x00000010);
+          type_ = 0;
+          onChanged();
+          return this;
+        }
+
+        private java.lang.Object unit_ = "";
+        /**
+         *
+         *
+         * 
+         * metric unit name
+         * 

+         *
+         * string unit = 6;
+         *
+         * @return The unit.
+         */
+        public java.lang.String getUnit() {
+          java.lang.Object ref = unit_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            unit_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * metric unit name
+         * 

+         *
+         * string unit = 6;
+         *
+         * @return The bytes for unit.
+         */
+        public com.google.protobuf.ByteString getUnitBytes() {
+          java.lang.Object ref = unit_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            unit_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * metric unit name
+         * 

+         *
+         * string unit = 6;
+         *
+         * @param value The unit to set.
+         * @return This builder for chaining.
+         */
+        public Builder setUnit(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          unit_ = value;
+          bitField0_ |= 0x00000020;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * metric unit name
+         * 

+         *
+         * string unit = 6;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearUnit() {
+          unit_ = getDefaultInstance().getUnit();
+          bitField0_ = (bitField0_ & ~0x00000020);
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * metric unit name
+         * 

+         *
+         * string unit = 6;
+         *
+         * @param value The bytes for unit to set.
+         * @return This builder for chaining.
+         */
+        public Builder setUnitBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          unit_ = value;
+          bitField0_ |= 0x00000020;
+          onChanged();
+          return this;
+        }
+
+        private java.lang.Object sourceTypeName_ = "";
+        /**
+         *
+         *
+         * 
+         * source of this metric (check name, etc.)
+         * 

+         *
+         * string source_type_name = 7;
+         *
+         * @return The sourceTypeName.
+         */
+        public java.lang.String getSourceTypeName() {
+          java.lang.Object ref = sourceTypeName_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            sourceTypeName_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * source of this metric (check name, etc.)
+         * 

+         *
+         * string source_type_name = 7;
+         *
+         * @return The bytes for sourceTypeName.
+         */
+        public com.google.protobuf.ByteString getSourceTypeNameBytes() {
+          java.lang.Object ref = sourceTypeName_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            sourceTypeName_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         *
+         *
+         * 
+         * source of this metric (check name, etc.)
+         * 

+         *
+         * string source_type_name = 7;
+         *
+         * @param value The sourceTypeName to set.
+         * @return This builder for chaining.
+         */
+        public Builder setSourceTypeName(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          sourceTypeName_ = value;
+          bitField0_ |= 0x00000040;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * source of this metric (check name, etc.)
+         * 

+         *
+         * string source_type_name = 7;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearSourceTypeName() {
+          sourceTypeName_ = getDefaultInstance().getSourceTypeName();
+          bitField0_ = (bitField0_ & ~0x00000040);
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * source of this metric (check name, etc.)
+         * 

+         *
+         * string source_type_name = 7;
+         *
+         * @param value The bytes for sourceTypeName to set.
+         * @return This builder for chaining.
+         */
+        public Builder setSourceTypeNameBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          sourceTypeName_ = value;
+          bitField0_ |= 0x00000040;
+          onChanged();
+          return this;
+        }
+
+        private long interval_;
+        /**
+         *
+         *
+         * 
+         * interval, in seconds, between samples of this metric
+         * 

+         *
+         * int64 interval = 8;
+         *
+         * @return The interval.
+         */
+        @java.lang.Override
+        public long getInterval() {
+          return interval_;
+        }
+        /**
+         *
+         *
+         * 
+         * interval, in seconds, between samples of this metric
+         * 

+         *
+         * int64 interval = 8;
+         *
+         * @param value The interval to set.
+         * @return This builder for chaining.
+         */
+        public Builder setInterval(long value) {
+
+          interval_ = value;
+          bitField0_ |= 0x00000080;
+          onChanged();
+          return this;
+        }
+        /**
+         *
+         *
+         * 
+         * interval, in seconds, between samples of this metric
+         * 

+         *
+         * int64 interval = 8;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearInterval() {
+          bitField0_ = (bitField0_ & ~0x00000080);
+          interval_ = 0L;
+          onChanged();
+          return this;
+        }
+
+        @java.lang.Override
+        public final Builder setUnknownFields(
+            final com.google.protobuf.UnknownFieldSet unknownFields) {
+          return super.setUnknownFields(unknownFields);
+        }
+
+        @java.lang.Override
+        public final Builder mergeUnknownFields(
+            final com.google.protobuf.UnknownFieldSet unknownFields) {
+          return super.mergeUnknownFields(unknownFields);
+        }
+
+        // @@protoc_insertion_point(builder_scope:datadog.agentpayload.MetricPayload.MetricSeries)
+      }
+
+      // @@protoc_insertion_point(class_scope:datadog.agentpayload.MetricPayload.MetricSeries)
+      private static final datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries
+          DEFAULT_INSTANCE;
+
+      static {
+        DEFAULT_INSTANCE = new datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries();
+      }
+
+      public static datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries
+          getDefaultInstance() {
+        return DEFAULT_INSTANCE;
+      }
+
+      private static final com.google.protobuf.Parser PARSER =
+          new com.google.protobuf.AbstractParser() {
+            @java.lang.Override
+            public MetricSeries parsePartialFrom(
+                com.google.protobuf.CodedInputStream input,
+                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                throws com.google.protobuf.InvalidProtocolBufferException {
+              Builder builder = newBuilder();
+              try {
+                builder.mergeFrom(input, extensionRegistry);
+              } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                throw e.setUnfinishedMessage(builder.buildPartial());
+              } catch (com.google.protobuf.UninitializedMessageException e) {
+                throw e.asInvalidProtocolBufferException()
+                    .setUnfinishedMessage(builder.buildPartial());
+              } catch (java.io.IOException e) {
+                throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                    .setUnfinishedMessage(builder.buildPartial());
+              }
+              return builder.buildPartial();
+            }
+          };
+
+      public static com.google.protobuf.Parser parser() {
+        return PARSER;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Parser getParserForType() {
+        return PARSER;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries
+          getDefaultInstanceForType() {
+        return DEFAULT_INSTANCE;
+      }
+    }
+
+    public static final int SERIES_FIELD_NUMBER = 1;
+
+    @SuppressWarnings("serial")
+    private java.util.List series_;
+    /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+    @java.lang.Override
+    public java.util.List
+        getSeriesList() {
+      return series_;
+    }
+    /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+    @java.lang.Override
+    public java.util.List<
+            ? extends datadog.agentpayload.AgentPayload.MetricPayload.MetricSeriesOrBuilder>
+        getSeriesOrBuilderList() {
+      return series_;
+    }
+    /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+    @java.lang.Override
+    public int getSeriesCount() {
+      return series_.size();
+    }
+    /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries getSeries(int index) {
+      return series_.get(index);
+    }
+    /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeriesOrBuilder getSeriesOrBuilder(
+        int index) {
+      return series_.get(index);
+    }
+
+    private byte memoizedIsInitialized = -1;
+
+    @java.lang.Override
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized == 1) return true;
+      if (isInitialized == 0) return false;
+
+      memoizedIsInitialized = 1;
+      return true;
+    }
+
+    @java.lang.Override
+    public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+      for (int i = 0; i < series_.size(); i++) {
+        output.writeMessage(1, series_.get(i));
+      }
+      getUnknownFields().writeTo(output);
+    }
+
+    @java.lang.Override
+    public int getSerializedSize() {
+      int size = memoizedSize;
+      if (size != -1) return size;
+
+      size = 0;
+      for (int i = 0; i < series_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream.computeMessageSize(1, series_.get(i));
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSize = size;
+      return size;
+    }
+
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof datadog.agentpayload.AgentPayload.MetricPayload)) {
+        return super.equals(obj);
+      }
+      datadog.agentpayload.AgentPayload.MetricPayload other =
+          (datadog.agentpayload.AgentPayload.MetricPayload) obj;
+
+      if (!getSeriesList().equals(other.getSeriesList())) return false;
+      if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+      return true;
+    }
+
+    @java.lang.Override
+    public int hashCode() {
+      if (memoizedHashCode != 0) {
+        return memoizedHashCode;
+      }
+      int hash = 41;
+      hash = (19 * hash) + getDescriptor().hashCode();
+      if (getSeriesCount() > 0) {
+        hash = (37 * hash) + SERIES_FIELD_NUMBER;
+        hash = (53 * hash) + getSeriesList().hashCode();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      memoizedHashCode = hash;
+      return hash;
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseFrom(
+        java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseFrom(
+        java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseFrom(
+        byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseFrom(
+        java.io.InputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseFrom(
+        java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseDelimitedFrom(
+        java.io.InputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseDelimitedFrom(
+        java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseFrom(
+        com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    @java.lang.Override
+    public Builder newBuilderForType() {
+      return newBuilder();
+    }
+
+    public static Builder newBuilder() {
+      return DEFAULT_INSTANCE.toBuilder();
+    }
+
+    public static Builder newBuilder(datadog.agentpayload.AgentPayload.MetricPayload prototype) {
+      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+    }
+
+    @java.lang.Override
+    public Builder toBuilder() {
+      return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+    }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    /** Protobuf type {@code datadog.agentpayload.MetricPayload} */
+    public static final class Builder
+        extends com.google.protobuf.GeneratedMessageV3.Builder
+        implements
+        // @@protoc_insertion_point(builder_implements:datadog.agentpayload.MetricPayload)
+        datadog.agentpayload.AgentPayload.MetricPayloadOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_MetricPayload_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_MetricPayload_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                datadog.agentpayload.AgentPayload.MetricPayload.class,
+                datadog.agentpayload.AgentPayload.MetricPayload.Builder.class);
+      }
+
+      // Construct using datadog.agentpayload.AgentPayload.MetricPayload.newBuilder()
+      private Builder() {}
+
+      private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        super(parent);
+      }
+
+      @java.lang.Override
+      public Builder clear() {
+        super.clear();
+        bitField0_ = 0;
+        if (seriesBuilder_ == null) {
+          series_ = java.util.Collections.emptyList();
+        } else {
+          series_ = null;
+          seriesBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000001);
+        return this;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_MetricPayload_descriptor;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload getDefaultInstanceForType() {
+        return datadog.agentpayload.AgentPayload.MetricPayload.getDefaultInstance();
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload build() {
+        datadog.agentpayload.AgentPayload.MetricPayload result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.MetricPayload buildPartial() {
+        datadog.agentpayload.AgentPayload.MetricPayload result =
+            new datadog.agentpayload.AgentPayload.MetricPayload(this);
+        buildPartialRepeatedFields(result);
+        if (bitField0_ != 0) {
+          buildPartial0(result);
+        }
+        onBuilt();
+        return result;
+      }
+
+      private void buildPartialRepeatedFields(
+          datadog.agentpayload.AgentPayload.MetricPayload result) {
+        if (seriesBuilder_ == null) {
+          if (((bitField0_ & 0x00000001) != 0)) {
+            series_ = java.util.Collections.unmodifiableList(series_);
+            bitField0_ = (bitField0_ & ~0x00000001);
+          }
+          result.series_ = series_;
+        } else {
+          result.series_ = seriesBuilder_.build();
+        }
+      }
+
+      private void buildPartial0(datadog.agentpayload.AgentPayload.MetricPayload result) {
+        int from_bitField0_ = bitField0_;
+      }
+
+      @java.lang.Override
+      public Builder clone() {
+        return super.clone();
+      }
+
+      @java.lang.Override
+      public Builder setField(
+          com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+        return super.setField(field, value);
+      }
+
+      @java.lang.Override
+      public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+        return super.clearField(field);
+      }
+
+      @java.lang.Override
+      public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+        return super.clearOneof(oneof);
+      }
+
+      @java.lang.Override
+      public Builder setRepeatedField(
+          com.google.protobuf.Descriptors.FieldDescriptor field,
+          int index,
+          java.lang.Object value) {
+        return super.setRepeatedField(field, index, value);
+      }
+
+      @java.lang.Override
+      public Builder addRepeatedField(
+          com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+        return super.addRepeatedField(field, value);
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof datadog.agentpayload.AgentPayload.MetricPayload) {
+          return mergeFrom((datadog.agentpayload.AgentPayload.MetricPayload) other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+
+      public Builder mergeFrom(datadog.agentpayload.AgentPayload.MetricPayload other) {
+        if (other == datadog.agentpayload.AgentPayload.MetricPayload.getDefaultInstance())
+          return this;
+        if (seriesBuilder_ == null) {
+          if (!other.series_.isEmpty()) {
+            if (series_.isEmpty()) {
+              series_ = other.series_;
+              bitField0_ = (bitField0_ & ~0x00000001);
+            } else {
+              ensureSeriesIsMutable();
+              series_.addAll(other.series_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.series_.isEmpty()) {
+            if (seriesBuilder_.isEmpty()) {
+              seriesBuilder_.dispose();
+              seriesBuilder_ = null;
+              series_ = other.series_;
+              bitField0_ = (bitField0_ & ~0x00000001);
+              seriesBuilder_ =
+                  com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders
+                      ? getSeriesFieldBuilder()
+                      : null;
+            } else {
+              seriesBuilder_.addAllMessages(other.series_);
+            }
+          }
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        onChanged();
+        return this;
+      }
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        return true;
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        if (extensionRegistry == null) {
+          throw new java.lang.NullPointerException();
+        }
+        try {
+          boolean done = false;
+          while (!done) {
+            int tag = input.readTag();
+            switch (tag) {
+              case 0:
+                done = true;
+                break;
+              case 10:
+                {
+                  datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries m =
+                      input.readMessage(
+                          datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.parser(),
+                          extensionRegistry);
+                  if (seriesBuilder_ == null) {
+                    ensureSeriesIsMutable();
+                    series_.add(m);
+                  } else {
+                    seriesBuilder_.addMessage(m);
+                  }
+                  break;
+                } // case 10
+              default:
+                {
+                  if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                    done = true; // was an endgroup tag
+                  }
+                  break;
+                } // default:
+            } // switch (tag)
+          } // while (!done)
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          throw e.unwrapIOException();
+        } finally {
+          onChanged();
+        } // finally
+        return this;
+      }
+
+      private int bitField0_;
+
+      private java.util.List series_ =
+          java.util.Collections.emptyList();
+
+      private void ensureSeriesIsMutable() {
+        if (!((bitField0_ & 0x00000001) != 0)) {
+          series_ =
+              new java.util.ArrayList(
+                  series_);
+          bitField0_ |= 0x00000001;
+        }
+      }
+
+      private com.google.protobuf.RepeatedFieldBuilderV3<
+              datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries,
+              datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder,
+              datadog.agentpayload.AgentPayload.MetricPayload.MetricSeriesOrBuilder>
+          seriesBuilder_;
+
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public java.util.List
+          getSeriesList() {
+        if (seriesBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(series_);
+        } else {
+          return seriesBuilder_.getMessageList();
+        }
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public int getSeriesCount() {
+        if (seriesBuilder_ == null) {
+          return series_.size();
+        } else {
+          return seriesBuilder_.getCount();
+        }
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries getSeries(int index) {
+        if (seriesBuilder_ == null) {
+          return series_.get(index);
+        } else {
+          return seriesBuilder_.getMessage(index);
+        }
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public Builder setSeries(
+          int index, datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries value) {
+        if (seriesBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureSeriesIsMutable();
+          series_.set(index, value);
+          onChanged();
+        } else {
+          seriesBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public Builder setSeries(
+          int index,
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder builderForValue) {
+        if (seriesBuilder_ == null) {
+          ensureSeriesIsMutable();
+          series_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          seriesBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public Builder addSeries(datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries value) {
+        if (seriesBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureSeriesIsMutable();
+          series_.add(value);
+          onChanged();
+        } else {
+          seriesBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public Builder addSeries(
+          int index, datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries value) {
+        if (seriesBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureSeriesIsMutable();
+          series_.add(index, value);
+          onChanged();
+        } else {
+          seriesBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public Builder addSeries(
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder builderForValue) {
+        if (seriesBuilder_ == null) {
+          ensureSeriesIsMutable();
+          series_.add(builderForValue.build());
+          onChanged();
+        } else {
+          seriesBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public Builder addSeries(
+          int index,
+          datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder builderForValue) {
+        if (seriesBuilder_ == null) {
+          ensureSeriesIsMutable();
+          series_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          seriesBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public Builder addAllSeries(
+          java.lang.Iterable
+              values) {
+        if (seriesBuilder_ == null) {
+          ensureSeriesIsMutable();
+          com.google.protobuf.AbstractMessageLite.Builder.addAll(values, series_);
+          onChanged();
+        } else {
+          seriesBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public Builder clearSeries() {
+        if (seriesBuilder_ == null) {
+          series_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000001);
+          onChanged();
+        } else {
+          seriesBuilder_.clear();
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public Builder removeSeries(int index) {
+        if (seriesBuilder_ == null) {
+          ensureSeriesIsMutable();
+          series_.remove(index);
+          onChanged();
+        } else {
+          seriesBuilder_.remove(index);
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder getSeriesBuilder(
+          int index) {
+        return getSeriesFieldBuilder().getBuilder(index);
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeriesOrBuilder
+          getSeriesOrBuilder(int index) {
+        if (seriesBuilder_ == null) {
+          return series_.get(index);
+        } else {
+          return seriesBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public java.util.List<
+              ? extends datadog.agentpayload.AgentPayload.MetricPayload.MetricSeriesOrBuilder>
+          getSeriesOrBuilderList() {
+        if (seriesBuilder_ != null) {
+          return seriesBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(series_);
+        }
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder
+          addSeriesBuilder() {
+        return getSeriesFieldBuilder()
+            .addBuilder(
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.getDefaultInstance());
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder addSeriesBuilder(
+          int index) {
+        return getSeriesFieldBuilder()
+            .addBuilder(
+                index,
+                datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.getDefaultInstance());
+      }
+      /** repeated .datadog.agentpayload.MetricPayload.MetricSeries series = 1; */
+      public java.util.List
+          getSeriesBuilderList() {
+        return getSeriesFieldBuilder().getBuilderList();
+      }
+
+      private com.google.protobuf.RepeatedFieldBuilderV3<
+              datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries,
+              datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder,
+              datadog.agentpayload.AgentPayload.MetricPayload.MetricSeriesOrBuilder>
+          getSeriesFieldBuilder() {
+        if (seriesBuilder_ == null) {
+          seriesBuilder_ =
+              new com.google.protobuf.RepeatedFieldBuilderV3<
+                  datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries,
+                  datadog.agentpayload.AgentPayload.MetricPayload.MetricSeries.Builder,
+                  datadog.agentpayload.AgentPayload.MetricPayload.MetricSeriesOrBuilder>(
+                  series_, ((bitField0_ & 0x00000001) != 0), getParentForChildren(), isClean());
+          series_ = null;
+        }
+        return seriesBuilder_;
+      }
+
+      @java.lang.Override
+      public final Builder setUnknownFields(
+          final com.google.protobuf.UnknownFieldSet unknownFields) {
+        return super.setUnknownFields(unknownFields);
+      }
+
+      @java.lang.Override
+      public final Builder mergeUnknownFields(
+          final com.google.protobuf.UnknownFieldSet unknownFields) {
+        return super.mergeUnknownFields(unknownFields);
+      }
+
+      // @@protoc_insertion_point(builder_scope:datadog.agentpayload.MetricPayload)
+    }
+
+    // @@protoc_insertion_point(class_scope:datadog.agentpayload.MetricPayload)
+    private static final datadog.agentpayload.AgentPayload.MetricPayload DEFAULT_INSTANCE;
+
+    static {
+      DEFAULT_INSTANCE = new datadog.agentpayload.AgentPayload.MetricPayload();
+    }
+
+    public static datadog.agentpayload.AgentPayload.MetricPayload getDefaultInstance() {
+      return DEFAULT_INSTANCE;
+    }
+
+    private static final com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+          @java.lang.Override
+          public MetricPayload parsePartialFrom(
+              com.google.protobuf.CodedInputStream input,
+              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+              throws com.google.protobuf.InvalidProtocolBufferException {
+            Builder builder = newBuilder();
+            try {
+              builder.mergeFrom(input, extensionRegistry);
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+              throw e.setUnfinishedMessage(builder.buildPartial());
+            } catch (com.google.protobuf.UninitializedMessageException e) {
+              throw e.asInvalidProtocolBufferException()
+                  .setUnfinishedMessage(builder.buildPartial());
+            } catch (java.io.IOException e) {
+              throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                  .setUnfinishedMessage(builder.buildPartial());
+            }
+            return builder.buildPartial();
+          }
+        };
+
+    public static com.google.protobuf.Parser parser() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.MetricPayload getDefaultInstanceForType() {
+      return DEFAULT_INSTANCE;
+    }
+  }
+
+  public interface EventsPayloadOrBuilder
+      extends
+      // @@protoc_insertion_point(interface_extends:datadog.agentpayload.EventsPayload)
+      com.google.protobuf.MessageOrBuilder {
+
+    /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+    java.util.List getEventsList();
+    /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+    datadog.agentpayload.AgentPayload.EventsPayload.Event getEvents(int index);
+    /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+    int getEventsCount();
+    /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+    java.util.List
+        getEventsOrBuilderList();
+    /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+    datadog.agentpayload.AgentPayload.EventsPayload.EventOrBuilder getEventsOrBuilder(int index);
+
+    /**
+     * .datadog.agentpayload.CommonMetadata metadata = 2;
+     *
+     * @return Whether the metadata field is set.
+     */
+    boolean hasMetadata();
+    /**
+     * .datadog.agentpayload.CommonMetadata metadata = 2;
+     *
+     * @return The metadata.
+     */
+    datadog.agentpayload.AgentPayload.CommonMetadata getMetadata();
+    /** .datadog.agentpayload.CommonMetadata metadata = 2; */
+    datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder getMetadataOrBuilder();
+  }
+  /** Protobuf type {@code datadog.agentpayload.EventsPayload} */
+  public static final class EventsPayload extends com.google.protobuf.GeneratedMessageV3
+      implements
+      // @@protoc_insertion_point(message_implements:datadog.agentpayload.EventsPayload)
+      EventsPayloadOrBuilder {
+    private static final long serialVersionUID = 0L;
+    // Use EventsPayload.newBuilder() to construct.
+    private EventsPayload(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+      super(builder);
+    }
+
+    private EventsPayload() {
+      events_ = java.util.Collections.emptyList();
+    }
+
+    @java.lang.Override
+    @SuppressWarnings({"unused"})
+    protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+      return new EventsPayload();
+    }
+
+    public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+      return datadog.agentpayload.AgentPayload
+          .internal_static_datadog_agentpayload_EventsPayload_descriptor;
+    }
+
+    @java.lang.Override
+    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return datadog.agentpayload.AgentPayload
+          .internal_static_datadog_agentpayload_EventsPayload_fieldAccessorTable
+          .ensureFieldAccessorsInitialized(
+              datadog.agentpayload.AgentPayload.EventsPayload.class,
+              datadog.agentpayload.AgentPayload.EventsPayload.Builder.class);
+    }
+
+    public interface EventOrBuilder
+        extends
+        // @@protoc_insertion_point(interface_extends:datadog.agentpayload.EventsPayload.Event)
+        com.google.protobuf.MessageOrBuilder {
+
+      /**
+       * string title = 1;
+       *
+       * @return The title.
+       */
+      java.lang.String getTitle();
+      /**
+       * string title = 1;
+       *
+       * @return The bytes for title.
+       */
+      com.google.protobuf.ByteString getTitleBytes();
+
+      /**
+       * string text = 2;
+       *
+       * @return The text.
+       */
+      java.lang.String getText();
+      /**
+       * string text = 2;
+       *
+       * @return The bytes for text.
+       */
+      com.google.protobuf.ByteString getTextBytes();
+
+      /**
+       * int64 ts = 3;
+       *
+       * @return The ts.
+       */
+      long getTs();
+
+      /**
+       * string priority = 4;
+       *
+       * @return The priority.
+       */
+      java.lang.String getPriority();
+      /**
+       * string priority = 4;
+       *
+       * @return The bytes for priority.
+       */
+      com.google.protobuf.ByteString getPriorityBytes();
+
+      /**
+       * string host = 5;
+       *
+       * @return The host.
+       */
+      java.lang.String getHost();
+      /**
+       * string host = 5;
+       *
+       * @return The bytes for host.
+       */
+      com.google.protobuf.ByteString getHostBytes();
+
+      /**
+       * repeated string tags = 6;
+       *
+       * @return A list containing the tags.
+       */
+      java.util.List getTagsList();
+      /**
+       * repeated string tags = 6;
+       *
+       * @return The count of tags.
+       */
+      int getTagsCount();
+      /**
+       * repeated string tags = 6;
+       *
+       * @param index The index of the element to return.
+       * @return The tags at the given index.
+       */
+      java.lang.String getTags(int index);
+      /**
+       * repeated string tags = 6;
+       *
+       * @param index The index of the value to return.
+       * @return The bytes of the tags at the given index.
+       */
+      com.google.protobuf.ByteString getTagsBytes(int index);
+
+      /**
+       * string alert_type = 7;
+       *
+       * @return The alertType.
+       */
+      java.lang.String getAlertType();
+      /**
+       * string alert_type = 7;
+       *
+       * @return The bytes for alertType.
+       */
+      com.google.protobuf.ByteString getAlertTypeBytes();
+
+      /**
+       * string aggregation_key = 8;
+       *
+       * @return The aggregationKey.
+       */
+      java.lang.String getAggregationKey();
+      /**
+       * string aggregation_key = 8;
+       *
+       * @return The bytes for aggregationKey.
+       */
+      com.google.protobuf.ByteString getAggregationKeyBytes();
+
+      /**
+       * string source_type_name = 9;
+       *
+       * @return The sourceTypeName.
+       */
+      java.lang.String getSourceTypeName();
+      /**
+       * string source_type_name = 9;
+       *
+       * @return The bytes for sourceTypeName.
+       */
+      com.google.protobuf.ByteString getSourceTypeNameBytes();
+    }
+    /** Protobuf type {@code datadog.agentpayload.EventsPayload.Event} */
+    public static final class Event extends com.google.protobuf.GeneratedMessageV3
+        implements
+        // @@protoc_insertion_point(message_implements:datadog.agentpayload.EventsPayload.Event)
+        EventOrBuilder {
+      private static final long serialVersionUID = 0L;
+      // Use Event.newBuilder() to construct.
+      private Event(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+        super(builder);
+      }
+
+      private Event() {
+        title_ = "";
+        text_ = "";
+        priority_ = "";
+        host_ = "";
+        tags_ = com.google.protobuf.LazyStringArrayList.emptyList();
+        alertType_ = "";
+        aggregationKey_ = "";
+        sourceTypeName_ = "";
+      }
+
+      @java.lang.Override
+      @SuppressWarnings({"unused"})
+      protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+        return new Event();
+      }
+
+      public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_EventsPayload_Event_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_EventsPayload_Event_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                datadog.agentpayload.AgentPayload.EventsPayload.Event.class,
+                datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder.class);
+      }
+
+      public static final int TITLE_FIELD_NUMBER = 1;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object title_ = "";
+      /**
+       * string title = 1;
+       *
+       * @return The title.
+       */
+      @java.lang.Override
+      public java.lang.String getTitle() {
+        java.lang.Object ref = title_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          title_ = s;
+          return s;
+        }
+      }
+      /**
+       * string title = 1;
+       *
+       * @return The bytes for title.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getTitleBytes() {
+        java.lang.Object ref = title_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          title_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int TEXT_FIELD_NUMBER = 2;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object text_ = "";
+      /**
+       * string text = 2;
+       *
+       * @return The text.
+       */
+      @java.lang.Override
+      public java.lang.String getText() {
+        java.lang.Object ref = text_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          text_ = s;
+          return s;
+        }
+      }
+      /**
+       * string text = 2;
+       *
+       * @return The bytes for text.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getTextBytes() {
+        java.lang.Object ref = text_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          text_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int TS_FIELD_NUMBER = 3;
+      private long ts_ = 0L;
+      /**
+       * int64 ts = 3;
+       *
+       * @return The ts.
+       */
+      @java.lang.Override
+      public long getTs() {
+        return ts_;
+      }
+
+      public static final int PRIORITY_FIELD_NUMBER = 4;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object priority_ = "";
+      /**
+       * string priority = 4;
+       *
+       * @return The priority.
+       */
+      @java.lang.Override
+      public java.lang.String getPriority() {
+        java.lang.Object ref = priority_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          priority_ = s;
+          return s;
+        }
+      }
+      /**
+       * string priority = 4;
+       *
+       * @return The bytes for priority.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getPriorityBytes() {
+        java.lang.Object ref = priority_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          priority_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int HOST_FIELD_NUMBER = 5;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object host_ = "";
+      /**
+       * string host = 5;
+       *
+       * @return The host.
+       */
+      @java.lang.Override
+      public java.lang.String getHost() {
+        java.lang.Object ref = host_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          host_ = s;
+          return s;
+        }
+      }
+      /**
+       * string host = 5;
+       *
+       * @return The bytes for host.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getHostBytes() {
+        java.lang.Object ref = host_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          host_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int TAGS_FIELD_NUMBER = 6;
+
+      @SuppressWarnings("serial")
+      private com.google.protobuf.LazyStringArrayList tags_ =
+          com.google.protobuf.LazyStringArrayList.emptyList();
+      /**
+       * repeated string tags = 6;
+       *
+       * @return A list containing the tags.
+       */
+      public com.google.protobuf.ProtocolStringList getTagsList() {
+        return tags_;
+      }
+      /**
+       * repeated string tags = 6;
+       *
+       * @return The count of tags.
+       */
+      public int getTagsCount() {
+        return tags_.size();
+      }
+      /**
+       * repeated string tags = 6;
+       *
+       * @param index The index of the element to return.
+       * @return The tags at the given index.
+       */
+      public java.lang.String getTags(int index) {
+        return tags_.get(index);
+      }
+      /**
+       * repeated string tags = 6;
+       *
+       * @param index The index of the value to return.
+       * @return The bytes of the tags at the given index.
+       */
+      public com.google.protobuf.ByteString getTagsBytes(int index) {
+        return tags_.getByteString(index);
+      }
+
+      public static final int ALERT_TYPE_FIELD_NUMBER = 7;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object alertType_ = "";
+      /**
+       * string alert_type = 7;
+       *
+       * @return The alertType.
+       */
+      @java.lang.Override
+      public java.lang.String getAlertType() {
+        java.lang.Object ref = alertType_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          alertType_ = s;
+          return s;
+        }
+      }
+      /**
+       * string alert_type = 7;
+       *
+       * @return The bytes for alertType.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getAlertTypeBytes() {
+        java.lang.Object ref = alertType_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          alertType_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int AGGREGATION_KEY_FIELD_NUMBER = 8;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object aggregationKey_ = "";
+      /**
+       * string aggregation_key = 8;
+       *
+       * @return The aggregationKey.
+       */
+      @java.lang.Override
+      public java.lang.String getAggregationKey() {
+        java.lang.Object ref = aggregationKey_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          aggregationKey_ = s;
+          return s;
+        }
+      }
+      /**
+       * string aggregation_key = 8;
+       *
+       * @return The bytes for aggregationKey.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getAggregationKeyBytes() {
+        java.lang.Object ref = aggregationKey_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          aggregationKey_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int SOURCE_TYPE_NAME_FIELD_NUMBER = 9;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object sourceTypeName_ = "";
+      /**
+       * string source_type_name = 9;
+       *
+       * @return The sourceTypeName.
+       */
+      @java.lang.Override
+      public java.lang.String getSourceTypeName() {
+        java.lang.Object ref = sourceTypeName_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          sourceTypeName_ = s;
+          return s;
+        }
+      }
+      /**
+       * string source_type_name = 9;
+       *
+       * @return The bytes for sourceTypeName.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getSourceTypeNameBytes() {
+        java.lang.Object ref = sourceTypeName_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          sourceTypeName_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      private byte memoizedIsInitialized = -1;
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        byte isInitialized = memoizedIsInitialized;
+        if (isInitialized == 1) return true;
+        if (isInitialized == 0) return false;
+
+        memoizedIsInitialized = 1;
+        return true;
+      }
+
+      @java.lang.Override
+      public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(title_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 1, title_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(text_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 2, text_);
+        }
+        if (ts_ != 0L) {
+          output.writeInt64(3, ts_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(priority_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 4, priority_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(host_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 5, host_);
+        }
+        for (int i = 0; i < tags_.size(); i++) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 6, tags_.getRaw(i));
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(alertType_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 7, alertType_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(aggregationKey_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 8, aggregationKey_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(sourceTypeName_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 9, sourceTypeName_);
+        }
+        getUnknownFields().writeTo(output);
+      }
+
+      @java.lang.Override
+      public int getSerializedSize() {
+        int size = memoizedSize;
+        if (size != -1) return size;
+
+        size = 0;
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(title_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, title_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(text_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, text_);
+        }
+        if (ts_ != 0L) {
+          size += com.google.protobuf.CodedOutputStream.computeInt64Size(3, ts_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(priority_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(4, priority_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(host_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(5, host_);
+        }
+        {
+          int dataSize = 0;
+          for (int i = 0; i < tags_.size(); i++) {
+            dataSize += computeStringSizeNoTag(tags_.getRaw(i));
+          }
+          size += dataSize;
+          size += 1 * getTagsList().size();
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(alertType_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(7, alertType_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(aggregationKey_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(8, aggregationKey_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(sourceTypeName_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(9, sourceTypeName_);
+        }
+        size += getUnknownFields().getSerializedSize();
+        memoizedSize = size;
+        return size;
+      }
+
+      @java.lang.Override
+      public boolean equals(final java.lang.Object obj) {
+        if (obj == this) {
+          return true;
+        }
+        if (!(obj instanceof datadog.agentpayload.AgentPayload.EventsPayload.Event)) {
+          return super.equals(obj);
+        }
+        datadog.agentpayload.AgentPayload.EventsPayload.Event other =
+            (datadog.agentpayload.AgentPayload.EventsPayload.Event) obj;
+
+        if (!getTitle().equals(other.getTitle())) return false;
+        if (!getText().equals(other.getText())) return false;
+        if (getTs() != other.getTs()) return false;
+        if (!getPriority().equals(other.getPriority())) return false;
+        if (!getHost().equals(other.getHost())) return false;
+        if (!getTagsList().equals(other.getTagsList())) return false;
+        if (!getAlertType().equals(other.getAlertType())) return false;
+        if (!getAggregationKey().equals(other.getAggregationKey())) return false;
+        if (!getSourceTypeName().equals(other.getSourceTypeName())) return false;
+        if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+        return true;
+      }
+
+      @java.lang.Override
+      public int hashCode() {
+        if (memoizedHashCode != 0) {
+          return memoizedHashCode;
+        }
+        int hash = 41;
+        hash = (19 * hash) + getDescriptor().hashCode();
+        hash = (37 * hash) + TITLE_FIELD_NUMBER;
+        hash = (53 * hash) + getTitle().hashCode();
+        hash = (37 * hash) + TEXT_FIELD_NUMBER;
+        hash = (53 * hash) + getText().hashCode();
+        hash = (37 * hash) + TS_FIELD_NUMBER;
+        hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTs());
+        hash = (37 * hash) + PRIORITY_FIELD_NUMBER;
+        hash = (53 * hash) + getPriority().hashCode();
+        hash = (37 * hash) + HOST_FIELD_NUMBER;
+        hash = (53 * hash) + getHost().hashCode();
+        if (getTagsCount() > 0) {
+          hash = (37 * hash) + TAGS_FIELD_NUMBER;
+          hash = (53 * hash) + getTagsList().hashCode();
+        }
+        hash = (37 * hash) + ALERT_TYPE_FIELD_NUMBER;
+        hash = (53 * hash) + getAlertType().hashCode();
+        hash = (37 * hash) + AGGREGATION_KEY_FIELD_NUMBER;
+        hash = (53 * hash) + getAggregationKey().hashCode();
+        hash = (37 * hash) + SOURCE_TYPE_NAME_FIELD_NUMBER;
+        hash = (53 * hash) + getSourceTypeName().hashCode();
+        hash = (29 * hash) + getUnknownFields().hashCode();
+        memoizedHashCode = hash;
+        return hash;
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseFrom(
+          java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseFrom(
+          java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseFrom(
+          com.google.protobuf.ByteString data)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseFrom(
+          com.google.protobuf.ByteString data,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseFrom(byte[] data)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseFrom(
+          byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseFrom(
+          java.io.InputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseFrom(
+          java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseDelimitedFrom(
+          java.io.InputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseDelimitedFrom(
+          java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseFrom(
+          com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event parseFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      @java.lang.Override
+      public Builder newBuilderForType() {
+        return newBuilder();
+      }
+
+      public static Builder newBuilder() {
+        return DEFAULT_INSTANCE.toBuilder();
+      }
+
+      public static Builder newBuilder(
+          datadog.agentpayload.AgentPayload.EventsPayload.Event prototype) {
+        return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+      }
+
+      @java.lang.Override
+      public Builder toBuilder() {
+        return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+      }
+
+      @java.lang.Override
+      protected Builder newBuilderForType(
+          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        Builder builder = new Builder(parent);
+        return builder;
+      }
+      /** Protobuf type {@code datadog.agentpayload.EventsPayload.Event} */
+      public static final class Builder
+          extends com.google.protobuf.GeneratedMessageV3.Builder
+          implements
+          // @@protoc_insertion_point(builder_implements:datadog.agentpayload.EventsPayload.Event)
+          datadog.agentpayload.AgentPayload.EventsPayload.EventOrBuilder {
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_EventsPayload_Event_descriptor;
+        }
+
+        @java.lang.Override
+        protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+            internalGetFieldAccessorTable() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_EventsPayload_Event_fieldAccessorTable
+              .ensureFieldAccessorsInitialized(
+                  datadog.agentpayload.AgentPayload.EventsPayload.Event.class,
+                  datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder.class);
+        }
+
+        // Construct using datadog.agentpayload.AgentPayload.EventsPayload.Event.newBuilder()
+        private Builder() {}
+
+        private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          super(parent);
+        }
+
+        @java.lang.Override
+        public Builder clear() {
+          super.clear();
+          bitField0_ = 0;
+          title_ = "";
+          text_ = "";
+          ts_ = 0L;
+          priority_ = "";
+          host_ = "";
+          tags_ = com.google.protobuf.LazyStringArrayList.emptyList();
+          alertType_ = "";
+          aggregationKey_ = "";
+          sourceTypeName_ = "";
+          return this;
+        }
+
+        @java.lang.Override
+        public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_EventsPayload_Event_descriptor;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.EventsPayload.Event getDefaultInstanceForType() {
+          return datadog.agentpayload.AgentPayload.EventsPayload.Event.getDefaultInstance();
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.EventsPayload.Event build() {
+          datadog.agentpayload.AgentPayload.EventsPayload.Event result = buildPartial();
+          if (!result.isInitialized()) {
+            throw newUninitializedMessageException(result);
+          }
+          return result;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.EventsPayload.Event buildPartial() {
+          datadog.agentpayload.AgentPayload.EventsPayload.Event result =
+              new datadog.agentpayload.AgentPayload.EventsPayload.Event(this);
+          if (bitField0_ != 0) {
+            buildPartial0(result);
+          }
+          onBuilt();
+          return result;
+        }
+
+        private void buildPartial0(datadog.agentpayload.AgentPayload.EventsPayload.Event result) {
+          int from_bitField0_ = bitField0_;
+          if (((from_bitField0_ & 0x00000001) != 0)) {
+            result.title_ = title_;
+          }
+          if (((from_bitField0_ & 0x00000002) != 0)) {
+            result.text_ = text_;
+          }
+          if (((from_bitField0_ & 0x00000004) != 0)) {
+            result.ts_ = ts_;
+          }
+          if (((from_bitField0_ & 0x00000008) != 0)) {
+            result.priority_ = priority_;
+          }
+          if (((from_bitField0_ & 0x00000010) != 0)) {
+            result.host_ = host_;
+          }
+          if (((from_bitField0_ & 0x00000020) != 0)) {
+            tags_.makeImmutable();
+            result.tags_ = tags_;
+          }
+          if (((from_bitField0_ & 0x00000040) != 0)) {
+            result.alertType_ = alertType_;
+          }
+          if (((from_bitField0_ & 0x00000080) != 0)) {
+            result.aggregationKey_ = aggregationKey_;
+          }
+          if (((from_bitField0_ & 0x00000100) != 0)) {
+            result.sourceTypeName_ = sourceTypeName_;
+          }
+        }
+
+        @java.lang.Override
+        public Builder clone() {
+          return super.clone();
+        }
+
+        @java.lang.Override
+        public Builder setField(
+            com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+          return super.setField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+          return super.clearField(field);
+        }
+
+        @java.lang.Override
+        public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+          return super.clearOneof(oneof);
+        }
+
+        @java.lang.Override
+        public Builder setRepeatedField(
+            com.google.protobuf.Descriptors.FieldDescriptor field,
+            int index,
+            java.lang.Object value) {
+          return super.setRepeatedField(field, index, value);
+        }
+
+        @java.lang.Override
+        public Builder addRepeatedField(
+            com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+          return super.addRepeatedField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(com.google.protobuf.Message other) {
+          if (other instanceof datadog.agentpayload.AgentPayload.EventsPayload.Event) {
+            return mergeFrom((datadog.agentpayload.AgentPayload.EventsPayload.Event) other);
+          } else {
+            super.mergeFrom(other);
+            return this;
+          }
+        }
+
+        public Builder mergeFrom(datadog.agentpayload.AgentPayload.EventsPayload.Event other) {
+          if (other == datadog.agentpayload.AgentPayload.EventsPayload.Event.getDefaultInstance())
+            return this;
+          if (!other.getTitle().isEmpty()) {
+            title_ = other.title_;
+            bitField0_ |= 0x00000001;
+            onChanged();
+          }
+          if (!other.getText().isEmpty()) {
+            text_ = other.text_;
+            bitField0_ |= 0x00000002;
+            onChanged();
+          }
+          if (other.getTs() != 0L) {
+            setTs(other.getTs());
+          }
+          if (!other.getPriority().isEmpty()) {
+            priority_ = other.priority_;
+            bitField0_ |= 0x00000008;
+            onChanged();
+          }
+          if (!other.getHost().isEmpty()) {
+            host_ = other.host_;
+            bitField0_ |= 0x00000010;
+            onChanged();
+          }
+          if (!other.tags_.isEmpty()) {
+            if (tags_.isEmpty()) {
+              tags_ = other.tags_;
+              bitField0_ |= 0x00000020;
+            } else {
+              ensureTagsIsMutable();
+              tags_.addAll(other.tags_);
+            }
+            onChanged();
+          }
+          if (!other.getAlertType().isEmpty()) {
+            alertType_ = other.alertType_;
+            bitField0_ |= 0x00000040;
+            onChanged();
+          }
+          if (!other.getAggregationKey().isEmpty()) {
+            aggregationKey_ = other.aggregationKey_;
+            bitField0_ |= 0x00000080;
+            onChanged();
+          }
+          if (!other.getSourceTypeName().isEmpty()) {
+            sourceTypeName_ = other.sourceTypeName_;
+            bitField0_ |= 0x00000100;
+            onChanged();
+          }
+          this.mergeUnknownFields(other.getUnknownFields());
+          onChanged();
+          return this;
+        }
+
+        @java.lang.Override
+        public final boolean isInitialized() {
+          return true;
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+          if (extensionRegistry == null) {
+            throw new java.lang.NullPointerException();
+          }
+          try {
+            boolean done = false;
+            while (!done) {
+              int tag = input.readTag();
+              switch (tag) {
+                case 0:
+                  done = true;
+                  break;
+                case 10:
+                  {
+                    title_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000001;
+                    break;
+                  } // case 10
+                case 18:
+                  {
+                    text_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000002;
+                    break;
+                  } // case 18
+                case 24:
+                  {
+                    ts_ = input.readInt64();
+                    bitField0_ |= 0x00000004;
+                    break;
+                  } // case 24
+                case 34:
+                  {
+                    priority_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000008;
+                    break;
+                  } // case 34
+                case 42:
+                  {
+                    host_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000010;
+                    break;
+                  } // case 42
+                case 50:
+                  {
+                    java.lang.String s = input.readStringRequireUtf8();
+                    ensureTagsIsMutable();
+                    tags_.add(s);
+                    break;
+                  } // case 50
+                case 58:
+                  {
+                    alertType_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000040;
+                    break;
+                  } // case 58
+                case 66:
+                  {
+                    aggregationKey_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000080;
+                    break;
+                  } // case 66
+                case 74:
+                  {
+                    sourceTypeName_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000100;
+                    break;
+                  } // case 74
+                default:
+                  {
+                    if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                      done = true; // was an endgroup tag
+                    }
+                    break;
+                  } // default:
+              } // switch (tag)
+            } // while (!done)
+          } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+            throw e.unwrapIOException();
+          } finally {
+            onChanged();
+          } // finally
+          return this;
+        }
+
+        private int bitField0_;
+
+        private java.lang.Object title_ = "";
+        /**
+         * string title = 1;
+         *
+         * @return The title.
+         */
+        public java.lang.String getTitle() {
+          java.lang.Object ref = title_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            title_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string title = 1;
+         *
+         * @return The bytes for title.
+         */
+        public com.google.protobuf.ByteString getTitleBytes() {
+          java.lang.Object ref = title_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            title_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string title = 1;
+         *
+         * @param value The title to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTitle(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          title_ = value;
+          bitField0_ |= 0x00000001;
+          onChanged();
+          return this;
+        }
+        /**
+         * string title = 1;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearTitle() {
+          title_ = getDefaultInstance().getTitle();
+          bitField0_ = (bitField0_ & ~0x00000001);
+          onChanged();
+          return this;
+        }
+        /**
+         * string title = 1;
+         *
+         * @param value The bytes for title to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTitleBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          title_ = value;
+          bitField0_ |= 0x00000001;
+          onChanged();
+          return this;
+        }
+
+        private java.lang.Object text_ = "";
+        /**
+         * string text = 2;
+         *
+         * @return The text.
+         */
+        public java.lang.String getText() {
+          java.lang.Object ref = text_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            text_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string text = 2;
+         *
+         * @return The bytes for text.
+         */
+        public com.google.protobuf.ByteString getTextBytes() {
+          java.lang.Object ref = text_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            text_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string text = 2;
+         *
+         * @param value The text to set.
+         * @return This builder for chaining.
+         */
+        public Builder setText(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          text_ = value;
+          bitField0_ |= 0x00000002;
+          onChanged();
+          return this;
+        }
+        /**
+         * string text = 2;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearText() {
+          text_ = getDefaultInstance().getText();
+          bitField0_ = (bitField0_ & ~0x00000002);
+          onChanged();
+          return this;
+        }
+        /**
+         * string text = 2;
+         *
+         * @param value The bytes for text to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTextBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          text_ = value;
+          bitField0_ |= 0x00000002;
+          onChanged();
+          return this;
+        }
+
+        private long ts_;
+        /**
+         * int64 ts = 3;
+         *
+         * @return The ts.
+         */
+        @java.lang.Override
+        public long getTs() {
+          return ts_;
+        }
+        /**
+         * int64 ts = 3;
+         *
+         * @param value The ts to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTs(long value) {
+
+          ts_ = value;
+          bitField0_ |= 0x00000004;
+          onChanged();
+          return this;
+        }
+        /**
+         * int64 ts = 3;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearTs() {
+          bitField0_ = (bitField0_ & ~0x00000004);
+          ts_ = 0L;
+          onChanged();
+          return this;
+        }
+
+        private java.lang.Object priority_ = "";
+        /**
+         * string priority = 4;
+         *
+         * @return The priority.
+         */
+        public java.lang.String getPriority() {
+          java.lang.Object ref = priority_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            priority_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string priority = 4;
+         *
+         * @return The bytes for priority.
+         */
+        public com.google.protobuf.ByteString getPriorityBytes() {
+          java.lang.Object ref = priority_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            priority_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string priority = 4;
+         *
+         * @param value The priority to set.
+         * @return This builder for chaining.
+         */
+        public Builder setPriority(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          priority_ = value;
+          bitField0_ |= 0x00000008;
+          onChanged();
+          return this;
+        }
+        /**
+         * string priority = 4;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearPriority() {
+          priority_ = getDefaultInstance().getPriority();
+          bitField0_ = (bitField0_ & ~0x00000008);
+          onChanged();
+          return this;
+        }
+        /**
+         * string priority = 4;
+         *
+         * @param value The bytes for priority to set.
+         * @return This builder for chaining.
+         */
+        public Builder setPriorityBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          priority_ = value;
+          bitField0_ |= 0x00000008;
+          onChanged();
+          return this;
+        }
+
+        private java.lang.Object host_ = "";
+        /**
+         * string host = 5;
+         *
+         * @return The host.
+         */
+        public java.lang.String getHost() {
+          java.lang.Object ref = host_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            host_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string host = 5;
+         *
+         * @return The bytes for host.
+         */
+        public com.google.protobuf.ByteString getHostBytes() {
+          java.lang.Object ref = host_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            host_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string host = 5;
+         *
+         * @param value The host to set.
+         * @return This builder for chaining.
+         */
+        public Builder setHost(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          host_ = value;
+          bitField0_ |= 0x00000010;
+          onChanged();
+          return this;
+        }
+        /**
+         * string host = 5;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearHost() {
+          host_ = getDefaultInstance().getHost();
+          bitField0_ = (bitField0_ & ~0x00000010);
+          onChanged();
+          return this;
+        }
+        /**
+         * string host = 5;
+         *
+         * @param value The bytes for host to set.
+         * @return This builder for chaining.
+         */
+        public Builder setHostBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          host_ = value;
+          bitField0_ |= 0x00000010;
+          onChanged();
+          return this;
+        }
+
+        private com.google.protobuf.LazyStringArrayList tags_ =
+            com.google.protobuf.LazyStringArrayList.emptyList();
+
+        private void ensureTagsIsMutable() {
+          if (!tags_.isModifiable()) {
+            tags_ = new com.google.protobuf.LazyStringArrayList(tags_);
+          }
+          bitField0_ |= 0x00000020;
+        }
+        /**
+         * repeated string tags = 6;
+         *
+         * @return A list containing the tags.
+         */
+        public com.google.protobuf.ProtocolStringList getTagsList() {
+          tags_.makeImmutable();
+          return tags_;
+        }
+        /**
+         * repeated string tags = 6;
+         *
+         * @return The count of tags.
+         */
+        public int getTagsCount() {
+          return tags_.size();
+        }
+        /**
+         * repeated string tags = 6;
+         *
+         * @param index The index of the element to return.
+         * @return The tags at the given index.
+         */
+        public java.lang.String getTags(int index) {
+          return tags_.get(index);
+        }
+        /**
+         * repeated string tags = 6;
+         *
+         * @param index The index of the value to return.
+         * @return The bytes of the tags at the given index.
+         */
+        public com.google.protobuf.ByteString getTagsBytes(int index) {
+          return tags_.getByteString(index);
+        }
+        /**
+         * repeated string tags = 6;
+         *
+         * @param index The index to set the value at.
+         * @param value The tags to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTags(int index, java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTagsIsMutable();
+          tags_.set(index, value);
+          bitField0_ |= 0x00000020;
+          onChanged();
+          return this;
+        }
+        /**
+         * repeated string tags = 6;
+         *
+         * @param value The tags to add.
+         * @return This builder for chaining.
+         */
+        public Builder addTags(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTagsIsMutable();
+          tags_.add(value);
+          bitField0_ |= 0x00000020;
+          onChanged();
+          return this;
+        }
+        /**
+         * repeated string tags = 6;
+         *
+         * @param values The tags to add.
+         * @return This builder for chaining.
+         */
+        public Builder addAllTags(java.lang.Iterable values) {
+          ensureTagsIsMutable();
+          com.google.protobuf.AbstractMessageLite.Builder.addAll(values, tags_);
+          bitField0_ |= 0x00000020;
+          onChanged();
+          return this;
+        }
+        /**
+         * repeated string tags = 6;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearTags() {
+          tags_ = com.google.protobuf.LazyStringArrayList.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000020);
+          ;
+          onChanged();
+          return this;
+        }
+        /**
+         * repeated string tags = 6;
+         *
+         * @param value The bytes of the tags to add.
+         * @return This builder for chaining.
+         */
+        public Builder addTagsBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          ensureTagsIsMutable();
+          tags_.add(value);
+          bitField0_ |= 0x00000020;
+          onChanged();
+          return this;
+        }
+
+        private java.lang.Object alertType_ = "";
+        /**
+         * string alert_type = 7;
+         *
+         * @return The alertType.
+         */
+        public java.lang.String getAlertType() {
+          java.lang.Object ref = alertType_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            alertType_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string alert_type = 7;
+         *
+         * @return The bytes for alertType.
+         */
+        public com.google.protobuf.ByteString getAlertTypeBytes() {
+          java.lang.Object ref = alertType_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            alertType_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string alert_type = 7;
+         *
+         * @param value The alertType to set.
+         * @return This builder for chaining.
+         */
+        public Builder setAlertType(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          alertType_ = value;
+          bitField0_ |= 0x00000040;
+          onChanged();
+          return this;
+        }
+        /**
+         * string alert_type = 7;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearAlertType() {
+          alertType_ = getDefaultInstance().getAlertType();
+          bitField0_ = (bitField0_ & ~0x00000040);
+          onChanged();
+          return this;
+        }
+        /**
+         * string alert_type = 7;
+         *
+         * @param value The bytes for alertType to set.
+         * @return This builder for chaining.
+         */
+        public Builder setAlertTypeBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          alertType_ = value;
+          bitField0_ |= 0x00000040;
+          onChanged();
+          return this;
+        }
+
+        private java.lang.Object aggregationKey_ = "";
+        /**
+         * string aggregation_key = 8;
+         *
+         * @return The aggregationKey.
+         */
+        public java.lang.String getAggregationKey() {
+          java.lang.Object ref = aggregationKey_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            aggregationKey_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string aggregation_key = 8;
+         *
+         * @return The bytes for aggregationKey.
+         */
+        public com.google.protobuf.ByteString getAggregationKeyBytes() {
+          java.lang.Object ref = aggregationKey_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            aggregationKey_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string aggregation_key = 8;
+         *
+         * @param value The aggregationKey to set.
+         * @return This builder for chaining.
+         */
+        public Builder setAggregationKey(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          aggregationKey_ = value;
+          bitField0_ |= 0x00000080;
+          onChanged();
+          return this;
+        }
+        /**
+         * string aggregation_key = 8;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearAggregationKey() {
+          aggregationKey_ = getDefaultInstance().getAggregationKey();
+          bitField0_ = (bitField0_ & ~0x00000080);
+          onChanged();
+          return this;
+        }
+        /**
+         * string aggregation_key = 8;
+         *
+         * @param value The bytes for aggregationKey to set.
+         * @return This builder for chaining.
+         */
+        public Builder setAggregationKeyBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          aggregationKey_ = value;
+          bitField0_ |= 0x00000080;
+          onChanged();
+          return this;
+        }
+
+        private java.lang.Object sourceTypeName_ = "";
+        /**
+         * string source_type_name = 9;
+         *
+         * @return The sourceTypeName.
+         */
+        public java.lang.String getSourceTypeName() {
+          java.lang.Object ref = sourceTypeName_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            sourceTypeName_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string source_type_name = 9;
+         *
+         * @return The bytes for sourceTypeName.
+         */
+        public com.google.protobuf.ByteString getSourceTypeNameBytes() {
+          java.lang.Object ref = sourceTypeName_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            sourceTypeName_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string source_type_name = 9;
+         *
+         * @param value The sourceTypeName to set.
+         * @return This builder for chaining.
+         */
+        public Builder setSourceTypeName(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          sourceTypeName_ = value;
+          bitField0_ |= 0x00000100;
+          onChanged();
+          return this;
+        }
+        /**
+         * string source_type_name = 9;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearSourceTypeName() {
+          sourceTypeName_ = getDefaultInstance().getSourceTypeName();
+          bitField0_ = (bitField0_ & ~0x00000100);
+          onChanged();
+          return this;
+        }
+        /**
+         * string source_type_name = 9;
+         *
+         * @param value The bytes for sourceTypeName to set.
+         * @return This builder for chaining.
+         */
+        public Builder setSourceTypeNameBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          sourceTypeName_ = value;
+          bitField0_ |= 0x00000100;
+          onChanged();
+          return this;
+        }
+
+        @java.lang.Override
+        public final Builder setUnknownFields(
+            final com.google.protobuf.UnknownFieldSet unknownFields) {
+          return super.setUnknownFields(unknownFields);
+        }
+
+        @java.lang.Override
+        public final Builder mergeUnknownFields(
+            final com.google.protobuf.UnknownFieldSet unknownFields) {
+          return super.mergeUnknownFields(unknownFields);
+        }
+
+        // @@protoc_insertion_point(builder_scope:datadog.agentpayload.EventsPayload.Event)
+      }
+
+      // @@protoc_insertion_point(class_scope:datadog.agentpayload.EventsPayload.Event)
+      private static final datadog.agentpayload.AgentPayload.EventsPayload.Event DEFAULT_INSTANCE;
+
+      static {
+        DEFAULT_INSTANCE = new datadog.agentpayload.AgentPayload.EventsPayload.Event();
+      }
+
+      public static datadog.agentpayload.AgentPayload.EventsPayload.Event getDefaultInstance() {
+        return DEFAULT_INSTANCE;
+      }
+
+      private static final com.google.protobuf.Parser PARSER =
+          new com.google.protobuf.AbstractParser() {
+            @java.lang.Override
+            public Event parsePartialFrom(
+                com.google.protobuf.CodedInputStream input,
+                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                throws com.google.protobuf.InvalidProtocolBufferException {
+              Builder builder = newBuilder();
+              try {
+                builder.mergeFrom(input, extensionRegistry);
+              } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                throw e.setUnfinishedMessage(builder.buildPartial());
+              } catch (com.google.protobuf.UninitializedMessageException e) {
+                throw e.asInvalidProtocolBufferException()
+                    .setUnfinishedMessage(builder.buildPartial());
+              } catch (java.io.IOException e) {
+                throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                    .setUnfinishedMessage(builder.buildPartial());
+              }
+              return builder.buildPartial();
+            }
+          };
+
+      public static com.google.protobuf.Parser parser() {
+        return PARSER;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Parser getParserForType() {
+        return PARSER;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.EventsPayload.Event getDefaultInstanceForType() {
+        return DEFAULT_INSTANCE;
+      }
+    }
+
+    private int bitField0_;
+    public static final int EVENTS_FIELD_NUMBER = 1;
+
+    @SuppressWarnings("serial")
+    private java.util.List events_;
+    /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+    @java.lang.Override
+    public java.util.List getEventsList() {
+      return events_;
+    }
+    /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+    @java.lang.Override
+    public java.util.List
+        getEventsOrBuilderList() {
+      return events_;
+    }
+    /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+    @java.lang.Override
+    public int getEventsCount() {
+      return events_.size();
+    }
+    /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.EventsPayload.Event getEvents(int index) {
+      return events_.get(index);
+    }
+    /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.EventsPayload.EventOrBuilder getEventsOrBuilder(
+        int index) {
+      return events_.get(index);
+    }
+
+    public static final int METADATA_FIELD_NUMBER = 2;
+    private datadog.agentpayload.AgentPayload.CommonMetadata metadata_;
+    /**
+     * .datadog.agentpayload.CommonMetadata metadata = 2;
+     *
+     * @return Whether the metadata field is set.
+     */
+    @java.lang.Override
+    public boolean hasMetadata() {
+      return ((bitField0_ & 0x00000001) != 0);
+    }
+    /**
+     * .datadog.agentpayload.CommonMetadata metadata = 2;
+     *
+     * @return The metadata.
+     */
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.CommonMetadata getMetadata() {
+      return metadata_ == null
+          ? datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance()
+          : metadata_;
+    }
+    /** .datadog.agentpayload.CommonMetadata metadata = 2; */
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder getMetadataOrBuilder() {
+      return metadata_ == null
+          ? datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance()
+          : metadata_;
+    }
+
+    private byte memoizedIsInitialized = -1;
+
+    @java.lang.Override
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized == 1) return true;
+      if (isInitialized == 0) return false;
+
+      memoizedIsInitialized = 1;
+      return true;
+    }
+
+    @java.lang.Override
+    public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+      for (int i = 0; i < events_.size(); i++) {
+        output.writeMessage(1, events_.get(i));
+      }
+      if (((bitField0_ & 0x00000001) != 0)) {
+        output.writeMessage(2, getMetadata());
+      }
+      getUnknownFields().writeTo(output);
+    }
+
+    @java.lang.Override
+    public int getSerializedSize() {
+      int size = memoizedSize;
+      if (size != -1) return size;
+
+      size = 0;
+      for (int i = 0; i < events_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream.computeMessageSize(1, events_.get(i));
+      }
+      if (((bitField0_ & 0x00000001) != 0)) {
+        size += com.google.protobuf.CodedOutputStream.computeMessageSize(2, getMetadata());
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSize = size;
+      return size;
+    }
+
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof datadog.agentpayload.AgentPayload.EventsPayload)) {
+        return super.equals(obj);
+      }
+      datadog.agentpayload.AgentPayload.EventsPayload other =
+          (datadog.agentpayload.AgentPayload.EventsPayload) obj;
+
+      if (!getEventsList().equals(other.getEventsList())) return false;
+      if (hasMetadata() != other.hasMetadata()) return false;
+      if (hasMetadata()) {
+        if (!getMetadata().equals(other.getMetadata())) return false;
+      }
+      if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+      return true;
+    }
+
+    @java.lang.Override
+    public int hashCode() {
+      if (memoizedHashCode != 0) {
+        return memoizedHashCode;
+      }
+      int hash = 41;
+      hash = (19 * hash) + getDescriptor().hashCode();
+      if (getEventsCount() > 0) {
+        hash = (37 * hash) + EVENTS_FIELD_NUMBER;
+        hash = (53 * hash) + getEventsList().hashCode();
+      }
+      if (hasMetadata()) {
+        hash = (37 * hash) + METADATA_FIELD_NUMBER;
+        hash = (53 * hash) + getMetadata().hashCode();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      memoizedHashCode = hash;
+      return hash;
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseFrom(
+        java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseFrom(
+        java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseFrom(
+        byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseFrom(
+        java.io.InputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseFrom(
+        java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseDelimitedFrom(
+        java.io.InputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseDelimitedFrom(
+        java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseFrom(
+        com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    @java.lang.Override
+    public Builder newBuilderForType() {
+      return newBuilder();
+    }
+
+    public static Builder newBuilder() {
+      return DEFAULT_INSTANCE.toBuilder();
+    }
+
+    public static Builder newBuilder(datadog.agentpayload.AgentPayload.EventsPayload prototype) {
+      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+    }
+
+    @java.lang.Override
+    public Builder toBuilder() {
+      return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+    }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    /** Protobuf type {@code datadog.agentpayload.EventsPayload} */
+    public static final class Builder
+        extends com.google.protobuf.GeneratedMessageV3.Builder
+        implements
+        // @@protoc_insertion_point(builder_implements:datadog.agentpayload.EventsPayload)
+        datadog.agentpayload.AgentPayload.EventsPayloadOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_EventsPayload_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_EventsPayload_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                datadog.agentpayload.AgentPayload.EventsPayload.class,
+                datadog.agentpayload.AgentPayload.EventsPayload.Builder.class);
+      }
+
+      // Construct using datadog.agentpayload.AgentPayload.EventsPayload.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+
+      private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) {
+          getEventsFieldBuilder();
+          getMetadataFieldBuilder();
+        }
+      }
+
+      @java.lang.Override
+      public Builder clear() {
+        super.clear();
+        bitField0_ = 0;
+        if (eventsBuilder_ == null) {
+          events_ = java.util.Collections.emptyList();
+        } else {
+          events_ = null;
+          eventsBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000001);
+        metadata_ = null;
+        if (metadataBuilder_ != null) {
+          metadataBuilder_.dispose();
+          metadataBuilder_ = null;
+        }
+        return this;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_EventsPayload_descriptor;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.EventsPayload getDefaultInstanceForType() {
+        return datadog.agentpayload.AgentPayload.EventsPayload.getDefaultInstance();
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.EventsPayload build() {
+        datadog.agentpayload.AgentPayload.EventsPayload result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.EventsPayload buildPartial() {
+        datadog.agentpayload.AgentPayload.EventsPayload result =
+            new datadog.agentpayload.AgentPayload.EventsPayload(this);
+        buildPartialRepeatedFields(result);
+        if (bitField0_ != 0) {
+          buildPartial0(result);
+        }
+        onBuilt();
+        return result;
+      }
+
+      private void buildPartialRepeatedFields(
+          datadog.agentpayload.AgentPayload.EventsPayload result) {
+        if (eventsBuilder_ == null) {
+          if (((bitField0_ & 0x00000001) != 0)) {
+            events_ = java.util.Collections.unmodifiableList(events_);
+            bitField0_ = (bitField0_ & ~0x00000001);
+          }
+          result.events_ = events_;
+        } else {
+          result.events_ = eventsBuilder_.build();
+        }
+      }
+
+      private void buildPartial0(datadog.agentpayload.AgentPayload.EventsPayload result) {
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000002) != 0)) {
+          result.metadata_ = metadataBuilder_ == null ? metadata_ : metadataBuilder_.build();
+          to_bitField0_ |= 0x00000001;
+        }
+        result.bitField0_ |= to_bitField0_;
+      }
+
+      @java.lang.Override
+      public Builder clone() {
+        return super.clone();
+      }
+
+      @java.lang.Override
+      public Builder setField(
+          com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+        return super.setField(field, value);
+      }
+
+      @java.lang.Override
+      public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+        return super.clearField(field);
+      }
+
+      @java.lang.Override
+      public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+        return super.clearOneof(oneof);
+      }
+
+      @java.lang.Override
+      public Builder setRepeatedField(
+          com.google.protobuf.Descriptors.FieldDescriptor field,
+          int index,
+          java.lang.Object value) {
+        return super.setRepeatedField(field, index, value);
+      }
+
+      @java.lang.Override
+      public Builder addRepeatedField(
+          com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+        return super.addRepeatedField(field, value);
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof datadog.agentpayload.AgentPayload.EventsPayload) {
+          return mergeFrom((datadog.agentpayload.AgentPayload.EventsPayload) other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+
+      public Builder mergeFrom(datadog.agentpayload.AgentPayload.EventsPayload other) {
+        if (other == datadog.agentpayload.AgentPayload.EventsPayload.getDefaultInstance())
+          return this;
+        if (eventsBuilder_ == null) {
+          if (!other.events_.isEmpty()) {
+            if (events_.isEmpty()) {
+              events_ = other.events_;
+              bitField0_ = (bitField0_ & ~0x00000001);
+            } else {
+              ensureEventsIsMutable();
+              events_.addAll(other.events_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.events_.isEmpty()) {
+            if (eventsBuilder_.isEmpty()) {
+              eventsBuilder_.dispose();
+              eventsBuilder_ = null;
+              events_ = other.events_;
+              bitField0_ = (bitField0_ & ~0x00000001);
+              eventsBuilder_ =
+                  com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders
+                      ? getEventsFieldBuilder()
+                      : null;
+            } else {
+              eventsBuilder_.addAllMessages(other.events_);
+            }
+          }
+        }
+        if (other.hasMetadata()) {
+          mergeMetadata(other.getMetadata());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        onChanged();
+        return this;
+      }
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        return true;
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        if (extensionRegistry == null) {
+          throw new java.lang.NullPointerException();
+        }
+        try {
+          boolean done = false;
+          while (!done) {
+            int tag = input.readTag();
+            switch (tag) {
+              case 0:
+                done = true;
+                break;
+              case 10:
+                {
+                  datadog.agentpayload.AgentPayload.EventsPayload.Event m =
+                      input.readMessage(
+                          datadog.agentpayload.AgentPayload.EventsPayload.Event.parser(),
+                          extensionRegistry);
+                  if (eventsBuilder_ == null) {
+                    ensureEventsIsMutable();
+                    events_.add(m);
+                  } else {
+                    eventsBuilder_.addMessage(m);
+                  }
+                  break;
+                } // case 10
+              case 18:
+                {
+                  input.readMessage(getMetadataFieldBuilder().getBuilder(), extensionRegistry);
+                  bitField0_ |= 0x00000002;
+                  break;
+                } // case 18
+              default:
+                {
+                  if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                    done = true; // was an endgroup tag
+                  }
+                  break;
+                } // default:
+            } // switch (tag)
+          } // while (!done)
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          throw e.unwrapIOException();
+        } finally {
+          onChanged();
+        } // finally
+        return this;
+      }
+
+      private int bitField0_;
+
+      private java.util.List events_ =
+          java.util.Collections.emptyList();
+
+      private void ensureEventsIsMutable() {
+        if (!((bitField0_ & 0x00000001) != 0)) {
+          events_ =
+              new java.util.ArrayList(
+                  events_);
+          bitField0_ |= 0x00000001;
+        }
+      }
+
+      private com.google.protobuf.RepeatedFieldBuilderV3<
+              datadog.agentpayload.AgentPayload.EventsPayload.Event,
+              datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder,
+              datadog.agentpayload.AgentPayload.EventsPayload.EventOrBuilder>
+          eventsBuilder_;
+
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public java.util.List getEventsList() {
+        if (eventsBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(events_);
+        } else {
+          return eventsBuilder_.getMessageList();
+        }
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public int getEventsCount() {
+        if (eventsBuilder_ == null) {
+          return events_.size();
+        } else {
+          return eventsBuilder_.getCount();
+        }
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public datadog.agentpayload.AgentPayload.EventsPayload.Event getEvents(int index) {
+        if (eventsBuilder_ == null) {
+          return events_.get(index);
+        } else {
+          return eventsBuilder_.getMessage(index);
+        }
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public Builder setEvents(
+          int index, datadog.agentpayload.AgentPayload.EventsPayload.Event value) {
+        if (eventsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureEventsIsMutable();
+          events_.set(index, value);
+          onChanged();
+        } else {
+          eventsBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public Builder setEvents(
+          int index,
+          datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder builderForValue) {
+        if (eventsBuilder_ == null) {
+          ensureEventsIsMutable();
+          events_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          eventsBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public Builder addEvents(datadog.agentpayload.AgentPayload.EventsPayload.Event value) {
+        if (eventsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureEventsIsMutable();
+          events_.add(value);
+          onChanged();
+        } else {
+          eventsBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public Builder addEvents(
+          int index, datadog.agentpayload.AgentPayload.EventsPayload.Event value) {
+        if (eventsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureEventsIsMutable();
+          events_.add(index, value);
+          onChanged();
+        } else {
+          eventsBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public Builder addEvents(
+          datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder builderForValue) {
+        if (eventsBuilder_ == null) {
+          ensureEventsIsMutable();
+          events_.add(builderForValue.build());
+          onChanged();
+        } else {
+          eventsBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public Builder addEvents(
+          int index,
+          datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder builderForValue) {
+        if (eventsBuilder_ == null) {
+          ensureEventsIsMutable();
+          events_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          eventsBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public Builder addAllEvents(
+          java.lang.Iterable
+              values) {
+        if (eventsBuilder_ == null) {
+          ensureEventsIsMutable();
+          com.google.protobuf.AbstractMessageLite.Builder.addAll(values, events_);
+          onChanged();
+        } else {
+          eventsBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public Builder clearEvents() {
+        if (eventsBuilder_ == null) {
+          events_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000001);
+          onChanged();
+        } else {
+          eventsBuilder_.clear();
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public Builder removeEvents(int index) {
+        if (eventsBuilder_ == null) {
+          ensureEventsIsMutable();
+          events_.remove(index);
+          onChanged();
+        } else {
+          eventsBuilder_.remove(index);
+        }
+        return this;
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder getEventsBuilder(
+          int index) {
+        return getEventsFieldBuilder().getBuilder(index);
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public datadog.agentpayload.AgentPayload.EventsPayload.EventOrBuilder getEventsOrBuilder(
+          int index) {
+        if (eventsBuilder_ == null) {
+          return events_.get(index);
+        } else {
+          return eventsBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public java.util.List<
+              ? extends datadog.agentpayload.AgentPayload.EventsPayload.EventOrBuilder>
+          getEventsOrBuilderList() {
+        if (eventsBuilder_ != null) {
+          return eventsBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(events_);
+        }
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder addEventsBuilder() {
+        return getEventsFieldBuilder()
+            .addBuilder(datadog.agentpayload.AgentPayload.EventsPayload.Event.getDefaultInstance());
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder addEventsBuilder(
+          int index) {
+        return getEventsFieldBuilder()
+            .addBuilder(
+                index, datadog.agentpayload.AgentPayload.EventsPayload.Event.getDefaultInstance());
+      }
+      /** repeated .datadog.agentpayload.EventsPayload.Event events = 1; */
+      public java.util.List
+          getEventsBuilderList() {
+        return getEventsFieldBuilder().getBuilderList();
+      }
+
+      private com.google.protobuf.RepeatedFieldBuilderV3<
+              datadog.agentpayload.AgentPayload.EventsPayload.Event,
+              datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder,
+              datadog.agentpayload.AgentPayload.EventsPayload.EventOrBuilder>
+          getEventsFieldBuilder() {
+        if (eventsBuilder_ == null) {
+          eventsBuilder_ =
+              new com.google.protobuf.RepeatedFieldBuilderV3<
+                  datadog.agentpayload.AgentPayload.EventsPayload.Event,
+                  datadog.agentpayload.AgentPayload.EventsPayload.Event.Builder,
+                  datadog.agentpayload.AgentPayload.EventsPayload.EventOrBuilder>(
+                  events_, ((bitField0_ & 0x00000001) != 0), getParentForChildren(), isClean());
+          events_ = null;
+        }
+        return eventsBuilder_;
+      }
+
+      private datadog.agentpayload.AgentPayload.CommonMetadata metadata_;
+      private com.google.protobuf.SingleFieldBuilderV3<
+              datadog.agentpayload.AgentPayload.CommonMetadata,
+              datadog.agentpayload.AgentPayload.CommonMetadata.Builder,
+              datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder>
+          metadataBuilder_;
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2;
+       *
+       * @return Whether the metadata field is set.
+       */
+      public boolean hasMetadata() {
+        return ((bitField0_ & 0x00000002) != 0);
+      }
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2;
+       *
+       * @return The metadata.
+       */
+      public datadog.agentpayload.AgentPayload.CommonMetadata getMetadata() {
+        if (metadataBuilder_ == null) {
+          return metadata_ == null
+              ? datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance()
+              : metadata_;
+        } else {
+          return metadataBuilder_.getMessage();
+        }
+      }
+      /** .datadog.agentpayload.CommonMetadata metadata = 2; */
+      public Builder setMetadata(datadog.agentpayload.AgentPayload.CommonMetadata value) {
+        if (metadataBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          metadata_ = value;
+        } else {
+          metadataBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return this;
+      }
+      /** .datadog.agentpayload.CommonMetadata metadata = 2; */
+      public Builder setMetadata(
+          datadog.agentpayload.AgentPayload.CommonMetadata.Builder builderForValue) {
+        if (metadataBuilder_ == null) {
+          metadata_ = builderForValue.build();
+        } else {
+          metadataBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return this;
+      }
+      /** .datadog.agentpayload.CommonMetadata metadata = 2; */
+      public Builder mergeMetadata(datadog.agentpayload.AgentPayload.CommonMetadata value) {
+        if (metadataBuilder_ == null) {
+          if (((bitField0_ & 0x00000002) != 0)
+              && metadata_ != null
+              && metadata_
+                  != datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance()) {
+            getMetadataBuilder().mergeFrom(value);
+          } else {
+            metadata_ = value;
+          }
+        } else {
+          metadataBuilder_.mergeFrom(value);
+        }
+        if (metadata_ != null) {
+          bitField0_ |= 0x00000002;
+          onChanged();
+        }
+        return this;
+      }
+      /** .datadog.agentpayload.CommonMetadata metadata = 2; */
+      public Builder clearMetadata() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        metadata_ = null;
+        if (metadataBuilder_ != null) {
+          metadataBuilder_.dispose();
+          metadataBuilder_ = null;
+        }
+        onChanged();
+        return this;
+      }
+      /** .datadog.agentpayload.CommonMetadata metadata = 2; */
+      public datadog.agentpayload.AgentPayload.CommonMetadata.Builder getMetadataBuilder() {
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return getMetadataFieldBuilder().getBuilder();
+      }
+      /** .datadog.agentpayload.CommonMetadata metadata = 2; */
+      public datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder getMetadataOrBuilder() {
+        if (metadataBuilder_ != null) {
+          return metadataBuilder_.getMessageOrBuilder();
+        } else {
+          return metadata_ == null
+              ? datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance()
+              : metadata_;
+        }
+      }
+      /** .datadog.agentpayload.CommonMetadata metadata = 2; */
+      private com.google.protobuf.SingleFieldBuilderV3<
+              datadog.agentpayload.AgentPayload.CommonMetadata,
+              datadog.agentpayload.AgentPayload.CommonMetadata.Builder,
+              datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder>
+          getMetadataFieldBuilder() {
+        if (metadataBuilder_ == null) {
+          metadataBuilder_ =
+              new com.google.protobuf.SingleFieldBuilderV3<
+                  datadog.agentpayload.AgentPayload.CommonMetadata,
+                  datadog.agentpayload.AgentPayload.CommonMetadata.Builder,
+                  datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder>(
+                  getMetadata(), getParentForChildren(), isClean());
+          metadata_ = null;
+        }
+        return metadataBuilder_;
+      }
+
+      @java.lang.Override
+      public final Builder setUnknownFields(
+          final com.google.protobuf.UnknownFieldSet unknownFields) {
+        return super.setUnknownFields(unknownFields);
+      }
+
+      @java.lang.Override
+      public final Builder mergeUnknownFields(
+          final com.google.protobuf.UnknownFieldSet unknownFields) {
+        return super.mergeUnknownFields(unknownFields);
+      }
+
+      // @@protoc_insertion_point(builder_scope:datadog.agentpayload.EventsPayload)
+    }
+
+    // @@protoc_insertion_point(class_scope:datadog.agentpayload.EventsPayload)
+    private static final datadog.agentpayload.AgentPayload.EventsPayload DEFAULT_INSTANCE;
+
+    static {
+      DEFAULT_INSTANCE = new datadog.agentpayload.AgentPayload.EventsPayload();
+    }
+
+    public static datadog.agentpayload.AgentPayload.EventsPayload getDefaultInstance() {
+      return DEFAULT_INSTANCE;
+    }
+
+    private static final com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+          @java.lang.Override
+          public EventsPayload parsePartialFrom(
+              com.google.protobuf.CodedInputStream input,
+              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+              throws com.google.protobuf.InvalidProtocolBufferException {
+            Builder builder = newBuilder();
+            try {
+              builder.mergeFrom(input, extensionRegistry);
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+              throw e.setUnfinishedMessage(builder.buildPartial());
+            } catch (com.google.protobuf.UninitializedMessageException e) {
+              throw e.asInvalidProtocolBufferException()
+                  .setUnfinishedMessage(builder.buildPartial());
+            } catch (java.io.IOException e) {
+              throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                  .setUnfinishedMessage(builder.buildPartial());
+            }
+            return builder.buildPartial();
+          }
+        };
+
+    public static com.google.protobuf.Parser parser() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.EventsPayload getDefaultInstanceForType() {
+      return DEFAULT_INSTANCE;
+    }
+  }
+
+  public interface SketchPayloadOrBuilder
+      extends
+      // @@protoc_insertion_point(interface_extends:datadog.agentpayload.SketchPayload)
+      com.google.protobuf.MessageOrBuilder {
+
+    /**
+     * 
+     * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+     * 
+     */
+    java.util.List getSketchesList();
+    /**
+     * 
+     * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+     * 
+     */
+    datadog.agentpayload.AgentPayload.SketchPayload.Sketch getSketches(int index);
+    /**
+     * 
+     * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+     * 
+     */
+    int getSketchesCount();
+    /**
+     * 
+     * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+     * 
+     */
+    java.util.List
+        getSketchesOrBuilderList();
+    /**
+     * 
+     * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+     * 
+     */
+    datadog.agentpayload.AgentPayload.SketchPayload.SketchOrBuilder getSketchesOrBuilder(int index);
+
+    /**
+     * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+     * 
+     *
+     * @return Whether the metadata field is set.
+     */
+    boolean hasMetadata();
+    /**
+     * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+     * 
+     *
+     * @return The metadata.
+     */
+    datadog.agentpayload.AgentPayload.CommonMetadata getMetadata();
+    /**
+     * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+     * 
+     */
+    datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder getMetadataOrBuilder();
+  }
+  /** Protobuf type {@code datadog.agentpayload.SketchPayload} */
+  public static final class SketchPayload extends com.google.protobuf.GeneratedMessageV3
+      implements
+      // @@protoc_insertion_point(message_implements:datadog.agentpayload.SketchPayload)
+      SketchPayloadOrBuilder {
+    private static final long serialVersionUID = 0L;
+    // Use SketchPayload.newBuilder() to construct.
+    private SketchPayload(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+      super(builder);
+    }
+
+    private SketchPayload() {
+      sketches_ = java.util.Collections.emptyList();
+    }
+
+    @java.lang.Override
+    @SuppressWarnings({"unused"})
+    protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+      return new SketchPayload();
+    }
+
+    public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+      return datadog.agentpayload.AgentPayload
+          .internal_static_datadog_agentpayload_SketchPayload_descriptor;
+    }
+
+    @java.lang.Override
+    protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return datadog.agentpayload.AgentPayload
+          .internal_static_datadog_agentpayload_SketchPayload_fieldAccessorTable
+          .ensureFieldAccessorsInitialized(
+              datadog.agentpayload.AgentPayload.SketchPayload.class,
+              datadog.agentpayload.AgentPayload.SketchPayload.Builder.class);
+    }
+
+    public interface SketchOrBuilder
+        extends
+        // @@protoc_insertion_point(interface_extends:datadog.agentpayload.SketchPayload.Sketch)
+        com.google.protobuf.MessageOrBuilder {
+
+      /**
+       * string metric = 1;
+       *
+       * @return The metric.
+       */
+      java.lang.String getMetric();
+      /**
+       * string metric = 1;
+       *
+       * @return The bytes for metric.
+       */
+      com.google.protobuf.ByteString getMetricBytes();
+
+      /**
+       * string host = 2;
+       *
+       * @return The host.
+       */
+      java.lang.String getHost();
+      /**
+       * string host = 2;
+       *
+       * @return The bytes for host.
+       */
+      com.google.protobuf.ByteString getHostBytes();
+
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+       * 
+       */
+      java.util.List
+          getDistributionsList();
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+       * 
+       */
+      datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution getDistributions(
+          int index);
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+       * 
+       */
+      int getDistributionsCount();
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+       * 
+       */
+      java.util.List<
+              ? extends
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DistributionOrBuilder>
+          getDistributionsOrBuilderList();
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+       * 
+       */
+      datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DistributionOrBuilder
+          getDistributionsOrBuilder(int index);
+
+      /**
+       * repeated string tags = 4;
+       *
+       * @return A list containing the tags.
+       */
+      java.util.List getTagsList();
+      /**
+       * repeated string tags = 4;
+       *
+       * @return The count of tags.
+       */
+      int getTagsCount();
+      /**
+       * repeated string tags = 4;
+       *
+       * @param index The index of the element to return.
+       * @return The tags at the given index.
+       */
+      java.lang.String getTags(int index);
+      /**
+       * repeated string tags = 4;
+       *
+       * @param index The index of the value to return.
+       * @return The bytes of the tags at the given index.
+       */
+      com.google.protobuf.ByteString getTagsBytes(int index);
+
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+       * 
+       */
+      java.util.List
+          getDogsketchesList();
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+       * 
+       */
+      datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch getDogsketches(int index);
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+       * 
+       */
+      int getDogsketchesCount();
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+       * 
+       */
+      java.util.List<
+              ? extends datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DogsketchOrBuilder>
+          getDogsketchesOrBuilderList();
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+       * 
+       */
+      datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DogsketchOrBuilder
+          getDogsketchesOrBuilder(int index);
+    }
+    /** Protobuf type {@code datadog.agentpayload.SketchPayload.Sketch} */
+    public static final class Sketch extends com.google.protobuf.GeneratedMessageV3
+        implements
+        // @@protoc_insertion_point(message_implements:datadog.agentpayload.SketchPayload.Sketch)
+        SketchOrBuilder {
+      private static final long serialVersionUID = 0L;
+      // Use Sketch.newBuilder() to construct.
+      private Sketch(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+        super(builder);
+      }
+
+      private Sketch() {
+        metric_ = "";
+        host_ = "";
+        distributions_ = java.util.Collections.emptyList();
+        tags_ = com.google.protobuf.LazyStringArrayList.emptyList();
+        dogsketches_ = java.util.Collections.emptyList();
+      }
+
+      @java.lang.Override
+      @SuppressWarnings({"unused"})
+      protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+        return new Sketch();
+      }
+
+      public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_SketchPayload_Sketch_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_SketchPayload_Sketch_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.class,
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder.class);
+      }
+
+      public interface DistributionOrBuilder
+          extends
+          // @@protoc_insertion_point(interface_extends:datadog.agentpayload.SketchPayload.Sketch.Distribution)
+          com.google.protobuf.MessageOrBuilder {
+
+        /**
+         * int64 ts = 1;
+         *
+         * @return The ts.
+         */
+        long getTs();
+
+        /**
+         * int64 cnt = 2;
+         *
+         * @return The cnt.
+         */
+        long getCnt();
+
+        /**
+         * double min = 3;
+         *
+         * @return The min.
+         */
+        double getMin();
+
+        /**
+         * double max = 4;
+         *
+         * @return The max.
+         */
+        double getMax();
+
+        /**
+         * double avg = 5;
+         *
+         * @return The avg.
+         */
+        double getAvg();
+
+        /**
+         * double sum = 6;
+         *
+         * @return The sum.
+         */
+        double getSum();
+
+        /**
+         * repeated double v = 7;
+         *
+         * @return A list containing the v.
+         */
+        java.util.List getVList();
+        /**
+         * repeated double v = 7;
+         *
+         * @return The count of v.
+         */
+        int getVCount();
+        /**
+         * repeated double v = 7;
+         *
+         * @param index The index of the element to return.
+         * @return The v at the given index.
+         */
+        double getV(int index);
+
+        /**
+         * repeated uint32 g = 8;
+         *
+         * @return A list containing the g.
+         */
+        java.util.List getGList();
+        /**
+         * repeated uint32 g = 8;
+         *
+         * @return The count of g.
+         */
+        int getGCount();
+        /**
+         * repeated uint32 g = 8;
+         *
+         * @param index The index of the element to return.
+         * @return The g at the given index.
+         */
+        int getG(int index);
+
+        /**
+         * repeated uint32 delta = 9;
+         *
+         * @return A list containing the delta.
+         */
+        java.util.List getDeltaList();
+        /**
+         * repeated uint32 delta = 9;
+         *
+         * @return The count of delta.
+         */
+        int getDeltaCount();
+        /**
+         * repeated uint32 delta = 9;
+         *
+         * @param index The index of the element to return.
+         * @return The delta at the given index.
+         */
+        int getDelta(int index);
+
+        /**
+         * repeated double buf = 10;
+         *
+         * @return A list containing the buf.
+         */
+        java.util.List getBufList();
+        /**
+         * repeated double buf = 10;
+         *
+         * @return The count of buf.
+         */
+        int getBufCount();
+        /**
+         * repeated double buf = 10;
+         *
+         * @param index The index of the element to return.
+         * @return The buf at the given index.
+         */
+        double getBuf(int index);
+      }
+      /** Protobuf type {@code datadog.agentpayload.SketchPayload.Sketch.Distribution} */
+      public static final class Distribution extends com.google.protobuf.GeneratedMessageV3
+          implements
+          // @@protoc_insertion_point(message_implements:datadog.agentpayload.SketchPayload.Sketch.Distribution)
+          DistributionOrBuilder {
+        private static final long serialVersionUID = 0L;
+        // Use Distribution.newBuilder() to construct.
+        private Distribution(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+          super(builder);
+        }
+
+        private Distribution() {
+          v_ = emptyDoubleList();
+          g_ = emptyIntList();
+          delta_ = emptyIntList();
+          buf_ = emptyDoubleList();
+        }
+
+        @java.lang.Override
+        @SuppressWarnings({"unused"})
+        protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+          return new Distribution();
+        }
+
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_SketchPayload_Sketch_Distribution_descriptor;
+        }
+
+        @java.lang.Override
+        protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+            internalGetFieldAccessorTable() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_SketchPayload_Sketch_Distribution_fieldAccessorTable
+              .ensureFieldAccessorsInitialized(
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.class,
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder
+                      .class);
+        }
+
+        public static final int TS_FIELD_NUMBER = 1;
+        private long ts_ = 0L;
+        /**
+         * int64 ts = 1;
+         *
+         * @return The ts.
+         */
+        @java.lang.Override
+        public long getTs() {
+          return ts_;
+        }
+
+        public static final int CNT_FIELD_NUMBER = 2;
+        private long cnt_ = 0L;
+        /**
+         * int64 cnt = 2;
+         *
+         * @return The cnt.
+         */
+        @java.lang.Override
+        public long getCnt() {
+          return cnt_;
+        }
+
+        public static final int MIN_FIELD_NUMBER = 3;
+        private double min_ = 0D;
+        /**
+         * double min = 3;
+         *
+         * @return The min.
+         */
+        @java.lang.Override
+        public double getMin() {
+          return min_;
+        }
+
+        public static final int MAX_FIELD_NUMBER = 4;
+        private double max_ = 0D;
+        /**
+         * double max = 4;
+         *
+         * @return The max.
+         */
+        @java.lang.Override
+        public double getMax() {
+          return max_;
+        }
+
+        public static final int AVG_FIELD_NUMBER = 5;
+        private double avg_ = 0D;
+        /**
+         * double avg = 5;
+         *
+         * @return The avg.
+         */
+        @java.lang.Override
+        public double getAvg() {
+          return avg_;
+        }
+
+        public static final int SUM_FIELD_NUMBER = 6;
+        private double sum_ = 0D;
+        /**
+         * double sum = 6;
+         *
+         * @return The sum.
+         */
+        @java.lang.Override
+        public double getSum() {
+          return sum_;
+        }
+
+        public static final int V_FIELD_NUMBER = 7;
+
+        @SuppressWarnings("serial")
+        private com.google.protobuf.Internal.DoubleList v_ = emptyDoubleList();
+        /**
+         * repeated double v = 7;
+         *
+         * @return A list containing the v.
+         */
+        @java.lang.Override
+        public java.util.List getVList() {
+          return v_;
+        }
+        /**
+         * repeated double v = 7;
+         *
+         * @return The count of v.
+         */
+        public int getVCount() {
+          return v_.size();
+        }
+        /**
+         * repeated double v = 7;
+         *
+         * @param index The index of the element to return.
+         * @return The v at the given index.
+         */
+        public double getV(int index) {
+          return v_.getDouble(index);
+        }
+
+        private int vMemoizedSerializedSize = -1;
+
+        public static final int G_FIELD_NUMBER = 8;
+
+        @SuppressWarnings("serial")
+        private com.google.protobuf.Internal.IntList g_ = emptyIntList();
+        /**
+         * repeated uint32 g = 8;
+         *
+         * @return A list containing the g.
+         */
+        @java.lang.Override
+        public java.util.List getGList() {
+          return g_;
+        }
+        /**
+         * repeated uint32 g = 8;
+         *
+         * @return The count of g.
+         */
+        public int getGCount() {
+          return g_.size();
+        }
+        /**
+         * repeated uint32 g = 8;
+         *
+         * @param index The index of the element to return.
+         * @return The g at the given index.
+         */
+        public int getG(int index) {
+          return g_.getInt(index);
+        }
+
+        private int gMemoizedSerializedSize = -1;
+
+        public static final int DELTA_FIELD_NUMBER = 9;
+
+        @SuppressWarnings("serial")
+        private com.google.protobuf.Internal.IntList delta_ = emptyIntList();
+        /**
+         * repeated uint32 delta = 9;
+         *
+         * @return A list containing the delta.
+         */
+        @java.lang.Override
+        public java.util.List getDeltaList() {
+          return delta_;
+        }
+        /**
+         * repeated uint32 delta = 9;
+         *
+         * @return The count of delta.
+         */
+        public int getDeltaCount() {
+          return delta_.size();
+        }
+        /**
+         * repeated uint32 delta = 9;
+         *
+         * @param index The index of the element to return.
+         * @return The delta at the given index.
+         */
+        public int getDelta(int index) {
+          return delta_.getInt(index);
+        }
+
+        private int deltaMemoizedSerializedSize = -1;
+
+        public static final int BUF_FIELD_NUMBER = 10;
+
+        @SuppressWarnings("serial")
+        private com.google.protobuf.Internal.DoubleList buf_ = emptyDoubleList();
+        /**
+         * repeated double buf = 10;
+         *
+         * @return A list containing the buf.
+         */
+        @java.lang.Override
+        public java.util.List getBufList() {
+          return buf_;
+        }
+        /**
+         * repeated double buf = 10;
+         *
+         * @return The count of buf.
+         */
+        public int getBufCount() {
+          return buf_.size();
+        }
+        /**
+         * repeated double buf = 10;
+         *
+         * @param index The index of the element to return.
+         * @return The buf at the given index.
+         */
+        public double getBuf(int index) {
+          return buf_.getDouble(index);
+        }
+
+        private int bufMemoizedSerializedSize = -1;
+
+        private byte memoizedIsInitialized = -1;
+
+        @java.lang.Override
+        public final boolean isInitialized() {
+          byte isInitialized = memoizedIsInitialized;
+          if (isInitialized == 1) return true;
+          if (isInitialized == 0) return false;
+
+          memoizedIsInitialized = 1;
+          return true;
+        }
+
+        @java.lang.Override
+        public void writeTo(com.google.protobuf.CodedOutputStream output)
+            throws java.io.IOException {
+          getSerializedSize();
+          if (ts_ != 0L) {
+            output.writeInt64(1, ts_);
+          }
+          if (cnt_ != 0L) {
+            output.writeInt64(2, cnt_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(min_) != 0) {
+            output.writeDouble(3, min_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(max_) != 0) {
+            output.writeDouble(4, max_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(avg_) != 0) {
+            output.writeDouble(5, avg_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(sum_) != 0) {
+            output.writeDouble(6, sum_);
+          }
+          if (getVList().size() > 0) {
+            output.writeUInt32NoTag(58);
+            output.writeUInt32NoTag(vMemoizedSerializedSize);
+          }
+          for (int i = 0; i < v_.size(); i++) {
+            output.writeDoubleNoTag(v_.getDouble(i));
+          }
+          if (getGList().size() > 0) {
+            output.writeUInt32NoTag(66);
+            output.writeUInt32NoTag(gMemoizedSerializedSize);
+          }
+          for (int i = 0; i < g_.size(); i++) {
+            output.writeUInt32NoTag(g_.getInt(i));
+          }
+          if (getDeltaList().size() > 0) {
+            output.writeUInt32NoTag(74);
+            output.writeUInt32NoTag(deltaMemoizedSerializedSize);
+          }
+          for (int i = 0; i < delta_.size(); i++) {
+            output.writeUInt32NoTag(delta_.getInt(i));
+          }
+          if (getBufList().size() > 0) {
+            output.writeUInt32NoTag(82);
+            output.writeUInt32NoTag(bufMemoizedSerializedSize);
+          }
+          for (int i = 0; i < buf_.size(); i++) {
+            output.writeDoubleNoTag(buf_.getDouble(i));
+          }
+          getUnknownFields().writeTo(output);
+        }
+
+        @java.lang.Override
+        public int getSerializedSize() {
+          int size = memoizedSize;
+          if (size != -1) return size;
+
+          size = 0;
+          if (ts_ != 0L) {
+            size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, ts_);
+          }
+          if (cnt_ != 0L) {
+            size += com.google.protobuf.CodedOutputStream.computeInt64Size(2, cnt_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(min_) != 0) {
+            size += com.google.protobuf.CodedOutputStream.computeDoubleSize(3, min_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(max_) != 0) {
+            size += com.google.protobuf.CodedOutputStream.computeDoubleSize(4, max_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(avg_) != 0) {
+            size += com.google.protobuf.CodedOutputStream.computeDoubleSize(5, avg_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(sum_) != 0) {
+            size += com.google.protobuf.CodedOutputStream.computeDoubleSize(6, sum_);
+          }
+          {
+            int dataSize = 0;
+            dataSize = 8 * getVList().size();
+            size += dataSize;
+            if (!getVList().isEmpty()) {
+              size += 1;
+              size += com.google.protobuf.CodedOutputStream.computeInt32SizeNoTag(dataSize);
+            }
+            vMemoizedSerializedSize = dataSize;
+          }
+          {
+            int dataSize = 0;
+            for (int i = 0; i < g_.size(); i++) {
+              dataSize +=
+                  com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(g_.getInt(i));
+            }
+            size += dataSize;
+            if (!getGList().isEmpty()) {
+              size += 1;
+              size += com.google.protobuf.CodedOutputStream.computeInt32SizeNoTag(dataSize);
+            }
+            gMemoizedSerializedSize = dataSize;
+          }
+          {
+            int dataSize = 0;
+            for (int i = 0; i < delta_.size(); i++) {
+              dataSize +=
+                  com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(delta_.getInt(i));
+            }
+            size += dataSize;
+            if (!getDeltaList().isEmpty()) {
+              size += 1;
+              size += com.google.protobuf.CodedOutputStream.computeInt32SizeNoTag(dataSize);
+            }
+            deltaMemoizedSerializedSize = dataSize;
+          }
+          {
+            int dataSize = 0;
+            dataSize = 8 * getBufList().size();
+            size += dataSize;
+            if (!getBufList().isEmpty()) {
+              size += 1;
+              size += com.google.protobuf.CodedOutputStream.computeInt32SizeNoTag(dataSize);
+            }
+            bufMemoizedSerializedSize = dataSize;
+          }
+          size += getUnknownFields().getSerializedSize();
+          memoizedSize = size;
+          return size;
+        }
+
+        @java.lang.Override
+        public boolean equals(final java.lang.Object obj) {
+          if (obj == this) {
+            return true;
+          }
+          if (!(obj
+              instanceof datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution)) {
+            return super.equals(obj);
+          }
+          datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution other =
+              (datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution) obj;
+
+          if (getTs() != other.getTs()) return false;
+          if (getCnt() != other.getCnt()) return false;
+          if (java.lang.Double.doubleToLongBits(getMin())
+              != java.lang.Double.doubleToLongBits(other.getMin())) return false;
+          if (java.lang.Double.doubleToLongBits(getMax())
+              != java.lang.Double.doubleToLongBits(other.getMax())) return false;
+          if (java.lang.Double.doubleToLongBits(getAvg())
+              != java.lang.Double.doubleToLongBits(other.getAvg())) return false;
+          if (java.lang.Double.doubleToLongBits(getSum())
+              != java.lang.Double.doubleToLongBits(other.getSum())) return false;
+          if (!getVList().equals(other.getVList())) return false;
+          if (!getGList().equals(other.getGList())) return false;
+          if (!getDeltaList().equals(other.getDeltaList())) return false;
+          if (!getBufList().equals(other.getBufList())) return false;
+          if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+          return true;
+        }
+
+        @java.lang.Override
+        public int hashCode() {
+          if (memoizedHashCode != 0) {
+            return memoizedHashCode;
+          }
+          int hash = 41;
+          hash = (19 * hash) + getDescriptor().hashCode();
+          hash = (37 * hash) + TS_FIELD_NUMBER;
+          hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTs());
+          hash = (37 * hash) + CNT_FIELD_NUMBER;
+          hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getCnt());
+          hash = (37 * hash) + MIN_FIELD_NUMBER;
+          hash =
+              (53 * hash)
+                  + com.google.protobuf.Internal.hashLong(
+                      java.lang.Double.doubleToLongBits(getMin()));
+          hash = (37 * hash) + MAX_FIELD_NUMBER;
+          hash =
+              (53 * hash)
+                  + com.google.protobuf.Internal.hashLong(
+                      java.lang.Double.doubleToLongBits(getMax()));
+          hash = (37 * hash) + AVG_FIELD_NUMBER;
+          hash =
+              (53 * hash)
+                  + com.google.protobuf.Internal.hashLong(
+                      java.lang.Double.doubleToLongBits(getAvg()));
+          hash = (37 * hash) + SUM_FIELD_NUMBER;
+          hash =
+              (53 * hash)
+                  + com.google.protobuf.Internal.hashLong(
+                      java.lang.Double.doubleToLongBits(getSum()));
+          if (getVCount() > 0) {
+            hash = (37 * hash) + V_FIELD_NUMBER;
+            hash = (53 * hash) + getVList().hashCode();
+          }
+          if (getGCount() > 0) {
+            hash = (37 * hash) + G_FIELD_NUMBER;
+            hash = (53 * hash) + getGList().hashCode();
+          }
+          if (getDeltaCount() > 0) {
+            hash = (37 * hash) + DELTA_FIELD_NUMBER;
+            hash = (53 * hash) + getDeltaList().hashCode();
+          }
+          if (getBufCount() > 0) {
+            hash = (37 * hash) + BUF_FIELD_NUMBER;
+            hash = (53 * hash) + getBufList().hashCode();
+          }
+          hash = (29 * hash) + getUnknownFields().hashCode();
+          memoizedHashCode = hash;
+          return hash;
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution parseFrom(
+            java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution parseFrom(
+            java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution parseFrom(
+            com.google.protobuf.ByteString data)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution parseFrom(
+            com.google.protobuf.ByteString data,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution parseFrom(
+            byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution parseFrom(
+            byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution parseFrom(
+            java.io.InputStream input) throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution parseFrom(
+            java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+              PARSER, input, extensionRegistry);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+            parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+              PARSER, input);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+            parseDelimitedFrom(
+                java.io.InputStream input,
+                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+              PARSER, input, extensionRegistry);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution parseFrom(
+            com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution parseFrom(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+              PARSER, input, extensionRegistry);
+        }
+
+        @java.lang.Override
+        public Builder newBuilderForType() {
+          return newBuilder();
+        }
+
+        public static Builder newBuilder() {
+          return DEFAULT_INSTANCE.toBuilder();
+        }
+
+        public static Builder newBuilder(
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution prototype) {
+          return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+        }
+
+        @java.lang.Override
+        public Builder toBuilder() {
+          return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+        }
+
+        @java.lang.Override
+        protected Builder newBuilderForType(
+            com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          Builder builder = new Builder(parent);
+          return builder;
+        }
+        /** Protobuf type {@code datadog.agentpayload.SketchPayload.Sketch.Distribution} */
+        public static final class Builder
+            extends com.google.protobuf.GeneratedMessageV3.Builder
+            implements
+            // @@protoc_insertion_point(builder_implements:datadog.agentpayload.SketchPayload.Sketch.Distribution)
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DistributionOrBuilder {
+          public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+            return datadog.agentpayload.AgentPayload
+                .internal_static_datadog_agentpayload_SketchPayload_Sketch_Distribution_descriptor;
+          }
+
+          @java.lang.Override
+          protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+              internalGetFieldAccessorTable() {
+            return datadog.agentpayload.AgentPayload
+                .internal_static_datadog_agentpayload_SketchPayload_Sketch_Distribution_fieldAccessorTable
+                .ensureFieldAccessorsInitialized(
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.class,
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder
+                        .class);
+          }
+
+          // Construct using
+          // datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.newBuilder()
+          private Builder() {}
+
+          private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+            super(parent);
+          }
+
+          @java.lang.Override
+          public Builder clear() {
+            super.clear();
+            bitField0_ = 0;
+            ts_ = 0L;
+            cnt_ = 0L;
+            min_ = 0D;
+            max_ = 0D;
+            avg_ = 0D;
+            sum_ = 0D;
+            v_ = emptyDoubleList();
+            g_ = emptyIntList();
+            delta_ = emptyIntList();
+            buf_ = emptyDoubleList();
+            return this;
+          }
+
+          @java.lang.Override
+          public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+            return datadog.agentpayload.AgentPayload
+                .internal_static_datadog_agentpayload_SketchPayload_Sketch_Distribution_descriptor;
+          }
+
+          @java.lang.Override
+          public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+              getDefaultInstanceForType() {
+            return datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+                .getDefaultInstance();
+          }
+
+          @java.lang.Override
+          public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution build() {
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution result =
+                buildPartial();
+            if (!result.isInitialized()) {
+              throw newUninitializedMessageException(result);
+            }
+            return result;
+          }
+
+          @java.lang.Override
+          public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+              buildPartial() {
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution result =
+                new datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution(this);
+            if (bitField0_ != 0) {
+              buildPartial0(result);
+            }
+            onBuilt();
+            return result;
+          }
+
+          private void buildPartial0(
+              datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution result) {
+            int from_bitField0_ = bitField0_;
+            if (((from_bitField0_ & 0x00000001) != 0)) {
+              result.ts_ = ts_;
+            }
+            if (((from_bitField0_ & 0x00000002) != 0)) {
+              result.cnt_ = cnt_;
+            }
+            if (((from_bitField0_ & 0x00000004) != 0)) {
+              result.min_ = min_;
+            }
+            if (((from_bitField0_ & 0x00000008) != 0)) {
+              result.max_ = max_;
+            }
+            if (((from_bitField0_ & 0x00000010) != 0)) {
+              result.avg_ = avg_;
+            }
+            if (((from_bitField0_ & 0x00000020) != 0)) {
+              result.sum_ = sum_;
+            }
+            if (((from_bitField0_ & 0x00000040) != 0)) {
+              v_.makeImmutable();
+              result.v_ = v_;
+            }
+            if (((from_bitField0_ & 0x00000080) != 0)) {
+              g_.makeImmutable();
+              result.g_ = g_;
+            }
+            if (((from_bitField0_ & 0x00000100) != 0)) {
+              delta_.makeImmutable();
+              result.delta_ = delta_;
+            }
+            if (((from_bitField0_ & 0x00000200) != 0)) {
+              buf_.makeImmutable();
+              result.buf_ = buf_;
+            }
+          }
+
+          @java.lang.Override
+          public Builder clone() {
+            return super.clone();
+          }
+
+          @java.lang.Override
+          public Builder setField(
+              com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+            return super.setField(field, value);
+          }
+
+          @java.lang.Override
+          public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+            return super.clearField(field);
+          }
+
+          @java.lang.Override
+          public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+            return super.clearOneof(oneof);
+          }
+
+          @java.lang.Override
+          public Builder setRepeatedField(
+              com.google.protobuf.Descriptors.FieldDescriptor field,
+              int index,
+              java.lang.Object value) {
+            return super.setRepeatedField(field, index, value);
+          }
+
+          @java.lang.Override
+          public Builder addRepeatedField(
+              com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+            return super.addRepeatedField(field, value);
+          }
+
+          @java.lang.Override
+          public Builder mergeFrom(com.google.protobuf.Message other) {
+            if (other
+                instanceof datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution) {
+              return mergeFrom(
+                  (datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution) other);
+            } else {
+              super.mergeFrom(other);
+              return this;
+            }
+          }
+
+          public Builder mergeFrom(
+              datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution other) {
+            if (other
+                == datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+                    .getDefaultInstance()) return this;
+            if (other.getTs() != 0L) {
+              setTs(other.getTs());
+            }
+            if (other.getCnt() != 0L) {
+              setCnt(other.getCnt());
+            }
+            if (other.getMin() != 0D) {
+              setMin(other.getMin());
+            }
+            if (other.getMax() != 0D) {
+              setMax(other.getMax());
+            }
+            if (other.getAvg() != 0D) {
+              setAvg(other.getAvg());
+            }
+            if (other.getSum() != 0D) {
+              setSum(other.getSum());
+            }
+            if (!other.v_.isEmpty()) {
+              if (v_.isEmpty()) {
+                v_ = other.v_;
+                v_.makeImmutable();
+                bitField0_ |= 0x00000040;
+              } else {
+                ensureVIsMutable();
+                v_.addAll(other.v_);
+              }
+              onChanged();
+            }
+            if (!other.g_.isEmpty()) {
+              if (g_.isEmpty()) {
+                g_ = other.g_;
+                g_.makeImmutable();
+                bitField0_ |= 0x00000080;
+              } else {
+                ensureGIsMutable();
+                g_.addAll(other.g_);
+              }
+              onChanged();
+            }
+            if (!other.delta_.isEmpty()) {
+              if (delta_.isEmpty()) {
+                delta_ = other.delta_;
+                delta_.makeImmutable();
+                bitField0_ |= 0x00000100;
+              } else {
+                ensureDeltaIsMutable();
+                delta_.addAll(other.delta_);
+              }
+              onChanged();
+            }
+            if (!other.buf_.isEmpty()) {
+              if (buf_.isEmpty()) {
+                buf_ = other.buf_;
+                buf_.makeImmutable();
+                bitField0_ |= 0x00000200;
+              } else {
+                ensureBufIsMutable();
+                buf_.addAll(other.buf_);
+              }
+              onChanged();
+            }
+            this.mergeUnknownFields(other.getUnknownFields());
+            onChanged();
+            return this;
+          }
+
+          @java.lang.Override
+          public final boolean isInitialized() {
+            return true;
+          }
+
+          @java.lang.Override
+          public Builder mergeFrom(
+              com.google.protobuf.CodedInputStream input,
+              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+              throws java.io.IOException {
+            if (extensionRegistry == null) {
+              throw new java.lang.NullPointerException();
+            }
+            try {
+              boolean done = false;
+              while (!done) {
+                int tag = input.readTag();
+                switch (tag) {
+                  case 0:
+                    done = true;
+                    break;
+                  case 8:
+                    {
+                      ts_ = input.readInt64();
+                      bitField0_ |= 0x00000001;
+                      break;
+                    } // case 8
+                  case 16:
+                    {
+                      cnt_ = input.readInt64();
+                      bitField0_ |= 0x00000002;
+                      break;
+                    } // case 16
+                  case 25:
+                    {
+                      min_ = input.readDouble();
+                      bitField0_ |= 0x00000004;
+                      break;
+                    } // case 25
+                  case 33:
+                    {
+                      max_ = input.readDouble();
+                      bitField0_ |= 0x00000008;
+                      break;
+                    } // case 33
+                  case 41:
+                    {
+                      avg_ = input.readDouble();
+                      bitField0_ |= 0x00000010;
+                      break;
+                    } // case 41
+                  case 49:
+                    {
+                      sum_ = input.readDouble();
+                      bitField0_ |= 0x00000020;
+                      break;
+                    } // case 49
+                  case 57:
+                    {
+                      double v = input.readDouble();
+                      ensureVIsMutable();
+                      v_.addDouble(v);
+                      break;
+                    } // case 57
+                  case 58:
+                    {
+                      int length = input.readRawVarint32();
+                      int limit = input.pushLimit(length);
+                      int alloc = length > 4096 ? 4096 : length;
+                      ensureVIsMutable(alloc / 8);
+                      while (input.getBytesUntilLimit() > 0) {
+                        v_.addDouble(input.readDouble());
+                      }
+                      input.popLimit(limit);
+                      break;
+                    } // case 58
+                  case 64:
+                    {
+                      int v = input.readUInt32();
+                      ensureGIsMutable();
+                      g_.addInt(v);
+                      break;
+                    } // case 64
+                  case 66:
+                    {
+                      int length = input.readRawVarint32();
+                      int limit = input.pushLimit(length);
+                      ensureGIsMutable();
+                      while (input.getBytesUntilLimit() > 0) {
+                        g_.addInt(input.readUInt32());
+                      }
+                      input.popLimit(limit);
+                      break;
+                    } // case 66
+                  case 72:
+                    {
+                      int v = input.readUInt32();
+                      ensureDeltaIsMutable();
+                      delta_.addInt(v);
+                      break;
+                    } // case 72
+                  case 74:
+                    {
+                      int length = input.readRawVarint32();
+                      int limit = input.pushLimit(length);
+                      ensureDeltaIsMutable();
+                      while (input.getBytesUntilLimit() > 0) {
+                        delta_.addInt(input.readUInt32());
+                      }
+                      input.popLimit(limit);
+                      break;
+                    } // case 74
+                  case 81:
+                    {
+                      double v = input.readDouble();
+                      ensureBufIsMutable();
+                      buf_.addDouble(v);
+                      break;
+                    } // case 81
+                  case 82:
+                    {
+                      int length = input.readRawVarint32();
+                      int limit = input.pushLimit(length);
+                      int alloc = length > 4096 ? 4096 : length;
+                      ensureBufIsMutable(alloc / 8);
+                      while (input.getBytesUntilLimit() > 0) {
+                        buf_.addDouble(input.readDouble());
+                      }
+                      input.popLimit(limit);
+                      break;
+                    } // case 82
+                  default:
+                    {
+                      if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                        done = true; // was an endgroup tag
+                      }
+                      break;
+                    } // default:
+                } // switch (tag)
+              } // while (!done)
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+              throw e.unwrapIOException();
+            } finally {
+              onChanged();
+            } // finally
+            return this;
+          }
+
+          private int bitField0_;
+
+          private long ts_;
+          /**
+           * int64 ts = 1;
+           *
+           * @return The ts.
+           */
+          @java.lang.Override
+          public long getTs() {
+            return ts_;
+          }
+          /**
+           * int64 ts = 1;
+           *
+           * @param value The ts to set.
+           * @return This builder for chaining.
+           */
+          public Builder setTs(long value) {
+
+            ts_ = value;
+            bitField0_ |= 0x00000001;
+            onChanged();
+            return this;
+          }
+          /**
+           * int64 ts = 1;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearTs() {
+            bitField0_ = (bitField0_ & ~0x00000001);
+            ts_ = 0L;
+            onChanged();
+            return this;
+          }
+
+          private long cnt_;
+          /**
+           * int64 cnt = 2;
+           *
+           * @return The cnt.
+           */
+          @java.lang.Override
+          public long getCnt() {
+            return cnt_;
+          }
+          /**
+           * int64 cnt = 2;
+           *
+           * @param value The cnt to set.
+           * @return This builder for chaining.
+           */
+          public Builder setCnt(long value) {
+
+            cnt_ = value;
+            bitField0_ |= 0x00000002;
+            onChanged();
+            return this;
+          }
+          /**
+           * int64 cnt = 2;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearCnt() {
+            bitField0_ = (bitField0_ & ~0x00000002);
+            cnt_ = 0L;
+            onChanged();
+            return this;
+          }
+
+          private double min_;
+          /**
+           * double min = 3;
+           *
+           * @return The min.
+           */
+          @java.lang.Override
+          public double getMin() {
+            return min_;
+          }
+          /**
+           * double min = 3;
+           *
+           * @param value The min to set.
+           * @return This builder for chaining.
+           */
+          public Builder setMin(double value) {
+
+            min_ = value;
+            bitField0_ |= 0x00000004;
+            onChanged();
+            return this;
+          }
+          /**
+           * double min = 3;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearMin() {
+            bitField0_ = (bitField0_ & ~0x00000004);
+            min_ = 0D;
+            onChanged();
+            return this;
+          }
+
+          private double max_;
+          /**
+           * double max = 4;
+           *
+           * @return The max.
+           */
+          @java.lang.Override
+          public double getMax() {
+            return max_;
+          }
+          /**
+           * double max = 4;
+           *
+           * @param value The max to set.
+           * @return This builder for chaining.
+           */
+          public Builder setMax(double value) {
+
+            max_ = value;
+            bitField0_ |= 0x00000008;
+            onChanged();
+            return this;
+          }
+          /**
+           * double max = 4;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearMax() {
+            bitField0_ = (bitField0_ & ~0x00000008);
+            max_ = 0D;
+            onChanged();
+            return this;
+          }
+
+          private double avg_;
+          /**
+           * double avg = 5;
+           *
+           * @return The avg.
+           */
+          @java.lang.Override
+          public double getAvg() {
+            return avg_;
+          }
+          /**
+           * double avg = 5;
+           *
+           * @param value The avg to set.
+           * @return This builder for chaining.
+           */
+          public Builder setAvg(double value) {
+
+            avg_ = value;
+            bitField0_ |= 0x00000010;
+            onChanged();
+            return this;
+          }
+          /**
+           * double avg = 5;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearAvg() {
+            bitField0_ = (bitField0_ & ~0x00000010);
+            avg_ = 0D;
+            onChanged();
+            return this;
+          }
+
+          private double sum_;
+          /**
+           * double sum = 6;
+           *
+           * @return The sum.
+           */
+          @java.lang.Override
+          public double getSum() {
+            return sum_;
+          }
+          /**
+           * double sum = 6;
+           *
+           * @param value The sum to set.
+           * @return This builder for chaining.
+           */
+          public Builder setSum(double value) {
+
+            sum_ = value;
+            bitField0_ |= 0x00000020;
+            onChanged();
+            return this;
+          }
+          /**
+           * double sum = 6;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearSum() {
+            bitField0_ = (bitField0_ & ~0x00000020);
+            sum_ = 0D;
+            onChanged();
+            return this;
+          }
+
+          private com.google.protobuf.Internal.DoubleList v_ = emptyDoubleList();
+
+          private void ensureVIsMutable() {
+            if (!v_.isModifiable()) {
+              v_ = makeMutableCopy(v_);
+            }
+            bitField0_ |= 0x00000040;
+          }
+
+          private void ensureVIsMutable(int capacity) {
+            if (!v_.isModifiable()) {
+              v_ = makeMutableCopy(v_, capacity);
+            }
+            bitField0_ |= 0x00000040;
+          }
+          /**
+           * repeated double v = 7;
+           *
+           * @return A list containing the v.
+           */
+          public java.util.List getVList() {
+            v_.makeImmutable();
+            return v_;
+          }
+          /**
+           * repeated double v = 7;
+           *
+           * @return The count of v.
+           */
+          public int getVCount() {
+            return v_.size();
+          }
+          /**
+           * repeated double v = 7;
+           *
+           * @param index The index of the element to return.
+           * @return The v at the given index.
+           */
+          public double getV(int index) {
+            return v_.getDouble(index);
+          }
+          /**
+           * repeated double v = 7;
+           *
+           * @param index The index to set the value at.
+           * @param value The v to set.
+           * @return This builder for chaining.
+           */
+          public Builder setV(int index, double value) {
+
+            ensureVIsMutable();
+            v_.setDouble(index, value);
+            bitField0_ |= 0x00000040;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated double v = 7;
+           *
+           * @param value The v to add.
+           * @return This builder for chaining.
+           */
+          public Builder addV(double value) {
+
+            ensureVIsMutable();
+            v_.addDouble(value);
+            bitField0_ |= 0x00000040;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated double v = 7;
+           *
+           * @param values The v to add.
+           * @return This builder for chaining.
+           */
+          public Builder addAllV(java.lang.Iterable values) {
+            ensureVIsMutable();
+            com.google.protobuf.AbstractMessageLite.Builder.addAll(values, v_);
+            bitField0_ |= 0x00000040;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated double v = 7;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearV() {
+            v_ = emptyDoubleList();
+            bitField0_ = (bitField0_ & ~0x00000040);
+            onChanged();
+            return this;
+          }
+
+          private com.google.protobuf.Internal.IntList g_ = emptyIntList();
+
+          private void ensureGIsMutable() {
+            if (!g_.isModifiable()) {
+              g_ = makeMutableCopy(g_);
+            }
+            bitField0_ |= 0x00000080;
+          }
+          /**
+           * repeated uint32 g = 8;
+           *
+           * @return A list containing the g.
+           */
+          public java.util.List getGList() {
+            g_.makeImmutable();
+            return g_;
+          }
+          /**
+           * repeated uint32 g = 8;
+           *
+           * @return The count of g.
+           */
+          public int getGCount() {
+            return g_.size();
+          }
+          /**
+           * repeated uint32 g = 8;
+           *
+           * @param index The index of the element to return.
+           * @return The g at the given index.
+           */
+          public int getG(int index) {
+            return g_.getInt(index);
+          }
+          /**
+           * repeated uint32 g = 8;
+           *
+           * @param index The index to set the value at.
+           * @param value The g to set.
+           * @return This builder for chaining.
+           */
+          public Builder setG(int index, int value) {
+
+            ensureGIsMutable();
+            g_.setInt(index, value);
+            bitField0_ |= 0x00000080;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated uint32 g = 8;
+           *
+           * @param value The g to add.
+           * @return This builder for chaining.
+           */
+          public Builder addG(int value) {
+
+            ensureGIsMutable();
+            g_.addInt(value);
+            bitField0_ |= 0x00000080;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated uint32 g = 8;
+           *
+           * @param values The g to add.
+           * @return This builder for chaining.
+           */
+          public Builder addAllG(java.lang.Iterable values) {
+            ensureGIsMutable();
+            com.google.protobuf.AbstractMessageLite.Builder.addAll(values, g_);
+            bitField0_ |= 0x00000080;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated uint32 g = 8;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearG() {
+            g_ = emptyIntList();
+            bitField0_ = (bitField0_ & ~0x00000080);
+            onChanged();
+            return this;
+          }
+
+          private com.google.protobuf.Internal.IntList delta_ = emptyIntList();
+
+          private void ensureDeltaIsMutable() {
+            if (!delta_.isModifiable()) {
+              delta_ = makeMutableCopy(delta_);
+            }
+            bitField0_ |= 0x00000100;
+          }
+          /**
+           * repeated uint32 delta = 9;
+           *
+           * @return A list containing the delta.
+           */
+          public java.util.List getDeltaList() {
+            delta_.makeImmutable();
+            return delta_;
+          }
+          /**
+           * repeated uint32 delta = 9;
+           *
+           * @return The count of delta.
+           */
+          public int getDeltaCount() {
+            return delta_.size();
+          }
+          /**
+           * repeated uint32 delta = 9;
+           *
+           * @param index The index of the element to return.
+           * @return The delta at the given index.
+           */
+          public int getDelta(int index) {
+            return delta_.getInt(index);
+          }
+          /**
+           * repeated uint32 delta = 9;
+           *
+           * @param index The index to set the value at.
+           * @param value The delta to set.
+           * @return This builder for chaining.
+           */
+          public Builder setDelta(int index, int value) {
+
+            ensureDeltaIsMutable();
+            delta_.setInt(index, value);
+            bitField0_ |= 0x00000100;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated uint32 delta = 9;
+           *
+           * @param value The delta to add.
+           * @return This builder for chaining.
+           */
+          public Builder addDelta(int value) {
+
+            ensureDeltaIsMutable();
+            delta_.addInt(value);
+            bitField0_ |= 0x00000100;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated uint32 delta = 9;
+           *
+           * @param values The delta to add.
+           * @return This builder for chaining.
+           */
+          public Builder addAllDelta(java.lang.Iterable values) {
+            ensureDeltaIsMutable();
+            com.google.protobuf.AbstractMessageLite.Builder.addAll(values, delta_);
+            bitField0_ |= 0x00000100;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated uint32 delta = 9;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearDelta() {
+            delta_ = emptyIntList();
+            bitField0_ = (bitField0_ & ~0x00000100);
+            onChanged();
+            return this;
+          }
+
+          private com.google.protobuf.Internal.DoubleList buf_ = emptyDoubleList();
+
+          private void ensureBufIsMutable() {
+            if (!buf_.isModifiable()) {
+              buf_ = makeMutableCopy(buf_);
+            }
+            bitField0_ |= 0x00000200;
+          }
+
+          private void ensureBufIsMutable(int capacity) {
+            if (!buf_.isModifiable()) {
+              buf_ = makeMutableCopy(buf_, capacity);
+            }
+            bitField0_ |= 0x00000200;
+          }
+          /**
+           * repeated double buf = 10;
+           *
+           * @return A list containing the buf.
+           */
+          public java.util.List getBufList() {
+            buf_.makeImmutable();
+            return buf_;
+          }
+          /**
+           * repeated double buf = 10;
+           *
+           * @return The count of buf.
+           */
+          public int getBufCount() {
+            return buf_.size();
+          }
+          /**
+           * repeated double buf = 10;
+           *
+           * @param index The index of the element to return.
+           * @return The buf at the given index.
+           */
+          public double getBuf(int index) {
+            return buf_.getDouble(index);
+          }
+          /**
+           * repeated double buf = 10;
+           *
+           * @param index The index to set the value at.
+           * @param value The buf to set.
+           * @return This builder for chaining.
+           */
+          public Builder setBuf(int index, double value) {
+
+            ensureBufIsMutable();
+            buf_.setDouble(index, value);
+            bitField0_ |= 0x00000200;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated double buf = 10;
+           *
+           * @param value The buf to add.
+           * @return This builder for chaining.
+           */
+          public Builder addBuf(double value) {
+
+            ensureBufIsMutable();
+            buf_.addDouble(value);
+            bitField0_ |= 0x00000200;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated double buf = 10;
+           *
+           * @param values The buf to add.
+           * @return This builder for chaining.
+           */
+          public Builder addAllBuf(java.lang.Iterable values) {
+            ensureBufIsMutable();
+            com.google.protobuf.AbstractMessageLite.Builder.addAll(values, buf_);
+            bitField0_ |= 0x00000200;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated double buf = 10;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearBuf() {
+            buf_ = emptyDoubleList();
+            bitField0_ = (bitField0_ & ~0x00000200);
+            onChanged();
+            return this;
+          }
+
+          @java.lang.Override
+          public final Builder setUnknownFields(
+              final com.google.protobuf.UnknownFieldSet unknownFields) {
+            return super.setUnknownFields(unknownFields);
+          }
+
+          @java.lang.Override
+          public final Builder mergeUnknownFields(
+              final com.google.protobuf.UnknownFieldSet unknownFields) {
+            return super.mergeUnknownFields(unknownFields);
+          }
+
+          // @@protoc_insertion_point(builder_scope:datadog.agentpayload.SketchPayload.Sketch.Distribution)
+        }
+
+        // @@protoc_insertion_point(class_scope:datadog.agentpayload.SketchPayload.Sketch.Distribution)
+        private static final datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+            DEFAULT_INSTANCE;
+
+        static {
+          DEFAULT_INSTANCE =
+              new datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution();
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+            getDefaultInstance() {
+          return DEFAULT_INSTANCE;
+        }
+
+        private static final com.google.protobuf.Parser PARSER =
+            new com.google.protobuf.AbstractParser() {
+              @java.lang.Override
+              public Distribution parsePartialFrom(
+                  com.google.protobuf.CodedInputStream input,
+                  com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                  throws com.google.protobuf.InvalidProtocolBufferException {
+                Builder builder = newBuilder();
+                try {
+                  builder.mergeFrom(input, extensionRegistry);
+                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                  throw e.setUnfinishedMessage(builder.buildPartial());
+                } catch (com.google.protobuf.UninitializedMessageException e) {
+                  throw e.asInvalidProtocolBufferException()
+                      .setUnfinishedMessage(builder.buildPartial());
+                } catch (java.io.IOException e) {
+                  throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                      .setUnfinishedMessage(builder.buildPartial());
+                }
+                return builder.buildPartial();
+              }
+            };
+
+        public static com.google.protobuf.Parser parser() {
+          return PARSER;
+        }
+
+        @java.lang.Override
+        public com.google.protobuf.Parser getParserForType() {
+          return PARSER;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+            getDefaultInstanceForType() {
+          return DEFAULT_INSTANCE;
+        }
+      }
+
+      public interface DogsketchOrBuilder
+          extends
+          // @@protoc_insertion_point(interface_extends:datadog.agentpayload.SketchPayload.Sketch.Dogsketch)
+          com.google.protobuf.MessageOrBuilder {
+
+        /**
+         * int64 ts = 1;
+         *
+         * @return The ts.
+         */
+        long getTs();
+
+        /**
+         * int64 cnt = 2;
+         *
+         * @return The cnt.
+         */
+        long getCnt();
+
+        /**
+         * double min = 3;
+         *
+         * @return The min.
+         */
+        double getMin();
+
+        /**
+         * double max = 4;
+         *
+         * @return The max.
+         */
+        double getMax();
+
+        /**
+         * double avg = 5;
+         *
+         * @return The avg.
+         */
+        double getAvg();
+
+        /**
+         * double sum = 6;
+         *
+         * @return The sum.
+         */
+        double getSum();
+
+        /**
+         * repeated sint32 k = 7;
+         *
+         * @return A list containing the k.
+         */
+        java.util.List getKList();
+        /**
+         * repeated sint32 k = 7;
+         *
+         * @return The count of k.
+         */
+        int getKCount();
+        /**
+         * repeated sint32 k = 7;
+         *
+         * @param index The index of the element to return.
+         * @return The k at the given index.
+         */
+        int getK(int index);
+
+        /**
+         * repeated uint32 n = 8;
+         *
+         * @return A list containing the n.
+         */
+        java.util.List getNList();
+        /**
+         * repeated uint32 n = 8;
+         *
+         * @return The count of n.
+         */
+        int getNCount();
+        /**
+         * repeated uint32 n = 8;
+         *
+         * @param index The index of the element to return.
+         * @return The n at the given index.
+         */
+        int getN(int index);
+      }
+      /** Protobuf type {@code datadog.agentpayload.SketchPayload.Sketch.Dogsketch} */
+      public static final class Dogsketch extends com.google.protobuf.GeneratedMessageV3
+          implements
+          // @@protoc_insertion_point(message_implements:datadog.agentpayload.SketchPayload.Sketch.Dogsketch)
+          DogsketchOrBuilder {
+        private static final long serialVersionUID = 0L;
+        // Use Dogsketch.newBuilder() to construct.
+        private Dogsketch(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+          super(builder);
+        }
+
+        private Dogsketch() {
+          k_ = emptyIntList();
+          n_ = emptyIntList();
+        }
+
+        @java.lang.Override
+        @SuppressWarnings({"unused"})
+        protected java.lang.Object newInstance(UnusedPrivateParameter unused) {
+          return new Dogsketch();
+        }
+
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_SketchPayload_Sketch_Dogsketch_descriptor;
+        }
+
+        @java.lang.Override
+        protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+            internalGetFieldAccessorTable() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_SketchPayload_Sketch_Dogsketch_fieldAccessorTable
+              .ensureFieldAccessorsInitialized(
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.class,
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder.class);
+        }
+
+        public static final int TS_FIELD_NUMBER = 1;
+        private long ts_ = 0L;
+        /**
+         * int64 ts = 1;
+         *
+         * @return The ts.
+         */
+        @java.lang.Override
+        public long getTs() {
+          return ts_;
+        }
+
+        public static final int CNT_FIELD_NUMBER = 2;
+        private long cnt_ = 0L;
+        /**
+         * int64 cnt = 2;
+         *
+         * @return The cnt.
+         */
+        @java.lang.Override
+        public long getCnt() {
+          return cnt_;
+        }
+
+        public static final int MIN_FIELD_NUMBER = 3;
+        private double min_ = 0D;
+        /**
+         * double min = 3;
+         *
+         * @return The min.
+         */
+        @java.lang.Override
+        public double getMin() {
+          return min_;
+        }
+
+        public static final int MAX_FIELD_NUMBER = 4;
+        private double max_ = 0D;
+        /**
+         * double max = 4;
+         *
+         * @return The max.
+         */
+        @java.lang.Override
+        public double getMax() {
+          return max_;
+        }
+
+        public static final int AVG_FIELD_NUMBER = 5;
+        private double avg_ = 0D;
+        /**
+         * double avg = 5;
+         *
+         * @return The avg.
+         */
+        @java.lang.Override
+        public double getAvg() {
+          return avg_;
+        }
+
+        public static final int SUM_FIELD_NUMBER = 6;
+        private double sum_ = 0D;
+        /**
+         * double sum = 6;
+         *
+         * @return The sum.
+         */
+        @java.lang.Override
+        public double getSum() {
+          return sum_;
+        }
+
+        public static final int K_FIELD_NUMBER = 7;
+
+        @SuppressWarnings("serial")
+        private com.google.protobuf.Internal.IntList k_ = emptyIntList();
+        /**
+         * repeated sint32 k = 7;
+         *
+         * @return A list containing the k.
+         */
+        @java.lang.Override
+        public java.util.List getKList() {
+          return k_;
+        }
+        /**
+         * repeated sint32 k = 7;
+         *
+         * @return The count of k.
+         */
+        public int getKCount() {
+          return k_.size();
+        }
+        /**
+         * repeated sint32 k = 7;
+         *
+         * @param index The index of the element to return.
+         * @return The k at the given index.
+         */
+        public int getK(int index) {
+          return k_.getInt(index);
+        }
+
+        private int kMemoizedSerializedSize = -1;
+
+        public static final int N_FIELD_NUMBER = 8;
+
+        @SuppressWarnings("serial")
+        private com.google.protobuf.Internal.IntList n_ = emptyIntList();
+        /**
+         * repeated uint32 n = 8;
+         *
+         * @return A list containing the n.
+         */
+        @java.lang.Override
+        public java.util.List getNList() {
+          return n_;
+        }
+        /**
+         * repeated uint32 n = 8;
+         *
+         * @return The count of n.
+         */
+        public int getNCount() {
+          return n_.size();
+        }
+        /**
+         * repeated uint32 n = 8;
+         *
+         * @param index The index of the element to return.
+         * @return The n at the given index.
+         */
+        public int getN(int index) {
+          return n_.getInt(index);
+        }
+
+        private int nMemoizedSerializedSize = -1;
+
+        private byte memoizedIsInitialized = -1;
+
+        @java.lang.Override
+        public final boolean isInitialized() {
+          byte isInitialized = memoizedIsInitialized;
+          if (isInitialized == 1) return true;
+          if (isInitialized == 0) return false;
+
+          memoizedIsInitialized = 1;
+          return true;
+        }
+
+        @java.lang.Override
+        public void writeTo(com.google.protobuf.CodedOutputStream output)
+            throws java.io.IOException {
+          getSerializedSize();
+          if (ts_ != 0L) {
+            output.writeInt64(1, ts_);
+          }
+          if (cnt_ != 0L) {
+            output.writeInt64(2, cnt_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(min_) != 0) {
+            output.writeDouble(3, min_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(max_) != 0) {
+            output.writeDouble(4, max_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(avg_) != 0) {
+            output.writeDouble(5, avg_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(sum_) != 0) {
+            output.writeDouble(6, sum_);
+          }
+          if (getKList().size() > 0) {
+            output.writeUInt32NoTag(58);
+            output.writeUInt32NoTag(kMemoizedSerializedSize);
+          }
+          for (int i = 0; i < k_.size(); i++) {
+            output.writeSInt32NoTag(k_.getInt(i));
+          }
+          if (getNList().size() > 0) {
+            output.writeUInt32NoTag(66);
+            output.writeUInt32NoTag(nMemoizedSerializedSize);
+          }
+          for (int i = 0; i < n_.size(); i++) {
+            output.writeUInt32NoTag(n_.getInt(i));
+          }
+          getUnknownFields().writeTo(output);
+        }
+
+        @java.lang.Override
+        public int getSerializedSize() {
+          int size = memoizedSize;
+          if (size != -1) return size;
+
+          size = 0;
+          if (ts_ != 0L) {
+            size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, ts_);
+          }
+          if (cnt_ != 0L) {
+            size += com.google.protobuf.CodedOutputStream.computeInt64Size(2, cnt_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(min_) != 0) {
+            size += com.google.protobuf.CodedOutputStream.computeDoubleSize(3, min_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(max_) != 0) {
+            size += com.google.protobuf.CodedOutputStream.computeDoubleSize(4, max_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(avg_) != 0) {
+            size += com.google.protobuf.CodedOutputStream.computeDoubleSize(5, avg_);
+          }
+          if (java.lang.Double.doubleToRawLongBits(sum_) != 0) {
+            size += com.google.protobuf.CodedOutputStream.computeDoubleSize(6, sum_);
+          }
+          {
+            int dataSize = 0;
+            for (int i = 0; i < k_.size(); i++) {
+              dataSize +=
+                  com.google.protobuf.CodedOutputStream.computeSInt32SizeNoTag(k_.getInt(i));
+            }
+            size += dataSize;
+            if (!getKList().isEmpty()) {
+              size += 1;
+              size += com.google.protobuf.CodedOutputStream.computeInt32SizeNoTag(dataSize);
+            }
+            kMemoizedSerializedSize = dataSize;
+          }
+          {
+            int dataSize = 0;
+            for (int i = 0; i < n_.size(); i++) {
+              dataSize +=
+                  com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(n_.getInt(i));
+            }
+            size += dataSize;
+            if (!getNList().isEmpty()) {
+              size += 1;
+              size += com.google.protobuf.CodedOutputStream.computeInt32SizeNoTag(dataSize);
+            }
+            nMemoizedSerializedSize = dataSize;
+          }
+          size += getUnknownFields().getSerializedSize();
+          memoizedSize = size;
+          return size;
+        }
+
+        @java.lang.Override
+        public boolean equals(final java.lang.Object obj) {
+          if (obj == this) {
+            return true;
+          }
+          if (!(obj instanceof datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch)) {
+            return super.equals(obj);
+          }
+          datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch other =
+              (datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch) obj;
+
+          if (getTs() != other.getTs()) return false;
+          if (getCnt() != other.getCnt()) return false;
+          if (java.lang.Double.doubleToLongBits(getMin())
+              != java.lang.Double.doubleToLongBits(other.getMin())) return false;
+          if (java.lang.Double.doubleToLongBits(getMax())
+              != java.lang.Double.doubleToLongBits(other.getMax())) return false;
+          if (java.lang.Double.doubleToLongBits(getAvg())
+              != java.lang.Double.doubleToLongBits(other.getAvg())) return false;
+          if (java.lang.Double.doubleToLongBits(getSum())
+              != java.lang.Double.doubleToLongBits(other.getSum())) return false;
+          if (!getKList().equals(other.getKList())) return false;
+          if (!getNList().equals(other.getNList())) return false;
+          if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+          return true;
+        }
+
+        @java.lang.Override
+        public int hashCode() {
+          if (memoizedHashCode != 0) {
+            return memoizedHashCode;
+          }
+          int hash = 41;
+          hash = (19 * hash) + getDescriptor().hashCode();
+          hash = (37 * hash) + TS_FIELD_NUMBER;
+          hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTs());
+          hash = (37 * hash) + CNT_FIELD_NUMBER;
+          hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getCnt());
+          hash = (37 * hash) + MIN_FIELD_NUMBER;
+          hash =
+              (53 * hash)
+                  + com.google.protobuf.Internal.hashLong(
+                      java.lang.Double.doubleToLongBits(getMin()));
+          hash = (37 * hash) + MAX_FIELD_NUMBER;
+          hash =
+              (53 * hash)
+                  + com.google.protobuf.Internal.hashLong(
+                      java.lang.Double.doubleToLongBits(getMax()));
+          hash = (37 * hash) + AVG_FIELD_NUMBER;
+          hash =
+              (53 * hash)
+                  + com.google.protobuf.Internal.hashLong(
+                      java.lang.Double.doubleToLongBits(getAvg()));
+          hash = (37 * hash) + SUM_FIELD_NUMBER;
+          hash =
+              (53 * hash)
+                  + com.google.protobuf.Internal.hashLong(
+                      java.lang.Double.doubleToLongBits(getSum()));
+          if (getKCount() > 0) {
+            hash = (37 * hash) + K_FIELD_NUMBER;
+            hash = (53 * hash) + getKList().hashCode();
+          }
+          if (getNCount() > 0) {
+            hash = (37 * hash) + N_FIELD_NUMBER;
+            hash = (53 * hash) + getNList().hashCode();
+          }
+          hash = (29 * hash) + getUnknownFields().hashCode();
+          memoizedHashCode = hash;
+          return hash;
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch parseFrom(
+            java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch parseFrom(
+            java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch parseFrom(
+            com.google.protobuf.ByteString data)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch parseFrom(
+            com.google.protobuf.ByteString data,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch parseFrom(
+            byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch parseFrom(
+            byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws com.google.protobuf.InvalidProtocolBufferException {
+          return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch parseFrom(
+            java.io.InputStream input) throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch parseFrom(
+            java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+              PARSER, input, extensionRegistry);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+            parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+              PARSER, input);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+            parseDelimitedFrom(
+                java.io.InputStream input,
+                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+              PARSER, input, extensionRegistry);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch parseFrom(
+            com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch parseFrom(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+          return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+              PARSER, input, extensionRegistry);
+        }
+
+        @java.lang.Override
+        public Builder newBuilderForType() {
+          return newBuilder();
+        }
+
+        public static Builder newBuilder() {
+          return DEFAULT_INSTANCE.toBuilder();
+        }
+
+        public static Builder newBuilder(
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch prototype) {
+          return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+        }
+
+        @java.lang.Override
+        public Builder toBuilder() {
+          return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+        }
+
+        @java.lang.Override
+        protected Builder newBuilderForType(
+            com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          Builder builder = new Builder(parent);
+          return builder;
+        }
+        /** Protobuf type {@code datadog.agentpayload.SketchPayload.Sketch.Dogsketch} */
+        public static final class Builder
+            extends com.google.protobuf.GeneratedMessageV3.Builder
+            implements
+            // @@protoc_insertion_point(builder_implements:datadog.agentpayload.SketchPayload.Sketch.Dogsketch)
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DogsketchOrBuilder {
+          public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+            return datadog.agentpayload.AgentPayload
+                .internal_static_datadog_agentpayload_SketchPayload_Sketch_Dogsketch_descriptor;
+          }
+
+          @java.lang.Override
+          protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+              internalGetFieldAccessorTable() {
+            return datadog.agentpayload.AgentPayload
+                .internal_static_datadog_agentpayload_SketchPayload_Sketch_Dogsketch_fieldAccessorTable
+                .ensureFieldAccessorsInitialized(
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.class,
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder.class);
+          }
+
+          // Construct using
+          // datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.newBuilder()
+          private Builder() {}
+
+          private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+            super(parent);
+          }
+
+          @java.lang.Override
+          public Builder clear() {
+            super.clear();
+            bitField0_ = 0;
+            ts_ = 0L;
+            cnt_ = 0L;
+            min_ = 0D;
+            max_ = 0D;
+            avg_ = 0D;
+            sum_ = 0D;
+            k_ = emptyIntList();
+            n_ = emptyIntList();
+            return this;
+          }
+
+          @java.lang.Override
+          public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+            return datadog.agentpayload.AgentPayload
+                .internal_static_datadog_agentpayload_SketchPayload_Sketch_Dogsketch_descriptor;
+          }
+
+          @java.lang.Override
+          public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+              getDefaultInstanceForType() {
+            return datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+                .getDefaultInstance();
+          }
+
+          @java.lang.Override
+          public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch build() {
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch result =
+                buildPartial();
+            if (!result.isInitialized()) {
+              throw newUninitializedMessageException(result);
+            }
+            return result;
+          }
+
+          @java.lang.Override
+          public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch buildPartial() {
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch result =
+                new datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch(this);
+            if (bitField0_ != 0) {
+              buildPartial0(result);
+            }
+            onBuilt();
+            return result;
+          }
+
+          private void buildPartial0(
+              datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch result) {
+            int from_bitField0_ = bitField0_;
+            if (((from_bitField0_ & 0x00000001) != 0)) {
+              result.ts_ = ts_;
+            }
+            if (((from_bitField0_ & 0x00000002) != 0)) {
+              result.cnt_ = cnt_;
+            }
+            if (((from_bitField0_ & 0x00000004) != 0)) {
+              result.min_ = min_;
+            }
+            if (((from_bitField0_ & 0x00000008) != 0)) {
+              result.max_ = max_;
+            }
+            if (((from_bitField0_ & 0x00000010) != 0)) {
+              result.avg_ = avg_;
+            }
+            if (((from_bitField0_ & 0x00000020) != 0)) {
+              result.sum_ = sum_;
+            }
+            if (((from_bitField0_ & 0x00000040) != 0)) {
+              k_.makeImmutable();
+              result.k_ = k_;
+            }
+            if (((from_bitField0_ & 0x00000080) != 0)) {
+              n_.makeImmutable();
+              result.n_ = n_;
+            }
+          }
+
+          @java.lang.Override
+          public Builder clone() {
+            return super.clone();
+          }
+
+          @java.lang.Override
+          public Builder setField(
+              com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+            return super.setField(field, value);
+          }
+
+          @java.lang.Override
+          public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+            return super.clearField(field);
+          }
+
+          @java.lang.Override
+          public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+            return super.clearOneof(oneof);
+          }
+
+          @java.lang.Override
+          public Builder setRepeatedField(
+              com.google.protobuf.Descriptors.FieldDescriptor field,
+              int index,
+              java.lang.Object value) {
+            return super.setRepeatedField(field, index, value);
+          }
+
+          @java.lang.Override
+          public Builder addRepeatedField(
+              com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+            return super.addRepeatedField(field, value);
+          }
+
+          @java.lang.Override
+          public Builder mergeFrom(com.google.protobuf.Message other) {
+            if (other instanceof datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch) {
+              return mergeFrom(
+                  (datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch) other);
+            } else {
+              super.mergeFrom(other);
+              return this;
+            }
+          }
+
+          public Builder mergeFrom(
+              datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch other) {
+            if (other
+                == datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+                    .getDefaultInstance()) return this;
+            if (other.getTs() != 0L) {
+              setTs(other.getTs());
+            }
+            if (other.getCnt() != 0L) {
+              setCnt(other.getCnt());
+            }
+            if (other.getMin() != 0D) {
+              setMin(other.getMin());
+            }
+            if (other.getMax() != 0D) {
+              setMax(other.getMax());
+            }
+            if (other.getAvg() != 0D) {
+              setAvg(other.getAvg());
+            }
+            if (other.getSum() != 0D) {
+              setSum(other.getSum());
+            }
+            if (!other.k_.isEmpty()) {
+              if (k_.isEmpty()) {
+                k_ = other.k_;
+                k_.makeImmutable();
+                bitField0_ |= 0x00000040;
+              } else {
+                ensureKIsMutable();
+                k_.addAll(other.k_);
+              }
+              onChanged();
+            }
+            if (!other.n_.isEmpty()) {
+              if (n_.isEmpty()) {
+                n_ = other.n_;
+                n_.makeImmutable();
+                bitField0_ |= 0x00000080;
+              } else {
+                ensureNIsMutable();
+                n_.addAll(other.n_);
+              }
+              onChanged();
+            }
+            this.mergeUnknownFields(other.getUnknownFields());
+            onChanged();
+            return this;
+          }
+
+          @java.lang.Override
+          public final boolean isInitialized() {
+            return true;
+          }
+
+          @java.lang.Override
+          public Builder mergeFrom(
+              com.google.protobuf.CodedInputStream input,
+              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+              throws java.io.IOException {
+            if (extensionRegistry == null) {
+              throw new java.lang.NullPointerException();
+            }
+            try {
+              boolean done = false;
+              while (!done) {
+                int tag = input.readTag();
+                switch (tag) {
+                  case 0:
+                    done = true;
+                    break;
+                  case 8:
+                    {
+                      ts_ = input.readInt64();
+                      bitField0_ |= 0x00000001;
+                      break;
+                    } // case 8
+                  case 16:
+                    {
+                      cnt_ = input.readInt64();
+                      bitField0_ |= 0x00000002;
+                      break;
+                    } // case 16
+                  case 25:
+                    {
+                      min_ = input.readDouble();
+                      bitField0_ |= 0x00000004;
+                      break;
+                    } // case 25
+                  case 33:
+                    {
+                      max_ = input.readDouble();
+                      bitField0_ |= 0x00000008;
+                      break;
+                    } // case 33
+                  case 41:
+                    {
+                      avg_ = input.readDouble();
+                      bitField0_ |= 0x00000010;
+                      break;
+                    } // case 41
+                  case 49:
+                    {
+                      sum_ = input.readDouble();
+                      bitField0_ |= 0x00000020;
+                      break;
+                    } // case 49
+                  case 56:
+                    {
+                      int v = input.readSInt32();
+                      ensureKIsMutable();
+                      k_.addInt(v);
+                      break;
+                    } // case 56
+                  case 58:
+                    {
+                      int length = input.readRawVarint32();
+                      int limit = input.pushLimit(length);
+                      ensureKIsMutable();
+                      while (input.getBytesUntilLimit() > 0) {
+                        k_.addInt(input.readSInt32());
+                      }
+                      input.popLimit(limit);
+                      break;
+                    } // case 58
+                  case 64:
+                    {
+                      int v = input.readUInt32();
+                      ensureNIsMutable();
+                      n_.addInt(v);
+                      break;
+                    } // case 64
+                  case 66:
+                    {
+                      int length = input.readRawVarint32();
+                      int limit = input.pushLimit(length);
+                      ensureNIsMutable();
+                      while (input.getBytesUntilLimit() > 0) {
+                        n_.addInt(input.readUInt32());
+                      }
+                      input.popLimit(limit);
+                      break;
+                    } // case 66
+                  default:
+                    {
+                      if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                        done = true; // was an endgroup tag
+                      }
+                      break;
+                    } // default:
+                } // switch (tag)
+              } // while (!done)
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+              throw e.unwrapIOException();
+            } finally {
+              onChanged();
+            } // finally
+            return this;
+          }
+
+          private int bitField0_;
+
+          private long ts_;
+          /**
+           * int64 ts = 1;
+           *
+           * @return The ts.
+           */
+          @java.lang.Override
+          public long getTs() {
+            return ts_;
+          }
+          /**
+           * int64 ts = 1;
+           *
+           * @param value The ts to set.
+           * @return This builder for chaining.
+           */
+          public Builder setTs(long value) {
+
+            ts_ = value;
+            bitField0_ |= 0x00000001;
+            onChanged();
+            return this;
+          }
+          /**
+           * int64 ts = 1;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearTs() {
+            bitField0_ = (bitField0_ & ~0x00000001);
+            ts_ = 0L;
+            onChanged();
+            return this;
+          }
+
+          private long cnt_;
+          /**
+           * int64 cnt = 2;
+           *
+           * @return The cnt.
+           */
+          @java.lang.Override
+          public long getCnt() {
+            return cnt_;
+          }
+          /**
+           * int64 cnt = 2;
+           *
+           * @param value The cnt to set.
+           * @return This builder for chaining.
+           */
+          public Builder setCnt(long value) {
+
+            cnt_ = value;
+            bitField0_ |= 0x00000002;
+            onChanged();
+            return this;
+          }
+          /**
+           * int64 cnt = 2;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearCnt() {
+            bitField0_ = (bitField0_ & ~0x00000002);
+            cnt_ = 0L;
+            onChanged();
+            return this;
+          }
+
+          private double min_;
+          /**
+           * double min = 3;
+           *
+           * @return The min.
+           */
+          @java.lang.Override
+          public double getMin() {
+            return min_;
+          }
+          /**
+           * double min = 3;
+           *
+           * @param value The min to set.
+           * @return This builder for chaining.
+           */
+          public Builder setMin(double value) {
+
+            min_ = value;
+            bitField0_ |= 0x00000004;
+            onChanged();
+            return this;
+          }
+          /**
+           * double min = 3;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearMin() {
+            bitField0_ = (bitField0_ & ~0x00000004);
+            min_ = 0D;
+            onChanged();
+            return this;
+          }
+
+          private double max_;
+          /**
+           * double max = 4;
+           *
+           * @return The max.
+           */
+          @java.lang.Override
+          public double getMax() {
+            return max_;
+          }
+          /**
+           * double max = 4;
+           *
+           * @param value The max to set.
+           * @return This builder for chaining.
+           */
+          public Builder setMax(double value) {
+
+            max_ = value;
+            bitField0_ |= 0x00000008;
+            onChanged();
+            return this;
+          }
+          /**
+           * double max = 4;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearMax() {
+            bitField0_ = (bitField0_ & ~0x00000008);
+            max_ = 0D;
+            onChanged();
+            return this;
+          }
+
+          private double avg_;
+          /**
+           * double avg = 5;
+           *
+           * @return The avg.
+           */
+          @java.lang.Override
+          public double getAvg() {
+            return avg_;
+          }
+          /**
+           * double avg = 5;
+           *
+           * @param value The avg to set.
+           * @return This builder for chaining.
+           */
+          public Builder setAvg(double value) {
+
+            avg_ = value;
+            bitField0_ |= 0x00000010;
+            onChanged();
+            return this;
+          }
+          /**
+           * double avg = 5;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearAvg() {
+            bitField0_ = (bitField0_ & ~0x00000010);
+            avg_ = 0D;
+            onChanged();
+            return this;
+          }
+
+          private double sum_;
+          /**
+           * double sum = 6;
+           *
+           * @return The sum.
+           */
+          @java.lang.Override
+          public double getSum() {
+            return sum_;
+          }
+          /**
+           * double sum = 6;
+           *
+           * @param value The sum to set.
+           * @return This builder for chaining.
+           */
+          public Builder setSum(double value) {
+
+            sum_ = value;
+            bitField0_ |= 0x00000020;
+            onChanged();
+            return this;
+          }
+          /**
+           * double sum = 6;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearSum() {
+            bitField0_ = (bitField0_ & ~0x00000020);
+            sum_ = 0D;
+            onChanged();
+            return this;
+          }
+
+          private com.google.protobuf.Internal.IntList k_ = emptyIntList();
+
+          private void ensureKIsMutable() {
+            if (!k_.isModifiable()) {
+              k_ = makeMutableCopy(k_);
+            }
+            bitField0_ |= 0x00000040;
+          }
+          /**
+           * repeated sint32 k = 7;
+           *
+           * @return A list containing the k.
+           */
+          public java.util.List getKList() {
+            k_.makeImmutable();
+            return k_;
+          }
+          /**
+           * repeated sint32 k = 7;
+           *
+           * @return The count of k.
+           */
+          public int getKCount() {
+            return k_.size();
+          }
+          /**
+           * repeated sint32 k = 7;
+           *
+           * @param index The index of the element to return.
+           * @return The k at the given index.
+           */
+          public int getK(int index) {
+            return k_.getInt(index);
+          }
+          /**
+           * repeated sint32 k = 7;
+           *
+           * @param index The index to set the value at.
+           * @param value The k to set.
+           * @return This builder for chaining.
+           */
+          public Builder setK(int index, int value) {
+
+            ensureKIsMutable();
+            k_.setInt(index, value);
+            bitField0_ |= 0x00000040;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated sint32 k = 7;
+           *
+           * @param value The k to add.
+           * @return This builder for chaining.
+           */
+          public Builder addK(int value) {
+
+            ensureKIsMutable();
+            k_.addInt(value);
+            bitField0_ |= 0x00000040;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated sint32 k = 7;
+           *
+           * @param values The k to add.
+           * @return This builder for chaining.
+           */
+          public Builder addAllK(java.lang.Iterable values) {
+            ensureKIsMutable();
+            com.google.protobuf.AbstractMessageLite.Builder.addAll(values, k_);
+            bitField0_ |= 0x00000040;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated sint32 k = 7;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearK() {
+            k_ = emptyIntList();
+            bitField0_ = (bitField0_ & ~0x00000040);
+            onChanged();
+            return this;
+          }
+
+          private com.google.protobuf.Internal.IntList n_ = emptyIntList();
+
+          private void ensureNIsMutable() {
+            if (!n_.isModifiable()) {
+              n_ = makeMutableCopy(n_);
+            }
+            bitField0_ |= 0x00000080;
+          }
+          /**
+           * repeated uint32 n = 8;
+           *
+           * @return A list containing the n.
+           */
+          public java.util.List getNList() {
+            n_.makeImmutable();
+            return n_;
+          }
+          /**
+           * repeated uint32 n = 8;
+           *
+           * @return The count of n.
+           */
+          public int getNCount() {
+            return n_.size();
+          }
+          /**
+           * repeated uint32 n = 8;
+           *
+           * @param index The index of the element to return.
+           * @return The n at the given index.
+           */
+          public int getN(int index) {
+            return n_.getInt(index);
+          }
+          /**
+           * repeated uint32 n = 8;
+           *
+           * @param index The index to set the value at.
+           * @param value The n to set.
+           * @return This builder for chaining.
+           */
+          public Builder setN(int index, int value) {
+
+            ensureNIsMutable();
+            n_.setInt(index, value);
+            bitField0_ |= 0x00000080;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated uint32 n = 8;
+           *
+           * @param value The n to add.
+           * @return This builder for chaining.
+           */
+          public Builder addN(int value) {
+
+            ensureNIsMutable();
+            n_.addInt(value);
+            bitField0_ |= 0x00000080;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated uint32 n = 8;
+           *
+           * @param values The n to add.
+           * @return This builder for chaining.
+           */
+          public Builder addAllN(java.lang.Iterable values) {
+            ensureNIsMutable();
+            com.google.protobuf.AbstractMessageLite.Builder.addAll(values, n_);
+            bitField0_ |= 0x00000080;
+            onChanged();
+            return this;
+          }
+          /**
+           * repeated uint32 n = 8;
+           *
+           * @return This builder for chaining.
+           */
+          public Builder clearN() {
+            n_ = emptyIntList();
+            bitField0_ = (bitField0_ & ~0x00000080);
+            onChanged();
+            return this;
+          }
+
+          @java.lang.Override
+          public final Builder setUnknownFields(
+              final com.google.protobuf.UnknownFieldSet unknownFields) {
+            return super.setUnknownFields(unknownFields);
+          }
+
+          @java.lang.Override
+          public final Builder mergeUnknownFields(
+              final com.google.protobuf.UnknownFieldSet unknownFields) {
+            return super.mergeUnknownFields(unknownFields);
+          }
+
+          // @@protoc_insertion_point(builder_scope:datadog.agentpayload.SketchPayload.Sketch.Dogsketch)
+        }
+
+        // @@protoc_insertion_point(class_scope:datadog.agentpayload.SketchPayload.Sketch.Dogsketch)
+        private static final datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+            DEFAULT_INSTANCE;
+
+        static {
+          DEFAULT_INSTANCE = new datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch();
+        }
+
+        public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+            getDefaultInstance() {
+          return DEFAULT_INSTANCE;
+        }
+
+        private static final com.google.protobuf.Parser PARSER =
+            new com.google.protobuf.AbstractParser() {
+              @java.lang.Override
+              public Dogsketch parsePartialFrom(
+                  com.google.protobuf.CodedInputStream input,
+                  com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                  throws com.google.protobuf.InvalidProtocolBufferException {
+                Builder builder = newBuilder();
+                try {
+                  builder.mergeFrom(input, extensionRegistry);
+                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                  throw e.setUnfinishedMessage(builder.buildPartial());
+                } catch (com.google.protobuf.UninitializedMessageException e) {
+                  throw e.asInvalidProtocolBufferException()
+                      .setUnfinishedMessage(builder.buildPartial());
+                } catch (java.io.IOException e) {
+                  throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                      .setUnfinishedMessage(builder.buildPartial());
+                }
+                return builder.buildPartial();
+              }
+            };
+
+        public static com.google.protobuf.Parser parser() {
+          return PARSER;
+        }
+
+        @java.lang.Override
+        public com.google.protobuf.Parser getParserForType() {
+          return PARSER;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+            getDefaultInstanceForType() {
+          return DEFAULT_INSTANCE;
+        }
+      }
+
+      public static final int METRIC_FIELD_NUMBER = 1;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object metric_ = "";
+      /**
+       * string metric = 1;
+       *
+       * @return The metric.
+       */
+      @java.lang.Override
+      public java.lang.String getMetric() {
+        java.lang.Object ref = metric_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          metric_ = s;
+          return s;
+        }
+      }
+      /**
+       * string metric = 1;
+       *
+       * @return The bytes for metric.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getMetricBytes() {
+        java.lang.Object ref = metric_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          metric_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int HOST_FIELD_NUMBER = 2;
+
+      @SuppressWarnings("serial")
+      private volatile java.lang.Object host_ = "";
+      /**
+       * string host = 2;
+       *
+       * @return The host.
+       */
+      @java.lang.Override
+      public java.lang.String getHost() {
+        java.lang.Object ref = host_;
+        if (ref instanceof java.lang.String) {
+          return (java.lang.String) ref;
+        } else {
+          com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          host_ = s;
+          return s;
+        }
+      }
+      /**
+       * string host = 2;
+       *
+       * @return The bytes for host.
+       */
+      @java.lang.Override
+      public com.google.protobuf.ByteString getHostBytes() {
+        java.lang.Object ref = host_;
+        if (ref instanceof java.lang.String) {
+          com.google.protobuf.ByteString b =
+              com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+          host_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+
+      public static final int DISTRIBUTIONS_FIELD_NUMBER = 3;
+
+      @SuppressWarnings("serial")
+      private java.util.List
+          distributions_;
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+       * 
+       */
+      @java.lang.Override
+      public java.util.List
+          getDistributionsList() {
+        return distributions_;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+       * 
+       */
+      @java.lang.Override
+      public java.util.List<
+              ? extends
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DistributionOrBuilder>
+          getDistributionsOrBuilderList() {
+        return distributions_;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+       * 
+       */
+      @java.lang.Override
+      public int getDistributionsCount() {
+        return distributions_.size();
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+       * 
+       */
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution getDistributions(
+          int index) {
+        return distributions_.get(index);
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+       * 
+       */
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DistributionOrBuilder
+          getDistributionsOrBuilder(int index) {
+        return distributions_.get(index);
+      }
+
+      public static final int TAGS_FIELD_NUMBER = 4;
+
+      @SuppressWarnings("serial")
+      private com.google.protobuf.LazyStringArrayList tags_ =
+          com.google.protobuf.LazyStringArrayList.emptyList();
+      /**
+       * repeated string tags = 4;
+       *
+       * @return A list containing the tags.
+       */
+      public com.google.protobuf.ProtocolStringList getTagsList() {
+        return tags_;
+      }
+      /**
+       * repeated string tags = 4;
+       *
+       * @return The count of tags.
+       */
+      public int getTagsCount() {
+        return tags_.size();
+      }
+      /**
+       * repeated string tags = 4;
+       *
+       * @param index The index of the element to return.
+       * @return The tags at the given index.
+       */
+      public java.lang.String getTags(int index) {
+        return tags_.get(index);
+      }
+      /**
+       * repeated string tags = 4;
+       *
+       * @param index The index of the value to return.
+       * @return The bytes of the tags at the given index.
+       */
+      public com.google.protobuf.ByteString getTagsBytes(int index) {
+        return tags_.getByteString(index);
+      }
+
+      public static final int DOGSKETCHES_FIELD_NUMBER = 7;
+
+      @SuppressWarnings("serial")
+      private java.util.List
+          dogsketches_;
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+       * 
+       */
+      @java.lang.Override
+      public java.util.List
+          getDogsketchesList() {
+        return dogsketches_;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+       * 
+       */
+      @java.lang.Override
+      public java.util.List<
+              ? extends datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DogsketchOrBuilder>
+          getDogsketchesOrBuilderList() {
+        return dogsketches_;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+       * 
+       */
+      @java.lang.Override
+      public int getDogsketchesCount() {
+        return dogsketches_.size();
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+       * 
+       */
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch getDogsketches(
+          int index) {
+        return dogsketches_.get(index);
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+       * 
+       */
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DogsketchOrBuilder
+          getDogsketchesOrBuilder(int index) {
+        return dogsketches_.get(index);
+      }
+
+      private byte memoizedIsInitialized = -1;
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        byte isInitialized = memoizedIsInitialized;
+        if (isInitialized == 1) return true;
+        if (isInitialized == 0) return false;
+
+        memoizedIsInitialized = 1;
+        return true;
+      }
+
+      @java.lang.Override
+      public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(metric_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 1, metric_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(host_)) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 2, host_);
+        }
+        for (int i = 0; i < distributions_.size(); i++) {
+          output.writeMessage(3, distributions_.get(i));
+        }
+        for (int i = 0; i < tags_.size(); i++) {
+          com.google.protobuf.GeneratedMessageV3.writeString(output, 4, tags_.getRaw(i));
+        }
+        for (int i = 0; i < dogsketches_.size(); i++) {
+          output.writeMessage(7, dogsketches_.get(i));
+        }
+        getUnknownFields().writeTo(output);
+      }
+
+      @java.lang.Override
+      public int getSerializedSize() {
+        int size = memoizedSize;
+        if (size != -1) return size;
+
+        size = 0;
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(metric_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, metric_);
+        }
+        if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(host_)) {
+          size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, host_);
+        }
+        for (int i = 0; i < distributions_.size(); i++) {
+          size +=
+              com.google.protobuf.CodedOutputStream.computeMessageSize(3, distributions_.get(i));
+        }
+        {
+          int dataSize = 0;
+          for (int i = 0; i < tags_.size(); i++) {
+            dataSize += computeStringSizeNoTag(tags_.getRaw(i));
+          }
+          size += dataSize;
+          size += 1 * getTagsList().size();
+        }
+        for (int i = 0; i < dogsketches_.size(); i++) {
+          size += com.google.protobuf.CodedOutputStream.computeMessageSize(7, dogsketches_.get(i));
+        }
+        size += getUnknownFields().getSerializedSize();
+        memoizedSize = size;
+        return size;
+      }
+
+      @java.lang.Override
+      public boolean equals(final java.lang.Object obj) {
+        if (obj == this) {
+          return true;
+        }
+        if (!(obj instanceof datadog.agentpayload.AgentPayload.SketchPayload.Sketch)) {
+          return super.equals(obj);
+        }
+        datadog.agentpayload.AgentPayload.SketchPayload.Sketch other =
+            (datadog.agentpayload.AgentPayload.SketchPayload.Sketch) obj;
+
+        if (!getMetric().equals(other.getMetric())) return false;
+        if (!getHost().equals(other.getHost())) return false;
+        if (!getDistributionsList().equals(other.getDistributionsList())) return false;
+        if (!getTagsList().equals(other.getTagsList())) return false;
+        if (!getDogsketchesList().equals(other.getDogsketchesList())) return false;
+        if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+        return true;
+      }
+
+      @java.lang.Override
+      public int hashCode() {
+        if (memoizedHashCode != 0) {
+          return memoizedHashCode;
+        }
+        int hash = 41;
+        hash = (19 * hash) + getDescriptor().hashCode();
+        hash = (37 * hash) + METRIC_FIELD_NUMBER;
+        hash = (53 * hash) + getMetric().hashCode();
+        hash = (37 * hash) + HOST_FIELD_NUMBER;
+        hash = (53 * hash) + getHost().hashCode();
+        if (getDistributionsCount() > 0) {
+          hash = (37 * hash) + DISTRIBUTIONS_FIELD_NUMBER;
+          hash = (53 * hash) + getDistributionsList().hashCode();
+        }
+        if (getTagsCount() > 0) {
+          hash = (37 * hash) + TAGS_FIELD_NUMBER;
+          hash = (53 * hash) + getTagsList().hashCode();
+        }
+        if (getDogsketchesCount() > 0) {
+          hash = (37 * hash) + DOGSKETCHES_FIELD_NUMBER;
+          hash = (53 * hash) + getDogsketchesList().hashCode();
+        }
+        hash = (29 * hash) + getUnknownFields().hashCode();
+        memoizedHashCode = hash;
+        return hash;
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseFrom(
+          java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseFrom(
+          java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseFrom(
+          com.google.protobuf.ByteString data)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseFrom(
+          com.google.protobuf.ByteString data,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseFrom(byte[] data)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseFrom(
+          byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return PARSER.parseFrom(data, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseFrom(
+          java.io.InputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseFrom(
+          java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseDelimitedFrom(
+          java.io.InputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseDelimitedFrom(
+          java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseFrom(
+          com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch parseFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+            PARSER, input, extensionRegistry);
+      }
+
+      @java.lang.Override
+      public Builder newBuilderForType() {
+        return newBuilder();
+      }
+
+      public static Builder newBuilder() {
+        return DEFAULT_INSTANCE.toBuilder();
+      }
+
+      public static Builder newBuilder(
+          datadog.agentpayload.AgentPayload.SketchPayload.Sketch prototype) {
+        return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+      }
+
+      @java.lang.Override
+      public Builder toBuilder() {
+        return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+      }
+
+      @java.lang.Override
+      protected Builder newBuilderForType(
+          com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        Builder builder = new Builder(parent);
+        return builder;
+      }
+      /** Protobuf type {@code datadog.agentpayload.SketchPayload.Sketch} */
+      public static final class Builder
+          extends com.google.protobuf.GeneratedMessageV3.Builder
+          implements
+          // @@protoc_insertion_point(builder_implements:datadog.agentpayload.SketchPayload.Sketch)
+          datadog.agentpayload.AgentPayload.SketchPayload.SketchOrBuilder {
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_SketchPayload_Sketch_descriptor;
+        }
+
+        @java.lang.Override
+        protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+            internalGetFieldAccessorTable() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_SketchPayload_Sketch_fieldAccessorTable
+              .ensureFieldAccessorsInitialized(
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.class,
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder.class);
+        }
+
+        // Construct using datadog.agentpayload.AgentPayload.SketchPayload.Sketch.newBuilder()
+        private Builder() {}
+
+        private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+          super(parent);
+        }
+
+        @java.lang.Override
+        public Builder clear() {
+          super.clear();
+          bitField0_ = 0;
+          metric_ = "";
+          host_ = "";
+          if (distributionsBuilder_ == null) {
+            distributions_ = java.util.Collections.emptyList();
+          } else {
+            distributions_ = null;
+            distributionsBuilder_.clear();
+          }
+          bitField0_ = (bitField0_ & ~0x00000004);
+          tags_ = com.google.protobuf.LazyStringArrayList.emptyList();
+          if (dogsketchesBuilder_ == null) {
+            dogsketches_ = java.util.Collections.emptyList();
+          } else {
+            dogsketches_ = null;
+            dogsketchesBuilder_.clear();
+          }
+          bitField0_ = (bitField0_ & ~0x00000010);
+          return this;
+        }
+
+        @java.lang.Override
+        public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+          return datadog.agentpayload.AgentPayload
+              .internal_static_datadog_agentpayload_SketchPayload_Sketch_descriptor;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch getDefaultInstanceForType() {
+          return datadog.agentpayload.AgentPayload.SketchPayload.Sketch.getDefaultInstance();
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch build() {
+          datadog.agentpayload.AgentPayload.SketchPayload.Sketch result = buildPartial();
+          if (!result.isInitialized()) {
+            throw newUninitializedMessageException(result);
+          }
+          return result;
+        }
+
+        @java.lang.Override
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch buildPartial() {
+          datadog.agentpayload.AgentPayload.SketchPayload.Sketch result =
+              new datadog.agentpayload.AgentPayload.SketchPayload.Sketch(this);
+          buildPartialRepeatedFields(result);
+          if (bitField0_ != 0) {
+            buildPartial0(result);
+          }
+          onBuilt();
+          return result;
+        }
+
+        private void buildPartialRepeatedFields(
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch result) {
+          if (distributionsBuilder_ == null) {
+            if (((bitField0_ & 0x00000004) != 0)) {
+              distributions_ = java.util.Collections.unmodifiableList(distributions_);
+              bitField0_ = (bitField0_ & ~0x00000004);
+            }
+            result.distributions_ = distributions_;
+          } else {
+            result.distributions_ = distributionsBuilder_.build();
+          }
+          if (dogsketchesBuilder_ == null) {
+            if (((bitField0_ & 0x00000010) != 0)) {
+              dogsketches_ = java.util.Collections.unmodifiableList(dogsketches_);
+              bitField0_ = (bitField0_ & ~0x00000010);
+            }
+            result.dogsketches_ = dogsketches_;
+          } else {
+            result.dogsketches_ = dogsketchesBuilder_.build();
+          }
+        }
+
+        private void buildPartial0(datadog.agentpayload.AgentPayload.SketchPayload.Sketch result) {
+          int from_bitField0_ = bitField0_;
+          if (((from_bitField0_ & 0x00000001) != 0)) {
+            result.metric_ = metric_;
+          }
+          if (((from_bitField0_ & 0x00000002) != 0)) {
+            result.host_ = host_;
+          }
+          if (((from_bitField0_ & 0x00000008) != 0)) {
+            tags_.makeImmutable();
+            result.tags_ = tags_;
+          }
+        }
+
+        @java.lang.Override
+        public Builder clone() {
+          return super.clone();
+        }
+
+        @java.lang.Override
+        public Builder setField(
+            com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+          return super.setField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+          return super.clearField(field);
+        }
+
+        @java.lang.Override
+        public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+          return super.clearOneof(oneof);
+        }
+
+        @java.lang.Override
+        public Builder setRepeatedField(
+            com.google.protobuf.Descriptors.FieldDescriptor field,
+            int index,
+            java.lang.Object value) {
+          return super.setRepeatedField(field, index, value);
+        }
+
+        @java.lang.Override
+        public Builder addRepeatedField(
+            com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+          return super.addRepeatedField(field, value);
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(com.google.protobuf.Message other) {
+          if (other instanceof datadog.agentpayload.AgentPayload.SketchPayload.Sketch) {
+            return mergeFrom((datadog.agentpayload.AgentPayload.SketchPayload.Sketch) other);
+          } else {
+            super.mergeFrom(other);
+            return this;
+          }
+        }
+
+        public Builder mergeFrom(datadog.agentpayload.AgentPayload.SketchPayload.Sketch other) {
+          if (other == datadog.agentpayload.AgentPayload.SketchPayload.Sketch.getDefaultInstance())
+            return this;
+          if (!other.getMetric().isEmpty()) {
+            metric_ = other.metric_;
+            bitField0_ |= 0x00000001;
+            onChanged();
+          }
+          if (!other.getHost().isEmpty()) {
+            host_ = other.host_;
+            bitField0_ |= 0x00000002;
+            onChanged();
+          }
+          if (distributionsBuilder_ == null) {
+            if (!other.distributions_.isEmpty()) {
+              if (distributions_.isEmpty()) {
+                distributions_ = other.distributions_;
+                bitField0_ = (bitField0_ & ~0x00000004);
+              } else {
+                ensureDistributionsIsMutable();
+                distributions_.addAll(other.distributions_);
+              }
+              onChanged();
+            }
+          } else {
+            if (!other.distributions_.isEmpty()) {
+              if (distributionsBuilder_.isEmpty()) {
+                distributionsBuilder_.dispose();
+                distributionsBuilder_ = null;
+                distributions_ = other.distributions_;
+                bitField0_ = (bitField0_ & ~0x00000004);
+                distributionsBuilder_ =
+                    com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders
+                        ? getDistributionsFieldBuilder()
+                        : null;
+              } else {
+                distributionsBuilder_.addAllMessages(other.distributions_);
+              }
+            }
+          }
+          if (!other.tags_.isEmpty()) {
+            if (tags_.isEmpty()) {
+              tags_ = other.tags_;
+              bitField0_ |= 0x00000008;
+            } else {
+              ensureTagsIsMutable();
+              tags_.addAll(other.tags_);
+            }
+            onChanged();
+          }
+          if (dogsketchesBuilder_ == null) {
+            if (!other.dogsketches_.isEmpty()) {
+              if (dogsketches_.isEmpty()) {
+                dogsketches_ = other.dogsketches_;
+                bitField0_ = (bitField0_ & ~0x00000010);
+              } else {
+                ensureDogsketchesIsMutable();
+                dogsketches_.addAll(other.dogsketches_);
+              }
+              onChanged();
+            }
+          } else {
+            if (!other.dogsketches_.isEmpty()) {
+              if (dogsketchesBuilder_.isEmpty()) {
+                dogsketchesBuilder_.dispose();
+                dogsketchesBuilder_ = null;
+                dogsketches_ = other.dogsketches_;
+                bitField0_ = (bitField0_ & ~0x00000010);
+                dogsketchesBuilder_ =
+                    com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders
+                        ? getDogsketchesFieldBuilder()
+                        : null;
+              } else {
+                dogsketchesBuilder_.addAllMessages(other.dogsketches_);
+              }
+            }
+          }
+          this.mergeUnknownFields(other.getUnknownFields());
+          onChanged();
+          return this;
+        }
+
+        @java.lang.Override
+        public final boolean isInitialized() {
+          return true;
+        }
+
+        @java.lang.Override
+        public Builder mergeFrom(
+            com.google.protobuf.CodedInputStream input,
+            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+            throws java.io.IOException {
+          if (extensionRegistry == null) {
+            throw new java.lang.NullPointerException();
+          }
+          try {
+            boolean done = false;
+            while (!done) {
+              int tag = input.readTag();
+              switch (tag) {
+                case 0:
+                  done = true;
+                  break;
+                case 10:
+                  {
+                    metric_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000001;
+                    break;
+                  } // case 10
+                case 18:
+                  {
+                    host_ = input.readStringRequireUtf8();
+                    bitField0_ |= 0x00000002;
+                    break;
+                  } // case 18
+                case 26:
+                  {
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution m =
+                        input.readMessage(
+                            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+                                .parser(),
+                            extensionRegistry);
+                    if (distributionsBuilder_ == null) {
+                      ensureDistributionsIsMutable();
+                      distributions_.add(m);
+                    } else {
+                      distributionsBuilder_.addMessage(m);
+                    }
+                    break;
+                  } // case 26
+                case 34:
+                  {
+                    java.lang.String s = input.readStringRequireUtf8();
+                    ensureTagsIsMutable();
+                    tags_.add(s);
+                    break;
+                  } // case 34
+                case 58:
+                  {
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch m =
+                        input.readMessage(
+                            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+                                .parser(),
+                            extensionRegistry);
+                    if (dogsketchesBuilder_ == null) {
+                      ensureDogsketchesIsMutable();
+                      dogsketches_.add(m);
+                    } else {
+                      dogsketchesBuilder_.addMessage(m);
+                    }
+                    break;
+                  } // case 58
+                default:
+                  {
+                    if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                      done = true; // was an endgroup tag
+                    }
+                    break;
+                  } // default:
+              } // switch (tag)
+            } // while (!done)
+          } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+            throw e.unwrapIOException();
+          } finally {
+            onChanged();
+          } // finally
+          return this;
+        }
+
+        private int bitField0_;
+
+        private java.lang.Object metric_ = "";
+        /**
+         * string metric = 1;
+         *
+         * @return The metric.
+         */
+        public java.lang.String getMetric() {
+          java.lang.Object ref = metric_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            metric_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string metric = 1;
+         *
+         * @return The bytes for metric.
+         */
+        public com.google.protobuf.ByteString getMetricBytes() {
+          java.lang.Object ref = metric_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            metric_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string metric = 1;
+         *
+         * @param value The metric to set.
+         * @return This builder for chaining.
+         */
+        public Builder setMetric(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          metric_ = value;
+          bitField0_ |= 0x00000001;
+          onChanged();
+          return this;
+        }
+        /**
+         * string metric = 1;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearMetric() {
+          metric_ = getDefaultInstance().getMetric();
+          bitField0_ = (bitField0_ & ~0x00000001);
+          onChanged();
+          return this;
+        }
+        /**
+         * string metric = 1;
+         *
+         * @param value The bytes for metric to set.
+         * @return This builder for chaining.
+         */
+        public Builder setMetricBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          metric_ = value;
+          bitField0_ |= 0x00000001;
+          onChanged();
+          return this;
+        }
+
+        private java.lang.Object host_ = "";
+        /**
+         * string host = 2;
+         *
+         * @return The host.
+         */
+        public java.lang.String getHost() {
+          java.lang.Object ref = host_;
+          if (!(ref instanceof java.lang.String)) {
+            com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+            java.lang.String s = bs.toStringUtf8();
+            host_ = s;
+            return s;
+          } else {
+            return (java.lang.String) ref;
+          }
+        }
+        /**
+         * string host = 2;
+         *
+         * @return The bytes for host.
+         */
+        public com.google.protobuf.ByteString getHostBytes() {
+          java.lang.Object ref = host_;
+          if (ref instanceof String) {
+            com.google.protobuf.ByteString b =
+                com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref);
+            host_ = b;
+            return b;
+          } else {
+            return (com.google.protobuf.ByteString) ref;
+          }
+        }
+        /**
+         * string host = 2;
+         *
+         * @param value The host to set.
+         * @return This builder for chaining.
+         */
+        public Builder setHost(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          host_ = value;
+          bitField0_ |= 0x00000002;
+          onChanged();
+          return this;
+        }
+        /**
+         * string host = 2;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearHost() {
+          host_ = getDefaultInstance().getHost();
+          bitField0_ = (bitField0_ & ~0x00000002);
+          onChanged();
+          return this;
+        }
+        /**
+         * string host = 2;
+         *
+         * @param value The bytes for host to set.
+         * @return This builder for chaining.
+         */
+        public Builder setHostBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          host_ = value;
+          bitField0_ |= 0x00000002;
+          onChanged();
+          return this;
+        }
+
+        private java.util.List
+            distributions_ = java.util.Collections.emptyList();
+
+        private void ensureDistributionsIsMutable() {
+          if (!((bitField0_ & 0x00000004) != 0)) {
+            distributions_ =
+                new java.util.ArrayList<
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution>(
+                    distributions_);
+            bitField0_ |= 0x00000004;
+          }
+        }
+
+        private com.google.protobuf.RepeatedFieldBuilderV3<
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution,
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder,
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DistributionOrBuilder>
+            distributionsBuilder_;
+
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public java.util.List
+            getDistributionsList() {
+          if (distributionsBuilder_ == null) {
+            return java.util.Collections.unmodifiableList(distributions_);
+          } else {
+            return distributionsBuilder_.getMessageList();
+          }
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public int getDistributionsCount() {
+          if (distributionsBuilder_ == null) {
+            return distributions_.size();
+          } else {
+            return distributionsBuilder_.getCount();
+          }
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution getDistributions(
+            int index) {
+          if (distributionsBuilder_ == null) {
+            return distributions_.get(index);
+          } else {
+            return distributionsBuilder_.getMessage(index);
+          }
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder setDistributions(
+            int index, datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution value) {
+          if (distributionsBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensureDistributionsIsMutable();
+            distributions_.set(index, value);
+            onChanged();
+          } else {
+            distributionsBuilder_.setMessage(index, value);
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder setDistributions(
+            int index,
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder
+                builderForValue) {
+          if (distributionsBuilder_ == null) {
+            ensureDistributionsIsMutable();
+            distributions_.set(index, builderForValue.build());
+            onChanged();
+          } else {
+            distributionsBuilder_.setMessage(index, builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder addDistributions(
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution value) {
+          if (distributionsBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensureDistributionsIsMutable();
+            distributions_.add(value);
+            onChanged();
+          } else {
+            distributionsBuilder_.addMessage(value);
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder addDistributions(
+            int index, datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution value) {
+          if (distributionsBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensureDistributionsIsMutable();
+            distributions_.add(index, value);
+            onChanged();
+          } else {
+            distributionsBuilder_.addMessage(index, value);
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder addDistributions(
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder
+                builderForValue) {
+          if (distributionsBuilder_ == null) {
+            ensureDistributionsIsMutable();
+            distributions_.add(builderForValue.build());
+            onChanged();
+          } else {
+            distributionsBuilder_.addMessage(builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder addDistributions(
+            int index,
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder
+                builderForValue) {
+          if (distributionsBuilder_ == null) {
+            ensureDistributionsIsMutable();
+            distributions_.add(index, builderForValue.build());
+            onChanged();
+          } else {
+            distributionsBuilder_.addMessage(index, builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder addAllDistributions(
+            java.lang.Iterable<
+                    ? extends datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution>
+                values) {
+          if (distributionsBuilder_ == null) {
+            ensureDistributionsIsMutable();
+            com.google.protobuf.AbstractMessageLite.Builder.addAll(values, distributions_);
+            onChanged();
+          } else {
+            distributionsBuilder_.addAllMessages(values);
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder clearDistributions() {
+          if (distributionsBuilder_ == null) {
+            distributions_ = java.util.Collections.emptyList();
+            bitField0_ = (bitField0_ & ~0x00000004);
+            onChanged();
+          } else {
+            distributionsBuilder_.clear();
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder removeDistributions(int index) {
+          if (distributionsBuilder_ == null) {
+            ensureDistributionsIsMutable();
+            distributions_.remove(index);
+            onChanged();
+          } else {
+            distributionsBuilder_.remove(index);
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder
+            getDistributionsBuilder(int index) {
+          return getDistributionsFieldBuilder().getBuilder(index);
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DistributionOrBuilder
+            getDistributionsOrBuilder(int index) {
+          if (distributionsBuilder_ == null) {
+            return distributions_.get(index);
+          } else {
+            return distributionsBuilder_.getMessageOrBuilder(index);
+          }
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public java.util.List<
+                ? extends
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DistributionOrBuilder>
+            getDistributionsOrBuilderList() {
+          if (distributionsBuilder_ != null) {
+            return distributionsBuilder_.getMessageOrBuilderList();
+          } else {
+            return java.util.Collections.unmodifiableList(distributions_);
+          }
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder
+            addDistributionsBuilder() {
+          return getDistributionsFieldBuilder()
+              .addBuilder(
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+                      .getDefaultInstance());
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder
+            addDistributionsBuilder(int index) {
+          return getDistributionsFieldBuilder()
+              .addBuilder(
+                  index,
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution
+                      .getDefaultInstance());
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Distribution distributions = 3 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public java.util.List<
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder>
+            getDistributionsBuilderList() {
+          return getDistributionsFieldBuilder().getBuilderList();
+        }
+
+        private com.google.protobuf.RepeatedFieldBuilderV3<
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution,
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder,
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DistributionOrBuilder>
+            getDistributionsFieldBuilder() {
+          if (distributionsBuilder_ == null) {
+            distributionsBuilder_ =
+                new com.google.protobuf.RepeatedFieldBuilderV3<
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution,
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Distribution.Builder,
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DistributionOrBuilder>(
+                    distributions_,
+                    ((bitField0_ & 0x00000004) != 0),
+                    getParentForChildren(),
+                    isClean());
+            distributions_ = null;
+          }
+          return distributionsBuilder_;
+        }
+
+        private com.google.protobuf.LazyStringArrayList tags_ =
+            com.google.protobuf.LazyStringArrayList.emptyList();
+
+        private void ensureTagsIsMutable() {
+          if (!tags_.isModifiable()) {
+            tags_ = new com.google.protobuf.LazyStringArrayList(tags_);
+          }
+          bitField0_ |= 0x00000008;
+        }
+        /**
+         * repeated string tags = 4;
+         *
+         * @return A list containing the tags.
+         */
+        public com.google.protobuf.ProtocolStringList getTagsList() {
+          tags_.makeImmutable();
+          return tags_;
+        }
+        /**
+         * repeated string tags = 4;
+         *
+         * @return The count of tags.
+         */
+        public int getTagsCount() {
+          return tags_.size();
+        }
+        /**
+         * repeated string tags = 4;
+         *
+         * @param index The index of the element to return.
+         * @return The tags at the given index.
+         */
+        public java.lang.String getTags(int index) {
+          return tags_.get(index);
+        }
+        /**
+         * repeated string tags = 4;
+         *
+         * @param index The index of the value to return.
+         * @return The bytes of the tags at the given index.
+         */
+        public com.google.protobuf.ByteString getTagsBytes(int index) {
+          return tags_.getByteString(index);
+        }
+        /**
+         * repeated string tags = 4;
+         *
+         * @param index The index to set the value at.
+         * @param value The tags to set.
+         * @return This builder for chaining.
+         */
+        public Builder setTags(int index, java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTagsIsMutable();
+          tags_.set(index, value);
+          bitField0_ |= 0x00000008;
+          onChanged();
+          return this;
+        }
+        /**
+         * repeated string tags = 4;
+         *
+         * @param value The tags to add.
+         * @return This builder for chaining.
+         */
+        public Builder addTags(java.lang.String value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureTagsIsMutable();
+          tags_.add(value);
+          bitField0_ |= 0x00000008;
+          onChanged();
+          return this;
+        }
+        /**
+         * repeated string tags = 4;
+         *
+         * @param values The tags to add.
+         * @return This builder for chaining.
+         */
+        public Builder addAllTags(java.lang.Iterable values) {
+          ensureTagsIsMutable();
+          com.google.protobuf.AbstractMessageLite.Builder.addAll(values, tags_);
+          bitField0_ |= 0x00000008;
+          onChanged();
+          return this;
+        }
+        /**
+         * repeated string tags = 4;
+         *
+         * @return This builder for chaining.
+         */
+        public Builder clearTags() {
+          tags_ = com.google.protobuf.LazyStringArrayList.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000008);
+          ;
+          onChanged();
+          return this;
+        }
+        /**
+         * repeated string tags = 4;
+         *
+         * @param value The bytes of the tags to add.
+         * @return This builder for chaining.
+         */
+        public Builder addTagsBytes(com.google.protobuf.ByteString value) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          checkByteStringIsUtf8(value);
+          ensureTagsIsMutable();
+          tags_.add(value);
+          bitField0_ |= 0x00000008;
+          onChanged();
+          return this;
+        }
+
+        private java.util.List
+            dogsketches_ = java.util.Collections.emptyList();
+
+        private void ensureDogsketchesIsMutable() {
+          if (!((bitField0_ & 0x00000010) != 0)) {
+            dogsketches_ =
+                new java.util.ArrayList<
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch>(dogsketches_);
+            bitField0_ |= 0x00000010;
+          }
+        }
+
+        private com.google.protobuf.RepeatedFieldBuilderV3<
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch,
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder,
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DogsketchOrBuilder>
+            dogsketchesBuilder_;
+
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public java.util.List
+            getDogsketchesList() {
+          if (dogsketchesBuilder_ == null) {
+            return java.util.Collections.unmodifiableList(dogsketches_);
+          } else {
+            return dogsketchesBuilder_.getMessageList();
+          }
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public int getDogsketchesCount() {
+          if (dogsketchesBuilder_ == null) {
+            return dogsketches_.size();
+          } else {
+            return dogsketchesBuilder_.getCount();
+          }
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch getDogsketches(
+            int index) {
+          if (dogsketchesBuilder_ == null) {
+            return dogsketches_.get(index);
+          } else {
+            return dogsketchesBuilder_.getMessage(index);
+          }
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder setDogsketches(
+            int index, datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch value) {
+          if (dogsketchesBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensureDogsketchesIsMutable();
+            dogsketches_.set(index, value);
+            onChanged();
+          } else {
+            dogsketchesBuilder_.setMessage(index, value);
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder setDogsketches(
+            int index,
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder
+                builderForValue) {
+          if (dogsketchesBuilder_ == null) {
+            ensureDogsketchesIsMutable();
+            dogsketches_.set(index, builderForValue.build());
+            onChanged();
+          } else {
+            dogsketchesBuilder_.setMessage(index, builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder addDogsketches(
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch value) {
+          if (dogsketchesBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensureDogsketchesIsMutable();
+            dogsketches_.add(value);
+            onChanged();
+          } else {
+            dogsketchesBuilder_.addMessage(value);
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder addDogsketches(
+            int index, datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch value) {
+          if (dogsketchesBuilder_ == null) {
+            if (value == null) {
+              throw new NullPointerException();
+            }
+            ensureDogsketchesIsMutable();
+            dogsketches_.add(index, value);
+            onChanged();
+          } else {
+            dogsketchesBuilder_.addMessage(index, value);
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder addDogsketches(
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder
+                builderForValue) {
+          if (dogsketchesBuilder_ == null) {
+            ensureDogsketchesIsMutable();
+            dogsketches_.add(builderForValue.build());
+            onChanged();
+          } else {
+            dogsketchesBuilder_.addMessage(builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder addDogsketches(
+            int index,
+            datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder
+                builderForValue) {
+          if (dogsketchesBuilder_ == null) {
+            ensureDogsketchesIsMutable();
+            dogsketches_.add(index, builderForValue.build());
+            onChanged();
+          } else {
+            dogsketchesBuilder_.addMessage(index, builderForValue.build());
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder addAllDogsketches(
+            java.lang.Iterable<
+                    ? extends datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch>
+                values) {
+          if (dogsketchesBuilder_ == null) {
+            ensureDogsketchesIsMutable();
+            com.google.protobuf.AbstractMessageLite.Builder.addAll(values, dogsketches_);
+            onChanged();
+          } else {
+            dogsketchesBuilder_.addAllMessages(values);
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder clearDogsketches() {
+          if (dogsketchesBuilder_ == null) {
+            dogsketches_ = java.util.Collections.emptyList();
+            bitField0_ = (bitField0_ & ~0x00000010);
+            onChanged();
+          } else {
+            dogsketchesBuilder_.clear();
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public Builder removeDogsketches(int index) {
+          if (dogsketchesBuilder_ == null) {
+            ensureDogsketchesIsMutable();
+            dogsketches_.remove(index);
+            onChanged();
+          } else {
+            dogsketchesBuilder_.remove(index);
+          }
+          return this;
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder
+            getDogsketchesBuilder(int index) {
+          return getDogsketchesFieldBuilder().getBuilder(index);
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DogsketchOrBuilder
+            getDogsketchesOrBuilder(int index) {
+          if (dogsketchesBuilder_ == null) {
+            return dogsketches_.get(index);
+          } else {
+            return dogsketchesBuilder_.getMessageOrBuilder(index);
+          }
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public java.util.List<
+                ? extends datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DogsketchOrBuilder>
+            getDogsketchesOrBuilderList() {
+          if (dogsketchesBuilder_ != null) {
+            return dogsketchesBuilder_.getMessageOrBuilderList();
+          } else {
+            return java.util.Collections.unmodifiableList(dogsketches_);
+          }
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder
+            addDogsketchesBuilder() {
+          return getDogsketchesFieldBuilder()
+              .addBuilder(
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+                      .getDefaultInstance());
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder
+            addDogsketchesBuilder(int index) {
+          return getDogsketchesFieldBuilder()
+              .addBuilder(
+                  index,
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch
+                      .getDefaultInstance());
+        }
+        /**
+         * 
+         * repeated .datadog.agentpayload.SketchPayload.Sketch.Dogsketch dogsketches = 7 [(.gogoproto.nullable) = false];
+         * 
+         */
+        public java.util.List<
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder>
+            getDogsketchesBuilderList() {
+          return getDogsketchesFieldBuilder().getBuilderList();
+        }
+
+        private com.google.protobuf.RepeatedFieldBuilderV3<
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch,
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder,
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DogsketchOrBuilder>
+            getDogsketchesFieldBuilder() {
+          if (dogsketchesBuilder_ == null) {
+            dogsketchesBuilder_ =
+                new com.google.protobuf.RepeatedFieldBuilderV3<
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch,
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Dogsketch.Builder,
+                    datadog.agentpayload.AgentPayload.SketchPayload.Sketch.DogsketchOrBuilder>(
+                    dogsketches_,
+                    ((bitField0_ & 0x00000010) != 0),
+                    getParentForChildren(),
+                    isClean());
+            dogsketches_ = null;
+          }
+          return dogsketchesBuilder_;
+        }
+
+        @java.lang.Override
+        public final Builder setUnknownFields(
+            final com.google.protobuf.UnknownFieldSet unknownFields) {
+          return super.setUnknownFields(unknownFields);
+        }
+
+        @java.lang.Override
+        public final Builder mergeUnknownFields(
+            final com.google.protobuf.UnknownFieldSet unknownFields) {
+          return super.mergeUnknownFields(unknownFields);
+        }
+
+        // @@protoc_insertion_point(builder_scope:datadog.agentpayload.SketchPayload.Sketch)
+      }
+
+      // @@protoc_insertion_point(class_scope:datadog.agentpayload.SketchPayload.Sketch)
+      private static final datadog.agentpayload.AgentPayload.SketchPayload.Sketch DEFAULT_INSTANCE;
+
+      static {
+        DEFAULT_INSTANCE = new datadog.agentpayload.AgentPayload.SketchPayload.Sketch();
+      }
+
+      public static datadog.agentpayload.AgentPayload.SketchPayload.Sketch getDefaultInstance() {
+        return DEFAULT_INSTANCE;
+      }
+
+      private static final com.google.protobuf.Parser PARSER =
+          new com.google.protobuf.AbstractParser() {
+            @java.lang.Override
+            public Sketch parsePartialFrom(
+                com.google.protobuf.CodedInputStream input,
+                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                throws com.google.protobuf.InvalidProtocolBufferException {
+              Builder builder = newBuilder();
+              try {
+                builder.mergeFrom(input, extensionRegistry);
+              } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                throw e.setUnfinishedMessage(builder.buildPartial());
+              } catch (com.google.protobuf.UninitializedMessageException e) {
+                throw e.asInvalidProtocolBufferException()
+                    .setUnfinishedMessage(builder.buildPartial());
+              } catch (java.io.IOException e) {
+                throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                    .setUnfinishedMessage(builder.buildPartial());
+              }
+              return builder.buildPartial();
+            }
+          };
+
+      public static com.google.protobuf.Parser parser() {
+        return PARSER;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Parser getParserForType() {
+        return PARSER;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.SketchPayload.Sketch getDefaultInstanceForType() {
+        return DEFAULT_INSTANCE;
+      }
+    }
+
+    private int bitField0_;
+    public static final int SKETCHES_FIELD_NUMBER = 1;
+
+    @SuppressWarnings("serial")
+    private java.util.List sketches_;
+    /**
+     * 
+     * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+     * 
+     */
+    @java.lang.Override
+    public java.util.List
+        getSketchesList() {
+      return sketches_;
+    }
+    /**
+     * 
+     * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+     * 
+     */
+    @java.lang.Override
+    public java.util.List
+        getSketchesOrBuilderList() {
+      return sketches_;
+    }
+    /**
+     * 
+     * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+     * 
+     */
+    @java.lang.Override
+    public int getSketchesCount() {
+      return sketches_.size();
+    }
+    /**
+     * 
+     * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+     * 
+     */
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.SketchPayload.Sketch getSketches(int index) {
+      return sketches_.get(index);
+    }
+    /**
+     * 
+     * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+     * 
+     */
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.SketchPayload.SketchOrBuilder getSketchesOrBuilder(
+        int index) {
+      return sketches_.get(index);
+    }
+
+    public static final int METADATA_FIELD_NUMBER = 2;
+    private datadog.agentpayload.AgentPayload.CommonMetadata metadata_;
+    /**
+     * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+     * 
+     *
+     * @return Whether the metadata field is set.
+     */
+    @java.lang.Override
+    public boolean hasMetadata() {
+      return ((bitField0_ & 0x00000001) != 0);
+    }
+    /**
+     * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+     * 
+     *
+     * @return The metadata.
+     */
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.CommonMetadata getMetadata() {
+      return metadata_ == null
+          ? datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance()
+          : metadata_;
+    }
+    /**
+     * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+     * 
+     */
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder getMetadataOrBuilder() {
+      return metadata_ == null
+          ? datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance()
+          : metadata_;
+    }
+
+    private byte memoizedIsInitialized = -1;
+
+    @java.lang.Override
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized == 1) return true;
+      if (isInitialized == 0) return false;
+
+      memoizedIsInitialized = 1;
+      return true;
+    }
+
+    @java.lang.Override
+    public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+      for (int i = 0; i < sketches_.size(); i++) {
+        output.writeMessage(1, sketches_.get(i));
+      }
+      if (((bitField0_ & 0x00000001) != 0)) {
+        output.writeMessage(2, getMetadata());
+      }
+      getUnknownFields().writeTo(output);
+    }
+
+    @java.lang.Override
+    public int getSerializedSize() {
+      int size = memoizedSize;
+      if (size != -1) return size;
+
+      size = 0;
+      for (int i = 0; i < sketches_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream.computeMessageSize(1, sketches_.get(i));
+      }
+      if (((bitField0_ & 0x00000001) != 0)) {
+        size += com.google.protobuf.CodedOutputStream.computeMessageSize(2, getMetadata());
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSize = size;
+      return size;
+    }
+
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof datadog.agentpayload.AgentPayload.SketchPayload)) {
+        return super.equals(obj);
+      }
+      datadog.agentpayload.AgentPayload.SketchPayload other =
+          (datadog.agentpayload.AgentPayload.SketchPayload) obj;
+
+      if (!getSketchesList().equals(other.getSketchesList())) return false;
+      if (hasMetadata() != other.hasMetadata()) return false;
+      if (hasMetadata()) {
+        if (!getMetadata().equals(other.getMetadata())) return false;
+      }
+      if (!getUnknownFields().equals(other.getUnknownFields())) return false;
+      return true;
+    }
+
+    @java.lang.Override
+    public int hashCode() {
+      if (memoizedHashCode != 0) {
+        return memoizedHashCode;
+      }
+      int hash = 41;
+      hash = (19 * hash) + getDescriptor().hashCode();
+      if (getSketchesCount() > 0) {
+        hash = (37 * hash) + SKETCHES_FIELD_NUMBER;
+        hash = (53 * hash) + getSketchesList().hashCode();
+      }
+      if (hasMetadata()) {
+        hash = (37 * hash) + METADATA_FIELD_NUMBER;
+        hash = (53 * hash) + getMetadata().hashCode();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      memoizedHashCode = hash;
+      return hash;
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseFrom(
+        java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseFrom(
+        java.nio.ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseFrom(
+        byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseFrom(
+        java.io.InputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseFrom(
+        java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseDelimitedFrom(
+        java.io.InputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseDelimitedFrom(
+        java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseFrom(
+        com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return com.google.protobuf.GeneratedMessageV3.parseWithIOException(
+          PARSER, input, extensionRegistry);
+    }
+
+    @java.lang.Override
+    public Builder newBuilderForType() {
+      return newBuilder();
+    }
+
+    public static Builder newBuilder() {
+      return DEFAULT_INSTANCE.toBuilder();
+    }
+
+    public static Builder newBuilder(datadog.agentpayload.AgentPayload.SketchPayload prototype) {
+      return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+    }
+
+    @java.lang.Override
+    public Builder toBuilder() {
+      return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+    }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    /** Protobuf type {@code datadog.agentpayload.SketchPayload} */
+    public static final class Builder
+        extends com.google.protobuf.GeneratedMessageV3.Builder
+        implements
+        // @@protoc_insertion_point(builder_implements:datadog.agentpayload.SketchPayload)
+        datadog.agentpayload.AgentPayload.SketchPayloadOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_SketchPayload_descriptor;
+      }
+
+      @java.lang.Override
+      protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_SketchPayload_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                datadog.agentpayload.AgentPayload.SketchPayload.class,
+                datadog.agentpayload.AgentPayload.SketchPayload.Builder.class);
+      }
+
+      // Construct using datadog.agentpayload.AgentPayload.SketchPayload.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+
+      private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) {
+          getSketchesFieldBuilder();
+          getMetadataFieldBuilder();
+        }
+      }
+
+      @java.lang.Override
+      public Builder clear() {
+        super.clear();
+        bitField0_ = 0;
+        if (sketchesBuilder_ == null) {
+          sketches_ = java.util.Collections.emptyList();
+        } else {
+          sketches_ = null;
+          sketchesBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000001);
+        metadata_ = null;
+        if (metadataBuilder_ != null) {
+          metadataBuilder_.dispose();
+          metadataBuilder_ = null;
+        }
+        return this;
+      }
+
+      @java.lang.Override
+      public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+        return datadog.agentpayload.AgentPayload
+            .internal_static_datadog_agentpayload_SketchPayload_descriptor;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.SketchPayload getDefaultInstanceForType() {
+        return datadog.agentpayload.AgentPayload.SketchPayload.getDefaultInstance();
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.SketchPayload build() {
+        datadog.agentpayload.AgentPayload.SketchPayload result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+
+      @java.lang.Override
+      public datadog.agentpayload.AgentPayload.SketchPayload buildPartial() {
+        datadog.agentpayload.AgentPayload.SketchPayload result =
+            new datadog.agentpayload.AgentPayload.SketchPayload(this);
+        buildPartialRepeatedFields(result);
+        if (bitField0_ != 0) {
+          buildPartial0(result);
+        }
+        onBuilt();
+        return result;
+      }
+
+      private void buildPartialRepeatedFields(
+          datadog.agentpayload.AgentPayload.SketchPayload result) {
+        if (sketchesBuilder_ == null) {
+          if (((bitField0_ & 0x00000001) != 0)) {
+            sketches_ = java.util.Collections.unmodifiableList(sketches_);
+            bitField0_ = (bitField0_ & ~0x00000001);
+          }
+          result.sketches_ = sketches_;
+        } else {
+          result.sketches_ = sketchesBuilder_.build();
+        }
+      }
+
+      private void buildPartial0(datadog.agentpayload.AgentPayload.SketchPayload result) {
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000002) != 0)) {
+          result.metadata_ = metadataBuilder_ == null ? metadata_ : metadataBuilder_.build();
+          to_bitField0_ |= 0x00000001;
+        }
+        result.bitField0_ |= to_bitField0_;
+      }
+
+      @java.lang.Override
+      public Builder clone() {
+        return super.clone();
+      }
+
+      @java.lang.Override
+      public Builder setField(
+          com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+        return super.setField(field, value);
+      }
+
+      @java.lang.Override
+      public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+        return super.clearField(field);
+      }
+
+      @java.lang.Override
+      public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+        return super.clearOneof(oneof);
+      }
+
+      @java.lang.Override
+      public Builder setRepeatedField(
+          com.google.protobuf.Descriptors.FieldDescriptor field,
+          int index,
+          java.lang.Object value) {
+        return super.setRepeatedField(field, index, value);
+      }
+
+      @java.lang.Override
+      public Builder addRepeatedField(
+          com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) {
+        return super.addRepeatedField(field, value);
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof datadog.agentpayload.AgentPayload.SketchPayload) {
+          return mergeFrom((datadog.agentpayload.AgentPayload.SketchPayload) other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+
+      public Builder mergeFrom(datadog.agentpayload.AgentPayload.SketchPayload other) {
+        if (other == datadog.agentpayload.AgentPayload.SketchPayload.getDefaultInstance())
+          return this;
+        if (sketchesBuilder_ == null) {
+          if (!other.sketches_.isEmpty()) {
+            if (sketches_.isEmpty()) {
+              sketches_ = other.sketches_;
+              bitField0_ = (bitField0_ & ~0x00000001);
+            } else {
+              ensureSketchesIsMutable();
+              sketches_.addAll(other.sketches_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.sketches_.isEmpty()) {
+            if (sketchesBuilder_.isEmpty()) {
+              sketchesBuilder_.dispose();
+              sketchesBuilder_ = null;
+              sketches_ = other.sketches_;
+              bitField0_ = (bitField0_ & ~0x00000001);
+              sketchesBuilder_ =
+                  com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders
+                      ? getSketchesFieldBuilder()
+                      : null;
+            } else {
+              sketchesBuilder_.addAllMessages(other.sketches_);
+            }
+          }
+        }
+        if (other.hasMetadata()) {
+          mergeMetadata(other.getMetadata());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        onChanged();
+        return this;
+      }
+
+      @java.lang.Override
+      public final boolean isInitialized() {
+        return true;
+      }
+
+      @java.lang.Override
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        if (extensionRegistry == null) {
+          throw new java.lang.NullPointerException();
+        }
+        try {
+          boolean done = false;
+          while (!done) {
+            int tag = input.readTag();
+            switch (tag) {
+              case 0:
+                done = true;
+                break;
+              case 10:
+                {
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch m =
+                      input.readMessage(
+                          datadog.agentpayload.AgentPayload.SketchPayload.Sketch.parser(),
+                          extensionRegistry);
+                  if (sketchesBuilder_ == null) {
+                    ensureSketchesIsMutable();
+                    sketches_.add(m);
+                  } else {
+                    sketchesBuilder_.addMessage(m);
+                  }
+                  break;
+                } // case 10
+              case 18:
+                {
+                  input.readMessage(getMetadataFieldBuilder().getBuilder(), extensionRegistry);
+                  bitField0_ |= 0x00000002;
+                  break;
+                } // case 18
+              default:
+                {
+                  if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                    done = true; // was an endgroup tag
+                  }
+                  break;
+                } // default:
+            } // switch (tag)
+          } // while (!done)
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          throw e.unwrapIOException();
+        } finally {
+          onChanged();
+        } // finally
+        return this;
+      }
+
+      private int bitField0_;
+
+      private java.util.List sketches_ =
+          java.util.Collections.emptyList();
+
+      private void ensureSketchesIsMutable() {
+        if (!((bitField0_ & 0x00000001) != 0)) {
+          sketches_ =
+              new java.util.ArrayList(
+                  sketches_);
+          bitField0_ |= 0x00000001;
+        }
+      }
+
+      private com.google.protobuf.RepeatedFieldBuilderV3<
+              datadog.agentpayload.AgentPayload.SketchPayload.Sketch,
+              datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder,
+              datadog.agentpayload.AgentPayload.SketchPayload.SketchOrBuilder>
+          sketchesBuilder_;
+
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public java.util.List
+          getSketchesList() {
+        if (sketchesBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(sketches_);
+        } else {
+          return sketchesBuilder_.getMessageList();
+        }
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public int getSketchesCount() {
+        if (sketchesBuilder_ == null) {
+          return sketches_.size();
+        } else {
+          return sketchesBuilder_.getCount();
+        }
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public datadog.agentpayload.AgentPayload.SketchPayload.Sketch getSketches(int index) {
+        if (sketchesBuilder_ == null) {
+          return sketches_.get(index);
+        } else {
+          return sketchesBuilder_.getMessage(index);
+        }
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder setSketches(
+          int index, datadog.agentpayload.AgentPayload.SketchPayload.Sketch value) {
+        if (sketchesBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureSketchesIsMutable();
+          sketches_.set(index, value);
+          onChanged();
+        } else {
+          sketchesBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder setSketches(
+          int index,
+          datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder builderForValue) {
+        if (sketchesBuilder_ == null) {
+          ensureSketchesIsMutable();
+          sketches_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          sketchesBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder addSketches(datadog.agentpayload.AgentPayload.SketchPayload.Sketch value) {
+        if (sketchesBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureSketchesIsMutable();
+          sketches_.add(value);
+          onChanged();
+        } else {
+          sketchesBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder addSketches(
+          int index, datadog.agentpayload.AgentPayload.SketchPayload.Sketch value) {
+        if (sketchesBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureSketchesIsMutable();
+          sketches_.add(index, value);
+          onChanged();
+        } else {
+          sketchesBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder addSketches(
+          datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder builderForValue) {
+        if (sketchesBuilder_ == null) {
+          ensureSketchesIsMutable();
+          sketches_.add(builderForValue.build());
+          onChanged();
+        } else {
+          sketchesBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder addSketches(
+          int index,
+          datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder builderForValue) {
+        if (sketchesBuilder_ == null) {
+          ensureSketchesIsMutable();
+          sketches_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          sketchesBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder addAllSketches(
+          java.lang.Iterable
+              values) {
+        if (sketchesBuilder_ == null) {
+          ensureSketchesIsMutable();
+          com.google.protobuf.AbstractMessageLite.Builder.addAll(values, sketches_);
+          onChanged();
+        } else {
+          sketchesBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder clearSketches() {
+        if (sketchesBuilder_ == null) {
+          sketches_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000001);
+          onChanged();
+        } else {
+          sketchesBuilder_.clear();
+        }
+        return this;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder removeSketches(int index) {
+        if (sketchesBuilder_ == null) {
+          ensureSketchesIsMutable();
+          sketches_.remove(index);
+          onChanged();
+        } else {
+          sketchesBuilder_.remove(index);
+        }
+        return this;
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder getSketchesBuilder(
+          int index) {
+        return getSketchesFieldBuilder().getBuilder(index);
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public datadog.agentpayload.AgentPayload.SketchPayload.SketchOrBuilder getSketchesOrBuilder(
+          int index) {
+        if (sketchesBuilder_ == null) {
+          return sketches_.get(index);
+        } else {
+          return sketchesBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public java.util.List<
+              ? extends datadog.agentpayload.AgentPayload.SketchPayload.SketchOrBuilder>
+          getSketchesOrBuilderList() {
+        if (sketchesBuilder_ != null) {
+          return sketchesBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(sketches_);
+        }
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder addSketchesBuilder() {
+        return getSketchesFieldBuilder()
+            .addBuilder(
+                datadog.agentpayload.AgentPayload.SketchPayload.Sketch.getDefaultInstance());
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder addSketchesBuilder(
+          int index) {
+        return getSketchesFieldBuilder()
+            .addBuilder(
+                index, datadog.agentpayload.AgentPayload.SketchPayload.Sketch.getDefaultInstance());
+      }
+      /**
+       * 
+       * repeated .datadog.agentpayload.SketchPayload.Sketch sketches = 1 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public java.util.List
+          getSketchesBuilderList() {
+        return getSketchesFieldBuilder().getBuilderList();
+      }
+
+      private com.google.protobuf.RepeatedFieldBuilderV3<
+              datadog.agentpayload.AgentPayload.SketchPayload.Sketch,
+              datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder,
+              datadog.agentpayload.AgentPayload.SketchPayload.SketchOrBuilder>
+          getSketchesFieldBuilder() {
+        if (sketchesBuilder_ == null) {
+          sketchesBuilder_ =
+              new com.google.protobuf.RepeatedFieldBuilderV3<
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch,
+                  datadog.agentpayload.AgentPayload.SketchPayload.Sketch.Builder,
+                  datadog.agentpayload.AgentPayload.SketchPayload.SketchOrBuilder>(
+                  sketches_, ((bitField0_ & 0x00000001) != 0), getParentForChildren(), isClean());
+          sketches_ = null;
+        }
+        return sketchesBuilder_;
+      }
+
+      private datadog.agentpayload.AgentPayload.CommonMetadata metadata_;
+      private com.google.protobuf.SingleFieldBuilderV3<
+              datadog.agentpayload.AgentPayload.CommonMetadata,
+              datadog.agentpayload.AgentPayload.CommonMetadata.Builder,
+              datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder>
+          metadataBuilder_;
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+       * 
+       *
+       * @return Whether the metadata field is set.
+       */
+      public boolean hasMetadata() {
+        return ((bitField0_ & 0x00000002) != 0);
+      }
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+       * 
+       *
+       * @return The metadata.
+       */
+      public datadog.agentpayload.AgentPayload.CommonMetadata getMetadata() {
+        if (metadataBuilder_ == null) {
+          return metadata_ == null
+              ? datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance()
+              : metadata_;
+        } else {
+          return metadataBuilder_.getMessage();
+        }
+      }
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder setMetadata(datadog.agentpayload.AgentPayload.CommonMetadata value) {
+        if (metadataBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          metadata_ = value;
+        } else {
+          metadataBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return this;
+      }
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder setMetadata(
+          datadog.agentpayload.AgentPayload.CommonMetadata.Builder builderForValue) {
+        if (metadataBuilder_ == null) {
+          metadata_ = builderForValue.build();
+        } else {
+          metadataBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return this;
+      }
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder mergeMetadata(datadog.agentpayload.AgentPayload.CommonMetadata value) {
+        if (metadataBuilder_ == null) {
+          if (((bitField0_ & 0x00000002) != 0)
+              && metadata_ != null
+              && metadata_
+                  != datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance()) {
+            getMetadataBuilder().mergeFrom(value);
+          } else {
+            metadata_ = value;
+          }
+        } else {
+          metadataBuilder_.mergeFrom(value);
+        }
+        if (metadata_ != null) {
+          bitField0_ |= 0x00000002;
+          onChanged();
+        }
+        return this;
+      }
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public Builder clearMetadata() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        metadata_ = null;
+        if (metadataBuilder_ != null) {
+          metadataBuilder_.dispose();
+          metadataBuilder_ = null;
+        }
+        onChanged();
+        return this;
+      }
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public datadog.agentpayload.AgentPayload.CommonMetadata.Builder getMetadataBuilder() {
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return getMetadataFieldBuilder().getBuilder();
+      }
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+       * 
+       */
+      public datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder getMetadataOrBuilder() {
+        if (metadataBuilder_ != null) {
+          return metadataBuilder_.getMessageOrBuilder();
+        } else {
+          return metadata_ == null
+              ? datadog.agentpayload.AgentPayload.CommonMetadata.getDefaultInstance()
+              : metadata_;
+        }
+      }
+      /**
+       * .datadog.agentpayload.CommonMetadata metadata = 2 [(.gogoproto.nullable) = false];
+       * 
+       */
+      private com.google.protobuf.SingleFieldBuilderV3<
+              datadog.agentpayload.AgentPayload.CommonMetadata,
+              datadog.agentpayload.AgentPayload.CommonMetadata.Builder,
+              datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder>
+          getMetadataFieldBuilder() {
+        if (metadataBuilder_ == null) {
+          metadataBuilder_ =
+              new com.google.protobuf.SingleFieldBuilderV3<
+                  datadog.agentpayload.AgentPayload.CommonMetadata,
+                  datadog.agentpayload.AgentPayload.CommonMetadata.Builder,
+                  datadog.agentpayload.AgentPayload.CommonMetadataOrBuilder>(
+                  getMetadata(), getParentForChildren(), isClean());
+          metadata_ = null;
+        }
+        return metadataBuilder_;
+      }
+
+      @java.lang.Override
+      public final Builder setUnknownFields(
+          final com.google.protobuf.UnknownFieldSet unknownFields) {
+        return super.setUnknownFields(unknownFields);
+      }
+
+      @java.lang.Override
+      public final Builder mergeUnknownFields(
+          final com.google.protobuf.UnknownFieldSet unknownFields) {
+        return super.mergeUnknownFields(unknownFields);
+      }
+
+      // @@protoc_insertion_point(builder_scope:datadog.agentpayload.SketchPayload)
+    }
+
+    // @@protoc_insertion_point(class_scope:datadog.agentpayload.SketchPayload)
+    private static final datadog.agentpayload.AgentPayload.SketchPayload DEFAULT_INSTANCE;
+
+    static {
+      DEFAULT_INSTANCE = new datadog.agentpayload.AgentPayload.SketchPayload();
+    }
+
+    public static datadog.agentpayload.AgentPayload.SketchPayload getDefaultInstance() {
+      return DEFAULT_INSTANCE;
+    }
+
+    private static final com.google.protobuf.Parser PARSER =
+        new com.google.protobuf.AbstractParser() {
+          @java.lang.Override
+          public SketchPayload parsePartialFrom(
+              com.google.protobuf.CodedInputStream input,
+              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+              throws com.google.protobuf.InvalidProtocolBufferException {
+            Builder builder = newBuilder();
+            try {
+              builder.mergeFrom(input, extensionRegistry);
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+              throw e.setUnfinishedMessage(builder.buildPartial());
+            } catch (com.google.protobuf.UninitializedMessageException e) {
+              throw e.asInvalidProtocolBufferException()
+                  .setUnfinishedMessage(builder.buildPartial());
+            } catch (java.io.IOException e) {
+              throw new com.google.protobuf.InvalidProtocolBufferException(e)
+                  .setUnfinishedMessage(builder.buildPartial());
+            }
+            return builder.buildPartial();
+          }
+        };
+
+    public static com.google.protobuf.Parser parser() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public com.google.protobuf.Parser getParserForType() {
+      return PARSER;
+    }
+
+    @java.lang.Override
+    public datadog.agentpayload.AgentPayload.SketchPayload getDefaultInstanceForType() {
+      return DEFAULT_INSTANCE;
+    }
+  }
+
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_CommonMetadata_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_CommonMetadata_fieldAccessorTable;
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_MetricPayload_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_MetricPayload_fieldAccessorTable;
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_MetricPayload_MetricPoint_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_MetricPayload_MetricPoint_fieldAccessorTable;
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_MetricPayload_Resource_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_MetricPayload_Resource_fieldAccessorTable;
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_MetricPayload_MetricSeries_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_MetricPayload_MetricSeries_fieldAccessorTable;
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_EventsPayload_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_EventsPayload_fieldAccessorTable;
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_EventsPayload_Event_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_EventsPayload_Event_fieldAccessorTable;
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_SketchPayload_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_SketchPayload_fieldAccessorTable;
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_SketchPayload_Sketch_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_SketchPayload_Sketch_fieldAccessorTable;
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_SketchPayload_Sketch_Distribution_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_SketchPayload_Sketch_Distribution_fieldAccessorTable;
+  private static final com.google.protobuf.Descriptors.Descriptor
+      internal_static_datadog_agentpayload_SketchPayload_Sketch_Dogsketch_descriptor;
+  private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_datadog_agentpayload_SketchPayload_Sketch_Dogsketch_fieldAccessorTable;
+
+  public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
+    return descriptor;
+  }
+
+  private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
+
+  static {
+    java.lang.String[] descriptorData = {
+      "\nBgithub.com/DataDog/agent-payload/proto"
+          + "/metrics/agent_payload.proto\022\024datadog.ag"
+          + "entpayload\032-github.com/gogo/protobuf/gog"
+          + "oproto/gogo.proto\"\211\001\n\016CommonMetadata\022\025\n\r"
+          + "agent_version\030\001 \001(\t\022\020\n\010timezone\030\002 \001(\t\022\025\n"
+          + "\rcurrent_epoch\030\003 \001(\001\022\023\n\013internal_ip\030\004 \001("
+          + "\t\022\021\n\tpublic_ip\030\005 \001(\t\022\017\n\007api_key\030\006 \001(\t\"\230\004"
+          + "\n\rMetricPayload\022@\n\006series\030\001 \003(\01320.datado"
+          + "g.agentpayload.MetricPayload.MetricSerie"
+          + "s\032/\n\013MetricPoint\022\r\n\005value\030\001 \001(\001\022\021\n\ttimes"
+          + "tamp\030\002 \001(\003\032&\n\010Resource\022\014\n\004type\030\001 \001(\t\022\014\n\004"
+          + "name\030\002 \001(\t\032\254\002\n\014MetricSeries\022?\n\tresources"
+          + "\030\001 \003(\0132,.datadog.agentpayload.MetricPayl"
+          + "oad.Resource\022\016\n\006metric\030\002 \001(\t\022\014\n\004tags\030\003 \003"
+          + "(\t\022?\n\006points\030\004 \003(\0132/.datadog.agentpayloa"
+          + "d.MetricPayload.MetricPoint\022<\n\004type\030\005 \001("
+          + "\0162..datadog.agentpayload.MetricPayload.M"
+          + "etricType\022\014\n\004unit\030\006 \001(\t\022\030\n\020source_type_n"
+          + "ame\030\007 \001(\t\022\020\n\010interval\030\010 \001(\003J\004\010\t\020\n\"=\n\nMet"
+          + "ricType\022\017\n\013UNSPECIFIED\020\000\022\t\n\005COUNT\020\001\022\010\n\004R"
+          + "ATE\020\002\022\t\n\005GAUGE\020\003\"\252\002\n\rEventsPayload\0229\n\006ev"
+          + "ents\030\001 \003(\0132).datadog.agentpayload.Events"
+          + "Payload.Event\0226\n\010metadata\030\002 \001(\0132$.datado"
+          + "g.agentpayload.CommonMetadata\032\245\001\n\005Event\022"
+          + "\r\n\005title\030\001 \001(\t\022\014\n\004text\030\002 \001(\t\022\n\n\002ts\030\003 \001(\003"
+          + "\022\020\n\010priority\030\004 \001(\t\022\014\n\004host\030\005 \001(\t\022\014\n\004tags"
+          + "\030\006 \003(\t\022\022\n\nalert_type\030\007 \001(\t\022\027\n\017aggregatio"
+          + "n_key\030\010 \001(\t\022\030\n\020source_type_name\030\t \001(\t\"\233\005"
+          + "\n\rSketchPayload\022B\n\010sketches\030\001 \003(\0132*.data"
+          + "dog.agentpayload.SketchPayload.SketchB\004\310"
+          + "\336\037\000\022<\n\010metadata\030\002 \001(\0132$.datadog.agentpay"
+          + "load.CommonMetadataB\004\310\336\037\000\032\207\004\n\006Sketch\022\016\n\006"
+          + "metric\030\001 \001(\t\022\014\n\004host\030\002 \001(\t\022T\n\rdistributi"
+          + "ons\030\003 \003(\01327.datadog.agentpayload.SketchP"
+          + "ayload.Sketch.DistributionB\004\310\336\037\000\022\014\n\004tags"
+          + "\030\004 \003(\t\022O\n\013dogsketches\030\007 \003(\01324.datadog.ag"
+          + "entpayload.SketchPayload.Sketch.Dogsketc"
+          + "hB\004\310\336\037\000\032\215\001\n\014Distribution\022\n\n\002ts\030\001 \001(\003\022\013\n\003"
+          + "cnt\030\002 \001(\003\022\013\n\003min\030\003 \001(\001\022\013\n\003max\030\004 \001(\001\022\013\n\003a"
+          + "vg\030\005 \001(\001\022\013\n\003sum\030\006 \001(\001\022\t\n\001v\030\007 \003(\001\022\t\n\001g\030\010 "
+          + "\003(\r\022\r\n\005delta\030\t \003(\r\022\013\n\003buf\030\n \003(\001\032n\n\tDogsk"
+          + "etch\022\n\n\002ts\030\001 \001(\003\022\013\n\003cnt\030\002 \001(\003\022\013\n\003min\030\003 \001"
+          + "(\001\022\013\n\003max\030\004 \001(\001\022\013\n\003avg\030\005 \001(\001\022\013\n\003sum\030\006 \001("
+          + "\001\022\t\n\001k\030\007 \003(\021\022\t\n\001n\030\010 \003(\rJ\004\010\005\020\006J\004\010\006\020\007R\016dis"
+          + "tributionsKR\016distributionsCB+Z)github.co"
+          + "m/DataDog/agent-payload/v5/gogenb\006proto3"
+    };
+    descriptor =
+        com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(
+            descriptorData,
+            new com.google.protobuf.Descriptors.FileDescriptor[] {
+              com.google.protobuf.GoGoProtos.getDescriptor(),
+            });
+    internal_static_datadog_agentpayload_CommonMetadata_descriptor =
+        getDescriptor().getMessageTypes().get(0);
+    internal_static_datadog_agentpayload_CommonMetadata_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_CommonMetadata_descriptor,
+            new java.lang.String[] {
+              "AgentVersion", "Timezone", "CurrentEpoch", "InternalIp", "PublicIp", "ApiKey",
+            });
+    internal_static_datadog_agentpayload_MetricPayload_descriptor =
+        getDescriptor().getMessageTypes().get(1);
+    internal_static_datadog_agentpayload_MetricPayload_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_MetricPayload_descriptor,
+            new java.lang.String[] {
+              "Series",
+            });
+    internal_static_datadog_agentpayload_MetricPayload_MetricPoint_descriptor =
+        internal_static_datadog_agentpayload_MetricPayload_descriptor.getNestedTypes().get(0);
+    internal_static_datadog_agentpayload_MetricPayload_MetricPoint_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_MetricPayload_MetricPoint_descriptor,
+            new java.lang.String[] {
+              "Value", "Timestamp",
+            });
+    internal_static_datadog_agentpayload_MetricPayload_Resource_descriptor =
+        internal_static_datadog_agentpayload_MetricPayload_descriptor.getNestedTypes().get(1);
+    internal_static_datadog_agentpayload_MetricPayload_Resource_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_MetricPayload_Resource_descriptor,
+            new java.lang.String[] {
+              "Type", "Name",
+            });
+    internal_static_datadog_agentpayload_MetricPayload_MetricSeries_descriptor =
+        internal_static_datadog_agentpayload_MetricPayload_descriptor.getNestedTypes().get(2);
+    internal_static_datadog_agentpayload_MetricPayload_MetricSeries_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_MetricPayload_MetricSeries_descriptor,
+            new java.lang.String[] {
+              "Resources", "Metric", "Tags", "Points", "Type", "Unit", "SourceTypeName", "Interval",
+            });
+    internal_static_datadog_agentpayload_EventsPayload_descriptor =
+        getDescriptor().getMessageTypes().get(2);
+    internal_static_datadog_agentpayload_EventsPayload_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_EventsPayload_descriptor,
+            new java.lang.String[] {
+              "Events", "Metadata",
+            });
+    internal_static_datadog_agentpayload_EventsPayload_Event_descriptor =
+        internal_static_datadog_agentpayload_EventsPayload_descriptor.getNestedTypes().get(0);
+    internal_static_datadog_agentpayload_EventsPayload_Event_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_EventsPayload_Event_descriptor,
+            new java.lang.String[] {
+              "Title",
+              "Text",
+              "Ts",
+              "Priority",
+              "Host",
+              "Tags",
+              "AlertType",
+              "AggregationKey",
+              "SourceTypeName",
+            });
+    internal_static_datadog_agentpayload_SketchPayload_descriptor =
+        getDescriptor().getMessageTypes().get(3);
+    internal_static_datadog_agentpayload_SketchPayload_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_SketchPayload_descriptor,
+            new java.lang.String[] {
+              "Sketches", "Metadata",
+            });
+    internal_static_datadog_agentpayload_SketchPayload_Sketch_descriptor =
+        internal_static_datadog_agentpayload_SketchPayload_descriptor.getNestedTypes().get(0);
+    internal_static_datadog_agentpayload_SketchPayload_Sketch_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_SketchPayload_Sketch_descriptor,
+            new java.lang.String[] {
+              "Metric", "Host", "Distributions", "Tags", "Dogsketches",
+            });
+    internal_static_datadog_agentpayload_SketchPayload_Sketch_Distribution_descriptor =
+        internal_static_datadog_agentpayload_SketchPayload_Sketch_descriptor
+            .getNestedTypes()
+            .get(0);
+    internal_static_datadog_agentpayload_SketchPayload_Sketch_Distribution_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_SketchPayload_Sketch_Distribution_descriptor,
+            new java.lang.String[] {
+              "Ts", "Cnt", "Min", "Max", "Avg", "Sum", "V", "G", "Delta", "Buf",
+            });
+    internal_static_datadog_agentpayload_SketchPayload_Sketch_Dogsketch_descriptor =
+        internal_static_datadog_agentpayload_SketchPayload_Sketch_descriptor
+            .getNestedTypes()
+            .get(1);
+    internal_static_datadog_agentpayload_SketchPayload_Sketch_Dogsketch_fieldAccessorTable =
+        new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_datadog_agentpayload_SketchPayload_Sketch_Dogsketch_descriptor,
+            new java.lang.String[] {
+              "Ts", "Cnt", "Min", "Max", "Avg", "Sum", "K", "N",
+            });
+    com.google.protobuf.ExtensionRegistry registry =
+        com.google.protobuf.ExtensionRegistry.newInstance();
+    registry.add(com.google.protobuf.GoGoProtos.nullable);
+    com.google.protobuf.Descriptors.FileDescriptor.internalUpdateFileDescriptor(
+        descriptor, registry);
+    com.google.protobuf.GoGoProtos.getDescriptor();
+  }
+
+  // @@protoc_insertion_point(outer_class_scope)
+}
diff --git a/proxy/src/main/java/org/logstash/beats/Ack.java b/proxy/src/main/java/org/logstash/beats/Ack.java
index a5e5be802..0cdf568da 100644
--- a/proxy/src/main/java/org/logstash/beats/Ack.java
+++ b/proxy/src/main/java/org/logstash/beats/Ack.java
@@ -1,18 +1,19 @@
 package org.logstash.beats;
 
 public class Ack {
-    private final byte protocol;
-    private final int sequence;
-    public Ack(byte protocol, int sequence) {
-        this.protocol = protocol;
-        this.sequence = sequence;
-    }
+  private final byte protocol;
+  private final int sequence;
 
-    public byte getProtocol() {
-        return protocol;
-    }
+  public Ack(byte protocol, int sequence) {
+    this.protocol = protocol;
+    this.sequence = sequence;
+  }
 
-    public int getSequence() {
-        return sequence;
-    }
+  public byte getProtocol() {
+    return protocol;
+  }
+
+  public int getSequence() {
+    return sequence;
+  }
 }
diff --git a/proxy/src/main/java/org/logstash/beats/AckEncoder.java b/proxy/src/main/java/org/logstash/beats/AckEncoder.java
index d78e1ddae..30b8770d7 100644
--- a/proxy/src/main/java/org/logstash/beats/AckEncoder.java
+++ b/proxy/src/main/java/org/logstash/beats/AckEncoder.java
@@ -5,15 +5,14 @@
 import io.netty.handler.codec.MessageToByteEncoder;
 
 /**
- *  This Class is mostly used in the test suite to make the right assertions with the encoded data frame.
- *  This class support creating v1 or v2 lumberjack frames.
- *
+ * This Class is mostly used in the test suite to make the right assertions with the encoded data
+ * frame. This class support creating v1 or v2 lumberjack frames.
  */
 public class AckEncoder extends MessageToByteEncoder {
-    @Override
-    protected void encode(ChannelHandlerContext ctx, Ack ack, ByteBuf out) throws Exception {
-        out.writeByte(ack.getProtocol());
-        out.writeByte('A');
-        out.writeInt(ack.getSequence());
-    }
+  @Override
+  protected void encode(ChannelHandlerContext ctx, Ack ack, ByteBuf out) throws Exception {
+    out.writeByte(ack.getProtocol());
+    out.writeByte('A');
+    out.writeInt(ack.getSequence());
+  }
 }
diff --git a/proxy/src/main/java/org/logstash/beats/Batch.java b/proxy/src/main/java/org/logstash/beats/Batch.java
index 7fb2ee52f..a1e86bf91 100644
--- a/proxy/src/main/java/org/logstash/beats/Batch.java
+++ b/proxy/src/main/java/org/logstash/beats/Batch.java
@@ -1,47 +1,58 @@
 package org.logstash.beats;
 
-import java.util.ArrayList;
-import java.util.List;
-
-public class Batch {
-    private byte protocol = Protocol.VERSION_2;
-    private int batchSize;
-    private List messages = new ArrayList();
-
-    public List getMessages() {
-        return messages;
-    }
-
-    public void addMessage(Message message) {
-        message.setBatch(this);
-        messages.add(message);
-    }
-
-    public int size() {
-        return messages.size();
-    }
-
-    public void setBatchSize(int size) {
-        batchSize = size;
-    }
-
-    public int getBatchSize() {
-        return batchSize;
-    }
-
-    public boolean isEmpty() {
-        return 0 == messages.size();
-    }
-
-    public boolean complete() {
-        return size() == getBatchSize();
-    }
-
-    public byte getProtocol() {
-        return protocol;
-    }
-
-    public void setProtocol(byte protocol) {
-        this.protocol = protocol;
-    }
+/** Interface representing a Batch of {@link Message}. */
+public interface Batch extends Iterable {
+  /**
+   * Returns the protocol of the sent messages that this batch was constructed from
+   *
+   * @return byte - either '1' or '2'
+   */
+  byte getProtocol();
+
+  /**
+   * Number of messages that the batch is expected to contain.
+   *
+   * @return int - number of messages
+   */
+  int getBatchSize();
+
+  /**
+   * Set the number of messages that the batch is expected to contain.
+   *
+   * @param batchSize int - number of messages
+   */
+  void setBatchSize(int batchSize);
+
+  /**
+   * Returns the highest sequence number of the batch.
+   *
+   * @return
+   */
+  int getHighestSequence();
+  /**
+   * Current number of messages in the batch
+   *
+   * @return int
+   */
+  int size();
+
+  /**
+   * Is the batch currently empty?
+   *
+   * @return boolean
+   */
+  boolean isEmpty();
+
+  /**
+   * Is the batch complete?
+   *
+   * @return boolean
+   */
+  boolean isComplete();
+
+  /**
+   * Release the resources associated with the batch. Consumers of the batch *must* release after
+   * use.
+   */
+  void release();
 }
diff --git a/proxy/src/main/java/org/logstash/beats/BatchIdentity.java b/proxy/src/main/java/org/logstash/beats/BatchIdentity.java
new file mode 100644
index 000000000..b2f2ab1eb
--- /dev/null
+++ b/proxy/src/main/java/org/logstash/beats/BatchIdentity.java
@@ -0,0 +1,110 @@
+package org.logstash.beats;
+
+import java.util.Map;
+import java.util.Objects;
+import javax.annotation.Nullable;
+
+/**
+ * Identity of a filebeat batch, based on the first message. Used for duplicate batch detection.
+ *
+ * @author vasily@wavefront.com.
+ */
+public class BatchIdentity {
+  private final String timestampStr;
+  private final int highestSequence;
+  private final int size;
+  @Nullable private final String logFile;
+  @Nullable private final Integer logFileOffset;
+
+  BatchIdentity(
+      String timestampStr,
+      int highestSequence,
+      int size,
+      @Nullable String logFile,
+      @Nullable Integer logFileOffset) {
+    this.timestampStr = timestampStr;
+    this.highestSequence = highestSequence;
+    this.size = size;
+    this.logFile = logFile;
+    this.logFileOffset = logFileOffset;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    BatchIdentity that = (BatchIdentity) o;
+    return this.highestSequence == that.highestSequence
+        && this.size == that.size
+        && Objects.equals(this.timestampStr, that.timestampStr)
+        && Objects.equals(this.logFile, that.logFile)
+        && Objects.equals(this.logFileOffset, that.logFileOffset);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = timestampStr != null ? timestampStr.hashCode() : 0;
+    result = 31 * result + highestSequence;
+    result = 31 * result + size;
+    result = 31 * result + (logFile != null ? logFile.hashCode() : 0);
+    result = 31 * result + (logFileOffset != null ? logFileOffset.hashCode() : 0);
+    return result;
+  }
+
+  @Override
+  public String toString() {
+    return "BatchIdentity{timestampStr="
+        + timestampStr
+        + ", highestSequence="
+        + highestSequence
+        + ", size="
+        + size
+        + ", logFile="
+        + logFile
+        + ", logFileOffset="
+        + logFileOffset
+        + "}";
+  }
+
+  @Nullable
+  public static BatchIdentity valueFrom(Message message) {
+    Map messageData = message.getData();
+    if (!messageData.containsKey("@timestamp")) return null;
+    String logFile = null;
+    Integer logFileOffset = null;
+    if (messageData.containsKey("log")) {
+      Map logData = (Map) messageData.get("log");
+      if (logData.containsKey("offset") && logData.containsKey("file")) {
+        Map logFileData = (Map) logData.get("file");
+        if (logFileData.containsKey("path")) {
+          logFile = (String) logFileData.get("path");
+          logFileOffset = (Integer) logData.get("offset");
+        }
+      }
+    }
+    return new BatchIdentity(
+        (String) messageData.get("@timestamp"),
+        message.getBatch().getHighestSequence(),
+        message.getBatch().size(),
+        logFile,
+        logFileOffset);
+  }
+
+  @Nullable
+  public static String keyFrom(Message message) {
+    Map messageData = message.getData();
+    if (messageData.containsKey("agent")) {
+      Map agentData = (Map) messageData.get("agent");
+      if (agentData.containsKey("id")) {
+        return (String) agentData.get("id");
+      }
+    }
+    if (messageData.containsKey("host")) {
+      Map hostData = (Map) messageData.get("host");
+      if (hostData.containsKey("name")) {
+        return (String) hostData.get("name");
+      }
+    }
+    return null;
+  }
+}
diff --git a/proxy/src/main/java/org/logstash/beats/BeatsHandler.java b/proxy/src/main/java/org/logstash/beats/BeatsHandler.java
index 764f40171..d5905e731 100644
--- a/proxy/src/main/java/org/logstash/beats/BeatsHandler.java
+++ b/proxy/src/main/java/org/logstash/beats/BeatsHandler.java
@@ -1,98 +1,188 @@
 package org.logstash.beats;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.wavefront.common.Utils;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandler;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.SimpleChannelInboundHandler;
-import io.netty.handler.timeout.IdleState;
-import io.netty.handler.timeout.IdleStateEvent;
-import org.apache.log4j.Logger;
-
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.net.InetSocketAddress;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.net.ssl.SSLHandshakeException;
 
 @ChannelHandler.Sharable
 public class BeatsHandler extends SimpleChannelInboundHandler {
-    private final static Logger logger = Logger.getLogger(BeatsHandler.class);
-    private final AtomicBoolean processing = new AtomicBoolean(false);
-    private final IMessageListener messageListener;
-    private ChannelHandlerContext context;
-
-
-    public BeatsHandler(IMessageListener listener) {
-        messageListener = listener;
+  private static final Logger logger = Logger.getLogger(BeatsHandler.class.getCanonicalName());
+  private final IMessageListener messageListener;
+  private final Supplier duplicateBatchesIgnored =
+      Utils.lazySupplier(
+          () ->
+              Metrics.newCounter(
+                  new MetricName("logsharvesting", "", "filebeat-duplicate-batches")));
+  private final Cache batchDedupeCache =
+      Caffeine.newBuilder().expireAfterAccess(1, TimeUnit.HOURS).build();
+
+  public BeatsHandler(IMessageListener listener) {
+    messageListener = listener;
+  }
+
+  @Override
+  public void channelActive(final ChannelHandlerContext ctx) throws Exception {
+    if (logger.isLoggable(Level.FINEST)) {
+      logger.finest(format(ctx, "Channel Active"));
     }
-
-    @Override
-    public void handlerAdded(ChannelHandlerContext ctx) throws Exception {
-        context = ctx;
-        messageListener.onNewConnection(ctx);
+    super.channelActive(ctx);
+    messageListener.onNewConnection(ctx);
+  }
+
+  @Override
+  public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+    super.channelInactive(ctx);
+    if (logger.isLoggable(Level.FINEST)) {
+      logger.finest(format(ctx, "Channel Inactive"));
     }
+    messageListener.onConnectionClose(ctx);
+  }
 
-    @Override
-    public void handlerRemoved(ChannelHandlerContext ctx) throws Exception {
-        messageListener.onConnectionClose(ctx);
+  @Override
+  public void channelRead0(ChannelHandlerContext ctx, Batch batch) throws Exception {
+    if (logger.isLoggable(Level.FINE)) {
+      logger.fine(format(ctx, "Received a new payload"));
     }
-
-    @Override
-    public void channelRead0(ChannelHandlerContext ctx, Batch batch) throws Exception {
-        logger.debug("Received a new payload");
-
-        processing.compareAndSet(false, true);
-
-        for(Message message : batch.getMessages()) {
-            logger.debug("Sending a new message for the listener, sequence: " + message.getSequence());
-            messageListener.onNewMessage(ctx, message);
-
-            if(needAck(message)) {
-                ack(ctx, message);
+    try {
+      boolean isFirstMessage = true;
+      String key;
+      BatchIdentity value;
+      for (Message message : batch) {
+        if (isFirstMessage) {
+          // check whether we've processed that batch already
+          isFirstMessage = false;
+          key = BatchIdentity.keyFrom(message);
+          value = BatchIdentity.valueFrom(message);
+          if (key != null && value != null) {
+            BatchIdentity cached = batchDedupeCache.getIfPresent(key);
+            if (value.equals(cached)) {
+              duplicateBatchesIgnored.get().inc();
+              if (logger.isLoggable(Level.FINE)) {
+                logger.fine(format(ctx, "Duplicate filebeat batch received, ignoring"));
+              }
+              // ack the entire batch and stop processing the rest of it
+              writeAck(
+                  ctx, message.getBatch().getProtocol(), message.getBatch().getHighestSequence());
+              break;
+            } else {
+              batchDedupeCache.put(key, value);
             }
+          }
         }
-        ctx.flush();
-        processing.compareAndSet(true, false);
+        if (logger.isLoggable(Level.FINE)) {
+          logger.fine(
+              format(
+                  ctx,
+                  "Sending a new message for the listener, sequence: " + message.getSequence()));
+        }
+        messageListener.onNewMessage(ctx, message);
 
+        if (needAck(message)) {
+          ack(ctx, message);
+        }
+      }
+    } finally {
+      // this channel is done processing this payload, instruct the connection handler to stop
+      // sending TCP keep alive
+      ctx.channel().attr(ConnectionHandler.CHANNEL_SEND_KEEP_ALIVE).get().set(false);
+      if (logger.isLoggable(Level.FINE)) {
+        logger.log(Level.FINE, () -> String.format(
+            "%s: batches pending: %s",
+            ctx.channel().id().asShortText(),
+            ctx.channel().attr(ConnectionHandler.CHANNEL_SEND_KEEP_ALIVE).get().get()));
+      }
+      batch.release();
+      ctx.flush();
     }
-
-    @Override
-    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+  }
+
+  /*
+   * Do not propagate the SSL handshake exception down to the ruby layer handle it locally instead and close the connection
+   * if the channel is still active. Calling `onException` will flush the content of the codec's buffer, this call
+   * may block the thread in the event loop until completion, this should only affect LS 5 because it still supports
+   * the multiline codec, v6 drop support for buffering codec in the beats input.
+   *
+   * For v5, I cannot drop the content of the buffer because this will create data loss because multiline content can
+   * overlap Filebeat transmission; we were recommending multiline at the source in v5 and in v6 we enforce it.
+   */
+  @Override
+  public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+    try {
+      if (!(cause instanceof SSLHandshakeException)) {
         messageListener.onException(ctx, cause);
-        logger.error("Exception: " + cause.getMessage());
-        ctx.close();
-    }
-
-    @Override
-    public void userEventTriggered(ChannelHandlerContext ctx, Object event) {
-        if(event instanceof IdleStateEvent) {
-            IdleStateEvent e = (IdleStateEvent) event;
-
-            if(e.state() == IdleState.WRITER_IDLE) {
-                sendKeepAlive();
-            } else if(e.state() == IdleState.READER_IDLE) {
-                clientTimeout();
-            }
-        }
+      }
+      String causeMessage =
+          cause.getMessage() == null ? cause.getClass().toString() : cause.getMessage();
+
+      if (logger.isLoggable(Level.FINE)) {
+        logger.log(Level.FINE, format(ctx, "Handling exception: " + causeMessage), cause);
+      }
+      logger.info(format(ctx, "Handling exception: " + causeMessage));
+    } finally {
+      super.exceptionCaught(ctx, cause);
+      ctx.flush();
+      ctx.close();
     }
+  }
 
-    private boolean needAck(Message message) {
-        return message.getSequence() == message.getBatch().getBatchSize();
-    }
+  private boolean needAck(Message message) {
+    return message.getSequence() == message.getBatch().getHighestSequence();
+  }
 
-    private void ack(ChannelHandlerContext ctx, Message message) {
-        writeAck(ctx, message.getBatch().getProtocol(), message.getSequence());
+  private void ack(ChannelHandlerContext ctx, Message message) {
+    if (logger.isLoggable(Level.FINEST)) {
+      logger.finest(format(ctx, "Acking message number " + message.getSequence()));
     }
-
-    private void writeAck(ChannelHandlerContext ctx, byte protocol, int sequence) {
-        ctx.write(new Ack(protocol, sequence));
+    writeAck(ctx, message.getBatch().getProtocol(), message.getSequence());
+    writeAck(ctx, message.getBatch().getProtocol(), 0); // send blank ack
+  }
+
+  private void writeAck(ChannelHandlerContext ctx, byte protocol, int sequence) {
+    ctx.writeAndFlush(new Ack(protocol, sequence))
+        .addListener(
+            (ChannelFutureListener)
+                channelFuture -> {
+                  if (channelFuture.isSuccess() && logger.isLoggable(Level.FINEST) && sequence > 0) {
+                    logger.finest(format(ctx, "Ack complete for message number " + sequence));
+                  }
+                });
+  }
+
+  /*
+   * There is no easy way in Netty to support MDC directly,
+   * we will use similar logic than Netty's LoggingHandler
+   */
+  private String format(ChannelHandlerContext ctx, String message) {
+    InetSocketAddress local = (InetSocketAddress) ctx.channel().localAddress();
+    InetSocketAddress remote = (InetSocketAddress) ctx.channel().remoteAddress();
+
+    String localhost;
+    if (local != null) {
+      localhost = local.getAddress().getHostAddress() + ":" + local.getPort();
+    } else {
+      localhost = "undefined";
     }
 
-    private void clientTimeout() {
-        logger.debug("Client Timeout");
-        this.context.close();
+    String remotehost;
+    if (remote != null) {
+      remotehost = remote.getAddress().getHostAddress() + ":" + remote.getPort();
+    } else {
+      remotehost = "undefined";
     }
 
-    private void sendKeepAlive() {
-        // If we are actually blocked on processing
-        // we can send a keep alive.
-        if(processing.get()) {
-            writeAck(context, Protocol.VERSION_2, 0);
-        }
-    }
+    return "[local: " + localhost + ", remote: " + remotehost + "] " + message;
+  }
 }
diff --git a/proxy/src/main/java/org/logstash/beats/BeatsParser.java b/proxy/src/main/java/org/logstash/beats/BeatsParser.java
index 0ee361722..61dcea82f 100644
--- a/proxy/src/main/java/org/logstash/beats/BeatsParser.java
+++ b/proxy/src/main/java/org/logstash/beats/BeatsParser.java
@@ -1,248 +1,264 @@
 package org.logstash.beats;
 
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.module.afterburner.AfterburnerModule;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.ByteBufOutputStream;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.ByteToMessageDecoder;
-import org.apache.log4j.Logger;
-
-
 import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.zip.Inflater;
 import java.util.zip.InflaterOutputStream;
-
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 public class BeatsParser extends ByteToMessageDecoder {
-    private static final int CHUNK_SIZE = 1024;
-    public final static ObjectMapper MAPPER = new ObjectMapper().registerModule(new AfterburnerModule());
-    private final static Logger logger = Logger.getLogger(BeatsParser.class);
-
-    private Batch batch = new Batch();
-
-    private enum States {
-        READ_HEADER(1),
-        READ_FRAME_TYPE(1),
-        READ_WINDOW_SIZE(4),
-        READ_JSON_HEADER(8),
-        READ_COMPRESSED_FRAME_HEADER(4),
-        READ_COMPRESSED_FRAME(-1), // -1 means the length to read is variable and defined in the frame itself.
-        READ_JSON(-1),
-        READ_DATA_FIELDS(-1);
-
-        private int length;
-
-        States(int length) {
-            this.length = length;
-        }
+  private static final Logger logger = Logger.getLogger(BeatsParser.class.getCanonicalName());
 
-    }
+  private Batch batch;
 
-    private States currentState = States.READ_HEADER;
-    private int requiredBytes = 0;
-    private int sequence = 0;
+  private enum States {
+    READ_HEADER(1),
+    READ_FRAME_TYPE(1),
+    READ_WINDOW_SIZE(4),
+    READ_JSON_HEADER(8),
+    READ_COMPRESSED_FRAME_HEADER(4),
+    READ_COMPRESSED_FRAME(
+        -1), // -1 means the length to read is variable and defined in the frame itself.
+    READ_JSON(-1),
+    READ_DATA_FIELDS(-1);
 
-    @Override
-    protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception {
-        if(!hasEnoughBytes(in)) {
-            return;
-        }
+    private int length;
 
-        switch (currentState) {
-            case READ_HEADER: {
-                logger.debug("Running: READ_HEADER");
+    States(int length) {
+      this.length = length;
+    }
+  }
 
-                byte currentVersion = in.readByte();
+  private States currentState = States.READ_HEADER;
+  private int requiredBytes = 0;
+  private int sequence = 0;
 
-                if(Protocol.isVersion2(currentVersion)) {
-                    logger.debug("Frame version 2 detected");
-                    batch.setProtocol(Protocol.VERSION_2);
-                } else {
-                    logger.debug("Frame version 1 detected");
-                    batch.setProtocol(Protocol.VERSION_1);
-                }
+  @Override
+  protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception {
+    if (!hasEnoughBytes(in)) {
+      return;
+    }
 
-                transition(States.READ_FRAME_TYPE);
-                break;
+    switch (currentState) {
+      case READ_HEADER:
+        {
+          logger.finest("Running: READ_HEADER");
+
+          byte currentVersion = in.readByte();
+          if (batch == null) {
+            if (Protocol.isVersion2(currentVersion)) {
+              batch = new V2Batch();
+              logger.finest("Frame version 2 detected");
+            } else {
+              logger.finest("Frame version 1 detected");
+              batch = new V1Batch();
             }
-            case READ_FRAME_TYPE: {
-                byte frameType = in.readByte();
-
-                switch(frameType) {
-                    case Protocol.CODE_WINDOW_SIZE: {
-                        transition(States.READ_WINDOW_SIZE);
-                        break;
-                    }
-                    case Protocol.CODE_JSON_FRAME: {
-                        // Reading Sequence + size of the payload
-                        transition(States.READ_JSON_HEADER);
-                        break;
-                    }
-                    case Protocol.CODE_COMPRESSED_FRAME: {
-                        transition(States.READ_COMPRESSED_FRAME_HEADER);
-                        break;
-                    }
-                    case Protocol.CODE_FRAME: {
-                        transition(States.READ_DATA_FIELDS);
-                        break;
-                    }
-                    default: {
-                        throw new InvalidFrameProtocolException("Invalid Frame Type, received: " + frameType);
-                    }
-                }
+          }
+          transition(States.READ_FRAME_TYPE);
+          break;
+        }
+      case READ_FRAME_TYPE:
+        {
+          byte frameType = in.readByte();
+
+          switch (frameType) {
+            case Protocol.CODE_WINDOW_SIZE:
+              {
+                transition(States.READ_WINDOW_SIZE);
                 break;
-            }
-            case READ_WINDOW_SIZE: {
-                logger.debug("Running: READ_WINDOW_SIZE");
-
-                batch.setBatchSize((int) in.readUnsignedInt());
-
-                // This is unlikely to happen but I have no way to known when a frame is
-                // actually completely done other than checking the windows and the sequence number,
-                // If the FSM read a new window and I have still
-                // events buffered I should send the current batch down to the next handler.
-                if(!batch.isEmpty()) {
-                    logger.warn("New window size received but the current batch was not complete, sending the current batch");
-                    out.add(batch);
-                    batchComplete();
-                }
-
-                transition(States.READ_HEADER);
+              }
+            case Protocol.CODE_JSON_FRAME:
+              {
+                // Reading Sequence + size of the payload
+                transition(States.READ_JSON_HEADER);
                 break;
-            }
-            case READ_DATA_FIELDS: {
-                // Lumberjack version 1 protocol, which use the Key:Value format.
-                logger.debug("Running: READ_DATA_FIELDS");
-                sequence = (int) in.readUnsignedInt();
-                int fieldsCount = (int) in.readUnsignedInt();
-                int count = 0;
-
-                if(fieldsCount <= 0) {
-                    throw new InvalidFrameProtocolException("Invalid number of fields, received: " + fieldsCount);
-                }
-
-                Map dataMap = new HashMap(fieldsCount);
-
-                while(count < fieldsCount) {
-                    int fieldLength = (int) in.readUnsignedInt();
-                    ByteBuf fieldBuf = in.readBytes(fieldLength);
-                    String field = fieldBuf.toString(Charset.forName("UTF8"));
-                    fieldBuf.release();
-
-                    int dataLength = (int) in.readUnsignedInt();
-                    ByteBuf dataBuf = in.readBytes(dataLength);
-                    String data = dataBuf.toString(Charset.forName("UTF8"));
-                    dataBuf.release();
-
-                    dataMap.put(field, data);
-
-                    count++;
-                }
-
-                Message message = new Message(sequence, dataMap);
-                batch.addMessage(message);
-
-                if(batch.complete()) {
-                    out.add(batch);
-                    batchComplete();
-                }
-
-                transition(States.READ_HEADER);
-
+              }
+            case Protocol.CODE_COMPRESSED_FRAME:
+              {
+                transition(States.READ_COMPRESSED_FRAME_HEADER);
                 break;
-            }
-            case READ_JSON_HEADER: {
-                logger.debug("Running: READ_JSON_HEADER");
-
-                sequence = (int) in.readUnsignedInt();
-                int jsonPayloadSize = (int) in.readUnsignedInt();
-
-                if(jsonPayloadSize <= 0) {
-                    throw new InvalidFrameProtocolException("Invalid json length, received: " + jsonPayloadSize);
-                }
-
-                transition(States.READ_JSON, jsonPayloadSize);
+              }
+            case Protocol.CODE_FRAME:
+              {
+                transition(States.READ_DATA_FIELDS);
                 break;
-            }
-            case READ_COMPRESSED_FRAME_HEADER: {
-                logger.debug("Running: READ_COMPRESSED_FRAME_HEADER");
-
-                transition(States.READ_COMPRESSED_FRAME, in.readInt());
-                break;
-            }
-
-            case READ_COMPRESSED_FRAME: {
-                logger.debug("Running: READ_COMPRESSED_FRAME");
-                // Use the compressed size as the safe start for the buffer.
-                ByteBuf buffer = ctx.alloc().buffer(requiredBytes);
-                try (
-                        ByteBufOutputStream buffOutput = new ByteBufOutputStream(buffer);
-                        InflaterOutputStream inflater = new InflaterOutputStream(buffOutput, new Inflater());
-                ) {
-                    ByteBuf bytesRead = in.readBytes(inflater, requiredBytes);
-                    transition(States.READ_HEADER);
-                    try {
-                        while (buffer.readableBytes() > 0) {
-                            decode(ctx, buffer, out);
-                        }
-                    } finally {
-                        buffer.release();
-                    }
-                }
-
-                break;
-            }
-            case READ_JSON: {
-                logger.debug("Running: READ_JSON");
-
-                byte[] bytes = new byte[requiredBytes];
-                in.readBytes(bytes);
-                Message message = new Message(sequence, (Map) MAPPER.readValue(bytes, Object.class));
+              }
+            default:
+              {
+                throw new InvalidFrameProtocolException(
+                    "Invalid Frame Type, received: " + frameType);
+              }
+          }
+          break;
+        }
+      case READ_WINDOW_SIZE:
+        {
+          logger.finest("Running: READ_WINDOW_SIZE");
+          batch.setBatchSize((int) in.readUnsignedInt());
+
+          // This is unlikely to happen but I have no way to known when a frame is
+          // actually completely done other than checking the windows and the sequence
+          // number,
+          // If the FSM read a new window and I have still
+          // events buffered I should send the current batch down to the next handler.
+          if (!batch.isEmpty()) {
+            logger.warning(
+                "New window size received but the current batch was not complete, sending the current batch");
+            out.add(batch);
+            batchComplete();
+          }
+
+          transition(States.READ_HEADER);
+          break;
+        }
+      case READ_DATA_FIELDS:
+        {
+          // Lumberjack version 1 protocol, which use the Key:Value format.
+          logger.finest("Running: READ_DATA_FIELDS");
+          sequence = (int) in.readUnsignedInt();
+          int fieldsCount = (int) in.readUnsignedInt();
+          int count = 0;
+
+          if (fieldsCount <= 0) {
+            throw new InvalidFrameProtocolException(
+                "Invalid number of fields, received: " + fieldsCount);
+          }
+
+          Map dataMap = new HashMap(fieldsCount);
+
+          while (count < fieldsCount) {
+            int fieldLength = (int) in.readUnsignedInt();
+            ByteBuf fieldBuf = in.readBytes(fieldLength);
+            String field = fieldBuf.toString(Charset.forName("UTF8"));
+            fieldBuf.release();
+
+            int dataLength = (int) in.readUnsignedInt();
+            ByteBuf dataBuf = in.readBytes(dataLength);
+            String data = dataBuf.toString(Charset.forName("UTF8"));
+            dataBuf.release();
+
+            dataMap.put(field, data);
+
+            count++;
+          }
+          Message message = new Message(sequence, dataMap);
+          ((V1Batch) batch).addMessage(message);
+
+          if (batch.isComplete()) {
+            out.add(batch);
+            batchComplete();
+          }
+          transition(States.READ_HEADER);
+
+          break;
+        }
+      case READ_JSON_HEADER:
+        {
+          logger.finest("Running: READ_JSON_HEADER");
 
-                batch.addMessage(message);
+          sequence = (int) in.readUnsignedInt();
+          int jsonPayloadSize = (int) in.readUnsignedInt();
 
-                if(batch.size() == batch.getBatchSize()) {
-                    logger.debug("Sending batch size: " + this.batch.size() + ", windowSize: " + batch.getBatchSize() +  " , seq: " + sequence);
+          if (jsonPayloadSize <= 0) {
+            throw new InvalidFrameProtocolException(
+                "Invalid json length, received: " + jsonPayloadSize);
+          }
 
-                    out.add(batch);
-                    batchComplete();
-                }
+          transition(States.READ_JSON, jsonPayloadSize);
+          break;
+        }
+      case READ_COMPRESSED_FRAME_HEADER:
+        {
+          logger.finest("Running: READ_COMPRESSED_FRAME_HEADER");
 
-                transition(States.READ_HEADER);
-                break;
-            }
+          transition(States.READ_COMPRESSED_FRAME, in.readInt());
+          break;
         }
-    }
 
-    private boolean hasEnoughBytes(ByteBuf in) {
-        return in.readableBytes() >= requiredBytes;
-    }
+      case READ_COMPRESSED_FRAME:
+        {
+          logger.finest("Running: READ_COMPRESSED_FRAME");
+          // Use the compressed size as the safe start for the buffer.
+          ByteBuf buffer = ctx.alloc().buffer(requiredBytes);
+          try (ByteBufOutputStream buffOutput = new ByteBufOutputStream(buffer);
+              InflaterOutputStream inflater =
+                  new InflaterOutputStream(buffOutput, new Inflater())) {
+            in.readBytes(inflater, requiredBytes);
+            transition(States.READ_HEADER);
+            try {
+              while (buffer.readableBytes() > 0) {
+                decode(ctx, buffer, out);
+              }
+            } finally {
+              buffer.release();
+            }
+          }
 
-    private void transition(States next) {
-        transition(next, next.length);
-    }
+          break;
+        }
+      case READ_JSON:
+        {
+          logger.finest("Running: READ_JSON");
+          ((V2Batch) batch).addMessage(sequence, in, requiredBytes);
+          if (batch.isComplete()) {
+            if (logger.isLoggable(Level.FINEST)) {
+              logger.finest(
+                  "Sending batch size: "
+                      + this.batch.size()
+                      + ", windowSize: "
+                      + batch.getBatchSize()
+                      + " , seq: "
+                      + sequence);
+            }
+            out.add(batch);
+            batchComplete();
+          }
 
-    private void transition(States nextState, int requiredBytes) {
-        logger.debug("Transition, from: " + currentState + ", to: " +  nextState + ", requiring " + requiredBytes + " bytes");
-        this.currentState = nextState;
-        this.requiredBytes = requiredBytes;
+          transition(States.READ_HEADER);
+          break;
+        }
     }
-
-    private void batchComplete() {
-        requiredBytes = 0;
-        sequence = 0;
-        batch = new Batch();
+  }
+
+  private boolean hasEnoughBytes(ByteBuf in) {
+    return in.readableBytes() >= requiredBytes;
+  }
+
+  private void transition(States next) {
+    transition(next, next.length);
+  }
+
+  private void transition(States nextState, int requiredBytes) {
+    if (logger.isLoggable(Level.FINEST)) {
+      logger.finest(
+          "Transition, from: "
+              + currentState
+              + ", to: "
+              + nextState
+              + ", requiring "
+              + requiredBytes
+              + " bytes");
     }
-
-    public class InvalidFrameProtocolException extends Exception {
-        InvalidFrameProtocolException(String message) {
-            super(message);
-        }
+    this.currentState = nextState;
+    this.requiredBytes = requiredBytes;
+  }
+
+  private void batchComplete() {
+    requiredBytes = 0;
+    sequence = 0;
+    batch = null;
+  }
+
+  public class InvalidFrameProtocolException extends Exception {
+    InvalidFrameProtocolException(String message) {
+      super(message);
     }
-
+  }
 }
diff --git a/proxy/src/main/java/org/logstash/beats/ConnectionHandler.java b/proxy/src/main/java/org/logstash/beats/ConnectionHandler.java
new file mode 100644
index 000000000..19230e9cb
--- /dev/null
+++ b/proxy/src/main/java/org/logstash/beats/ConnectionHandler.java
@@ -0,0 +1,129 @@
+package org.logstash.beats;
+
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.timeout.IdleState;
+import io.netty.handler.timeout.IdleStateEvent;
+import io.netty.util.AttributeKey;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/** Manages the connection state to the beats client. */
+public class ConnectionHandler extends ChannelDuplexHandler {
+  private static final Logger logger = Logger.getLogger(ConnectionHandler.class.getCanonicalName());
+
+  public static final AttributeKey CHANNEL_SEND_KEEP_ALIVE =
+      AttributeKey.valueOf("channel-send-keep-alive");
+
+  @Override
+  public void channelActive(final ChannelHandlerContext ctx) throws Exception {
+    ctx.channel().attr(CHANNEL_SEND_KEEP_ALIVE).set(new AtomicBoolean(false));
+    if (logger.isLoggable(Level.FINEST)) {
+      logger.log(Level.FINEST, "{}: channel activated", ctx.channel().id().asShortText());
+    }
+    super.channelActive(ctx);
+  }
+
+  /**
+   * {@inheritDoc} Sets the flag that the keep alive should be sent. {@link BeatsHandler} will
+   * un-set it. It is important that this handler comes before the {@link BeatsHandler} in the
+   * channel pipeline. Note - For large payloads, this method may be called many times more often
+   * then the BeatsHandler#channelRead due to decoder aggregating the payload.
+   */
+  @Override
+  public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+    ctx.channel().attr(CHANNEL_SEND_KEEP_ALIVE).get().set(true);
+    if (logger.isLoggable(Level.FINE)) {
+      logger.log(Level.FINE, () -> String.format(
+          "%s: batches pending: %s",
+          ctx.channel().id().asShortText(),
+          ctx.channel().attr(CHANNEL_SEND_KEEP_ALIVE).get().get()));
+    }
+    super.channelRead(ctx, msg);
+  }
+
+  /**
+   * {@inheritDoc} 

+   *
+   * 
IdleState.WRITER_IDLE 

+   *  If no response has been issued after the configured write idle timeout via {@link
+   * io.netty.handler.timeout.IdleStateHandler}, then start to issue a TCP keep alive. This can
+   * happen when the pipeline is blocked. Pending (blocked) batches are in either in the EventLoop
+   * attempting to write to the queue, or may be in a taskPending queue waiting for the EventLoop to
+   * unblock. This keep alive holds open the TCP connection from the Beats client so that it will
+   * not timeout and retry which could result in duplicates. 

+   *
+   * 
IdleState.ALL_IDLE 

+   *  If no read or write has been issued after the configured all idle timeout via {@link
+   * io.netty.handler.timeout.IdleStateHandler}, then close the connection. This is really only
+   * happens for beats that are sending sparse amounts of data, and helps to the keep the number of
+   * concurrent connections in check. Note that ChannelOption.SO_LINGER = 0 needs to be set to
+   * ensure we clean up quickly. Also note that the above keep alive counts as a not-idle, and thus
+   * the keep alive will prevent this logic from closing the connection. For this reason, we stop
+   * sending keep alives when there are no more pending batches to allow this idle close timer to
+   * take effect.
+   */
+  @Override
+  public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
+    IdleStateEvent e;
+    if (evt instanceof IdleStateEvent) {
+      e = (IdleStateEvent) evt;
+      if (e.state() == IdleState.WRITER_IDLE) {
+        if (sendKeepAlive(ctx)) {
+          ChannelFuture f = ctx.writeAndFlush(new Ack(Protocol.VERSION_2, 0));
+          if (logger.isLoggable(Level.FINEST)) {
+            logger.log(Level.FINEST, () -> String.format("%s: sending keep alive ack to libbeat", ctx.channel().id().asShortText()));
+            f.addListener(
+                (ChannelFutureListener)
+                    future -> {
+                      if (future.isSuccess()) {
+                        logger.log(Level.FINEST, "{0}: acking was successful", ctx.channel().id().asShortText());
+                      } else {
+                        logger.log(Level.FINEST, "{0}: acking failed", ctx.channel().id().asShortText());
+                      }
+                    });
+          }
+        }
+      } else if (e.state() == IdleState.ALL_IDLE) {
+        logger.log(Level.FINE,
+            "{}: reader and writer are idle, closing remote connection",
+            ctx.channel().id().asShortText());
+        ctx.flush();
+        ChannelFuture f = ctx.close();
+        if (logger.isLoggable(Level.FINEST)) {
+          f.addListener(
+              (future) -> {
+                if (future.isSuccess()) {
+                  logger.finest("closed ctx successfully");
+                } else {
+                  logger.finest("could not close ctx");
+                }
+              });
+        }
+      }
+    }
+  }
+
+  /**
+   * Determine if this channel has finished processing it's payload. If it has not, send a TCP keep
+   * alive. Note - for this to work, the following must be true:
+   *
+   * 
    
+   *   
  • This Handler comes before the {@link BeatsHandler} in the channel's pipeline
+   *   
  • This Handler is associated to an {@link io.netty.channel.EventLoopGroup} that has
+   *       guarantees that the associated {@link io.netty.channel.EventLoop} will never block.
+   *   
  • The {@link BeatsHandler} un-sets only after it has processed this channel's payload.
+   * 

+   *
+   * @param ctx the {@link ChannelHandlerContext} used to curry the flag.
+   * @return Returns true if this channel/connection has NOT finished processing it's payload. False
+   *     otherwise.
+   */
+  public boolean sendKeepAlive(ChannelHandlerContext ctx) {
+    return ctx.channel().hasAttr(CHANNEL_SEND_KEEP_ALIVE)
+        && ctx.channel().attr(CHANNEL_SEND_KEEP_ALIVE).get().get();
+  }
+}
diff --git a/proxy/src/main/java/org/logstash/beats/IMessageListener.java b/proxy/src/main/java/org/logstash/beats/IMessageListener.java
index 478567c0c..ab97bcb44 100644
--- a/proxy/src/main/java/org/logstash/beats/IMessageListener.java
+++ b/proxy/src/main/java/org/logstash/beats/IMessageListener.java
@@ -3,49 +3,50 @@
 import io.netty.channel.ChannelHandlerContext;
 
 /**
- * This class is implemented in ruby in `lib/logstash/inputs/beats/message_listener`,
- * this class is used to link the events triggered from the different connection to the actual
- * work inside the plugin.
+ * This class is implemented in ruby in `lib/logstash/inputs/beats/message_listener`, this class is
+ * used to link the events triggered from the different connection to the actual work inside the
+ * plugin.
  */
 public interface IMessageListener {
-    /**
-     * This is triggered on every new message parsed by the beats handler
-     * and should be executed in the ruby world.
-     *
-     * @param ctx
-     * @param message
-     */
-    public void onNewMessage(ChannelHandlerContext ctx, Message message);
+  /**
+   * This is triggered on every new message parsed by the beats handler and should be executed in
+   * the ruby world.
+   *
+   * @param ctx
+   * @param message
+   */
+  public void onNewMessage(ChannelHandlerContext ctx, Message message);
 
-    /**
-     * Triggered when a new client connect to the input, this is used to link a connection
-     * to a codec in the ruby world.
-     * @param ctx
-     */
-    public void onNewConnection(ChannelHandlerContext ctx);
+  /**
+   * Triggered when a new client connect to the input, this is used to link a connection to a codec
+   * in the ruby world.
+   *
+   * @param ctx
+   */
+  public void onNewConnection(ChannelHandlerContext ctx);
 
-    /**
-     * Triggered when a connection is close on the remote end and we need to flush buffered
-     * events to the queue.
-     *
-     * @param ctx
-     */
-    public void onConnectionClose(ChannelHandlerContext ctx);
+  /**
+   * Triggered when a connection is close on the remote end and we need to flush buffered events to
+   * the queue.
+   *
+   * @param ctx
+   */
+  public void onConnectionClose(ChannelHandlerContext ctx);
 
-    /**
-     * Called went something bad occur in the pipeline, allow to clear buffered codec went
-     * somethign goes wrong.
-     *
-     * @param ctx
-     * @param cause
-     */
-    public void onException(ChannelHandlerContext ctx, Throwable cause);
+  /**
+   * Called went something bad occur in the pipeline, allow to clear buffered codec went somethign
+   * goes wrong.
+   *
+   * @param ctx
+   * @param cause
+   */
+  public void onException(ChannelHandlerContext ctx, Throwable cause);
 
-    /**
-     * Called when a error occur in the channel initialize, usually ssl handshake error.
-     *
-     * @param ctx
-     * @param cause
-     */
-    public void onChannelInitializeException(ChannelHandlerContext ctx, Throwable cause);
+  /**
+   * Called when a error occur in the channel initialize, usually ssl handshake error.
+   *
+   * @param ctx
+   * @param cause
+   */
+  public void onChannelInitializeException(ChannelHandlerContext ctx, Throwable cause);
 }
diff --git a/proxy/src/main/java/org/logstash/beats/Message.java b/proxy/src/main/java/org/logstash/beats/Message.java
index 09094bb0f..984d02a11 100644
--- a/proxy/src/main/java/org/logstash/beats/Message.java
+++ b/proxy/src/main/java/org/logstash/beats/Message.java
@@ -1,61 +1,107 @@
 package org.logstash.beats;
 
-import java.util.HashMap;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.module.afterburner.AfterburnerModule;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.Map;
 
 public class Message implements Comparable {
-    private final int sequence;
-    private String identityStream;
-    private final Map data;
-    private Batch batch;
+  private final int sequence;
+  private String identityStream;
+  private Map data;
+  private Batch batch;
+  private ByteBuf buffer;
 
-    public Message(int sequence, Map map) {
-        this.sequence = sequence;
-        this.data = map;
+  public static final ObjectMapper MAPPER =
+      new ObjectMapper().registerModule(new AfterburnerModule());
 
-        identityStream = extractIdentityStream();
-    }
+  /**
+   * Create a message using a map of key, value pairs
+   *
+   * @param sequence sequence number of the message
+   * @param map key/value pairs representing the message
+   */
+  public Message(int sequence, Map map) {
+    this.sequence = sequence;
+    this.data = map;
+  }
 
-    public int getSequence() {
-        return sequence;
-    }
+  /**
+   * Create a message using a ByteBuf holding a Json object. Note that this ctr is *lazy* - it will
+   * not deserialize the Json object until it is needed.
+   *
+   * @param sequence sequence number of the message
+   * @param buffer {@link ByteBuf} buffer containing Json object
+   */
+  public Message(int sequence, ByteBuf buffer) {
+    this.sequence = sequence;
+    this.buffer = buffer;
+  }
 
+  /**
+   * Returns the sequence number of this messsage
+   *
+   * @return
+   */
+  public int getSequence() {
+    return sequence;
+  }
 
-    public Map getData() {
-        return data;
+  /**
+   * Returns a list of key/value pairs representing the contents of the message. Note that this
+   * method is lazy if the Message was created using a {@link ByteBuf}
+   *
+   * @return {@link Map} Map of key/value pairs
+   */
+  public Map getData() {
+    if (data == null && buffer != null) {
+      try (ByteBufInputStream byteBufInputStream = new ByteBufInputStream(buffer)) {
+        data = MAPPER.readValue((InputStream) byteBufInputStream, Map.class);
+        buffer = null;
+      } catch (IOException e) {
+        throw new RuntimeException("Unable to parse beats payload ", e);
+      }
     }
+    return data;
+  }
 
-    @Override
-    public int compareTo(Message o) {
-        return Integer.compare(getSequence(), o.getSequence());
-    }
+  @Override
+  public int compareTo(Message o) {
+    return Integer.compare(getSequence(), o.getSequence());
+  }
 
-    public Batch getBatch() {
-        return batch;
-    }
+  public Batch getBatch() {
+    return batch;
+  }
 
-    public void setBatch(Batch newBatch) {
-        batch = newBatch;
-    }
+  public void setBatch(Batch batch) {
+    this.batch = batch;
+  }
 
-    public String getIdentityStream() {
-        return identityStream;
+  public String getIdentityStream() {
+    if (identityStream == null) {
+      identityStream = extractIdentityStream();
     }
+    return identityStream;
+  }
 
-    private String extractIdentityStream() {
-        Map beatsData = (HashMap) this.getData().get("beat");
-
-        if(beatsData != null) {
-            String id = (String) beatsData.get("id");
-            String resourceId = (String) beatsData.get("resource_id");
+  private String extractIdentityStream() {
+    Map beatsData = (Map) this.getData().get("beat");
 
-            if(id != null && resourceId != null) {
-                return id + "-" + resourceId;
-            } else {
-                return (String) beatsData.get("name") + "-" + (String) beatsData.get("source");
-            }
-        }
+    if (beatsData != null) {
+      String id = (String) beatsData.get("id");
+      String resourceId = (String) beatsData.get("resource_id");
 
-        return null;
+      if (id != null && resourceId != null) {
+        return id + "-" + resourceId;
+      } else {
+        return beatsData.get("name") + "-" + beatsData.get("source");
+      }
     }
+
+    return null;
+  }
 }
diff --git a/proxy/src/main/java/org/logstash/beats/MessageListener.java b/proxy/src/main/java/org/logstash/beats/MessageListener.java
index 838340c88..cdb80f6d3 100644
--- a/proxy/src/main/java/org/logstash/beats/MessageListener.java
+++ b/proxy/src/main/java/org/logstash/beats/MessageListener.java
@@ -1,67 +1,67 @@
 package org.logstash.beats;
 
 import io.netty.channel.ChannelHandlerContext;
-import org.apache.log4j.Logger;
-
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
- * This class is implemented in ruby in `lib/logstash/inputs/beats/message_listener`,
- * this class is used to link the events triggered from the different connection to the actual
- * work inside the plugin.
+ * This class is implemented in ruby in `lib/logstash/inputs/beats/message_listener`, this class is
+ * used to link the events triggered from the different connection to the actual work inside the
+ * plugin.
  */
 // This need to be implemented in Ruby
 public class MessageListener implements IMessageListener {
-    private final static Logger logger = Logger.getLogger(MessageListener.class);
-
+  private static final Logger logger = Logger.getLogger(MessageListener.class.getCanonicalName());
 
-    /**
-     * This is triggered on every new message parsed by the beats handler
-     * and should be executed in the ruby world.
-     *
-     * @param ctx
-     * @param message
-     */
-    public void onNewMessage(ChannelHandlerContext ctx, Message message) {
-        logger.debug("onNewMessage");
-    }
+  /**
+   * This is triggered on every new message parsed by the beats handler and should be executed in
+   * the ruby world.
+   *
+   * @param ctx
+   * @param message
+   */
+  public void onNewMessage(ChannelHandlerContext ctx, Message message) {
+    logger.fine("onNewMessage");
+  }
 
-    /**
-     * Triggered when a new client connect to the input, this is used to link a connection
-     * to a codec in the ruby world.
-     * @param ctx
-     */
-    public void onNewConnection(ChannelHandlerContext ctx) {
-        logger.debug("onNewConnection");
-    }
+  /**
+   * Triggered when a new client connect to the input, this is used to link a connection to a codec
+   * in the ruby world.
+   *
+   * @param ctx
+   */
+  public void onNewConnection(ChannelHandlerContext ctx) {
+    logger.fine("onNewConnection");
+  }
 
-    /**
-     * Triggered when a connection is close on the remote end and we need to flush buffered
-     * events to the queue.
-     *
-     * @param ctx
-     */
-    public void onConnectionClose(ChannelHandlerContext ctx) {
-        logger.debug("onConnectionClose");
-    }
+  /**
+   * Triggered when a connection is close on the remote end and we need to flush buffered events to
+   * the queue.
+   *
+   * @param ctx
+   */
+  public void onConnectionClose(ChannelHandlerContext ctx) {
+    logger.fine("onConnectionClose");
+  }
 
-    /**
-     * Called went something bad occur in the pipeline, allow to clear buffered codec went
-     * somethign goes wrong.
-     *
-     * @param ctx
-     * @param cause
-     */
-    public void onException(ChannelHandlerContext ctx, Throwable cause) {
-        logger.debug("onException");
-    }
+  /**
+   * Called went something bad occur in the pipeline, allow to clear buffered codec went somethign
+   * goes wrong.
+   *
+   * @param ctx
+   * @param cause
+   */
+  public void onException(ChannelHandlerContext ctx, Throwable cause) {
+    logger.fine("onException");
+  }
 
-    /**
-     * Called when a error occur in the channel initialize, usually ssl handshake error.
-     *
-     * @param ctx
-     * @param cause
-     */
-    public void onChannelInitializeException(ChannelHandlerContext ctx, Throwable cause) {
-        logger.debug("onException");
-    }
+  /**
+   * Called when a error occur in the channel initialize, usually ssl handshake error.
+   *
+   * @param ctx
+   * @param cause
+   */
+  public void onChannelInitializeException(ChannelHandlerContext ctx, Throwable cause) {
+    logger.fine("onException");
+  }
 }
diff --git a/proxy/src/main/java/org/logstash/beats/Protocol.java b/proxy/src/main/java/org/logstash/beats/Protocol.java
index 2efd416b5..6d09f1b79 100644
--- a/proxy/src/main/java/org/logstash/beats/Protocol.java
+++ b/proxy/src/main/java/org/logstash/beats/Protocol.java
@@ -1,22 +1,20 @@
 package org.logstash.beats;
 
-/**
- * Created by ph on 2016-05-16.
- */
+/** Created by ph on 2016-05-16. */
 public class Protocol {
-    public static final byte VERSION_1 = '1';
-    public static final byte VERSION_2 = '2';
+  public static final byte VERSION_1 = '1';
+  public static final byte VERSION_2 = '2';
 
-    public static final byte CODE_WINDOW_SIZE = 'W';
-    public static final byte CODE_JSON_FRAME = 'J';
-    public static final byte CODE_COMPRESSED_FRAME = 'C';
-    public static final byte CODE_FRAME = 'D';
+  public static final byte CODE_WINDOW_SIZE = 'W';
+  public static final byte CODE_JSON_FRAME = 'J';
+  public static final byte CODE_COMPRESSED_FRAME = 'C';
+  public static final byte CODE_FRAME = 'D';
 
-    public static boolean isVersion2(byte versionRead) {
-        if(Protocol.VERSION_2 == versionRead){
-            return true;
-        } else {
-            return false;
-        }
+  public static boolean isVersion2(byte versionRead) {
+    if (Protocol.VERSION_2 == versionRead) {
+      return true;
+    } else {
+      return false;
     }
+  }
 }
diff --git a/proxy/src/main/java/org/logstash/beats/Runner.java b/proxy/src/main/java/org/logstash/beats/Runner.java
index 447a47a20..94d48e5d3 100644
--- a/proxy/src/main/java/org/logstash/beats/Runner.java
+++ b/proxy/src/main/java/org/logstash/beats/Runner.java
@@ -1,41 +1,39 @@
 package org.logstash.beats;
 
-import org.apache.log4j.Logger;
+import java.util.logging.Logger;
 import org.logstash.netty.SslSimpleBuilder;
 
-
 public class Runner {
-    private static final int DEFAULT_PORT = 5044;
-    private final static Logger logger = Logger.getLogger(Runner.class);
-
-
+  private static final int DEFAULT_PORT = 5044;
 
-    static public void main(String[] args) throws Exception {
-        logger.info("Starting Beats Bulk");
+  private static final Logger logger = Logger.getLogger(Runner.class.getCanonicalName());
 
-        // Check for leaks.
-        // ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID);
+  public static void main(String[] args) throws Exception {
+    logger.info("Starting Beats Bulk");
 
-        Server server = new Server(DEFAULT_PORT);
+    // Check for leaks.
+    // ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID);
 
-            if(args.length > 0 && args[0].equals("ssl")) {
-            logger.debug("Using SSL");
+    Server server =
+        new Server("0.0.0.0", DEFAULT_PORT, 15, Runtime.getRuntime().availableProcessors());
 
-            String sslCertificate = "/Users/ph/es/certificates/certificate.crt";
-            String sslKey = "/Users/ph/es/certificates/certificate.pkcs8.key";
-            String noPkcs7SslKey = "/Users/ph/es/certificates/certificate.key";
-            String[] certificateAuthorities = new String[] { "/Users/ph/es/certificates/certificate.crt" };
+    if (args.length > 0 && args[0].equals("ssl")) {
+      logger.fine("Using SSL");
 
+      String sslCertificate = "/Users/ph/es/certificates/certificate.crt";
+      String sslKey = "/Users/ph/es/certificates/certificate.pkcs8.key";
+      String noPkcs7SslKey = "/Users/ph/es/certificates/certificate.key";
+      String[] certificateAuthorities = new String[] {"/Users/ph/es/certificates/certificate.crt"};
 
+      SslSimpleBuilder sslBuilder =
+          new SslSimpleBuilder(sslCertificate, sslKey, null)
+              .setProtocols(new String[] {"TLSv1.2"})
+              .setCertificateAuthorities(certificateAuthorities)
+              .setHandshakeTimeoutMilliseconds(10000);
 
-            SslSimpleBuilder sslBuilder = new SslSimpleBuilder(sslCertificate, sslKey, null)
-                    .setProtocols(new String[] { "TLSv1.2" })
-                    .setCertificateAuthorities(certificateAuthorities)
-                    .setHandshakeTimeoutMilliseconds(10000);
-
-            server.enableSSL(sslBuilder);
-        }
-
-        server.listen();
+      server.enableSSL(sslBuilder);
     }
+
+    server.listen();
+  }
 }
diff --git a/proxy/src/main/java/org/logstash/beats/Server.java b/proxy/src/main/java/org/logstash/beats/Server.java
index 847347291..7025c10e2 100644
--- a/proxy/src/main/java/org/logstash/beats/Server.java
+++ b/proxy/src/main/java/org/logstash/beats/Server.java
@@ -1,159 +1,182 @@
 package org.logstash.beats;
 
 import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelPipeline;
+import io.netty.channel.*;
 import io.netty.channel.nio.NioEventLoopGroup;
 import io.netty.channel.socket.SocketChannel;
 import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.logging.LoggingHandler;
 import io.netty.handler.ssl.SslHandler;
 import io.netty.handler.timeout.IdleStateHandler;
 import io.netty.util.concurrent.DefaultEventExecutorGroup;
 import io.netty.util.concurrent.EventExecutorGroup;
-import io.netty.util.concurrent.Future;
-import org.apache.log4j.Logger;
-import org.logstash.netty.SslSimpleBuilder;
-
 import java.io.IOException;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateException;
-import java.util.concurrent.TimeUnit;
-
-
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.logstash.netty.SslSimpleBuilder;
 
 public class Server {
-    private final static Logger logger = Logger.getLogger(Server.class);
-
-    static final long SHUTDOWN_TIMEOUT_SECONDS = 10;
-    private static final int DEFAULT_CLIENT_TIMEOUT_SECONDS = 15;
-
-
-    private final int port;
-    private final NioEventLoopGroup bossGroup;
-    private final NioEventLoopGroup workGroup;
-    private IMessageListener messageListener = new MessageListener();
-    private SslSimpleBuilder sslBuilder;
-
-    private final int clientInactivityTimeoutSeconds;
-
-    public Server(int p) {
-        this(p, DEFAULT_CLIENT_TIMEOUT_SECONDS);
-    }
-
-    public Server(int p, int timeout) {
-        port = p;
-        clientInactivityTimeoutSeconds = timeout;
-        bossGroup = new NioEventLoopGroup(10);
-        //bossGroup.setIoRatio(10);
-        workGroup = new NioEventLoopGroup(50);
-        //workGroup.setIoRatio(10);
+  private static final Logger logger = Logger.getLogger(Server.class.getCanonicalName());
+
+  private final int port;
+  private final String host;
+  private final int beatsHeandlerThreadCount;
+  private NioEventLoopGroup workGroup;
+  private IMessageListener messageListener = new MessageListener();
+  private SslSimpleBuilder sslBuilder;
+  private BeatsInitializer beatsInitializer;
+
+  private final int clientInactivityTimeoutSeconds;
+
+  public Server(String host, int p, int timeout, int threadCount) {
+    this.host = host;
+    port = p;
+    clientInactivityTimeoutSeconds = timeout;
+    beatsHeandlerThreadCount = threadCount;
+  }
+
+  public void enableSSL(SslSimpleBuilder builder) {
+    sslBuilder = builder;
+  }
+
+  public Server listen() throws InterruptedException {
+    if (workGroup != null) {
+      try {
+        logger.fine("Shutting down existing worker group before starting");
+        workGroup.shutdownGracefully().sync();
+      } catch (Exception e) {
+        logger.log(Level.SEVERE, "Could not shut down worker group before starting", e);
+      }
     }
-
-    public void enableSSL(SslSimpleBuilder builder) {
-        sslBuilder = builder;
+    workGroup = new NioEventLoopGroup();
+    try {
+      logger.log(Level.INFO, "Starting server on port: {}", this.port);
+
+      beatsInitializer =
+          new BeatsInitializer(
+              isSslEnable(),
+              messageListener,
+              clientInactivityTimeoutSeconds,
+              beatsHeandlerThreadCount);
+
+      ServerBootstrap server = new ServerBootstrap();
+      server
+          .group(workGroup)
+          .channel(NioServerSocketChannel.class)
+          .childOption(
+              ChannelOption.SO_LINGER,
+              0) // Since the protocol doesn't support yet a remote close from the
+          // server and we
+          // don't want to have 'unclosed' socket lying around we have to use `SO_LINGER`
+          // to
+          // force the close of the socket.
+          .childHandler(beatsInitializer);
+
+      Channel channel = server.bind(host, port).sync().channel();
+      channel.closeFuture().sync();
+    } finally {
+      shutdown();
     }
 
-    public Server listen() throws InterruptedException {
-        BeatsInitializer beatsInitializer = null;
-
-        try {
-            logger.info("Starting server on port: " +  this.port);
-
-            beatsInitializer = new BeatsInitializer(isSslEnable(), messageListener, clientInactivityTimeoutSeconds);
-
-            ServerBootstrap server = new ServerBootstrap();
-            server.group(bossGroup, workGroup)
-                    .channel(NioServerSocketChannel.class)
-                    .childHandler(beatsInitializer);
-
-            Channel channel = server.bind(port).sync().channel();
-            channel.closeFuture().sync();
-        } finally {
-            beatsInitializer.shutdownEventExecutor();
-
-            bossGroup.shutdownGracefully(0, SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS);
-            workGroup.shutdownGracefully(0, SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS);
-        }
-
-        return this;
+    return this;
+  }
+
+  public void stop() {
+    logger.fine("Server shutting down");
+    shutdown();
+    logger.fine("Server stopped");
+  }
+
+  private void shutdown() {
+    try {
+      if (workGroup != null) {
+        workGroup.shutdownGracefully().sync();
+      }
+      if (beatsInitializer != null) {
+        beatsInitializer.shutdownEventExecutor();
+      }
+    } catch (InterruptedException e) {
+      throw new IllegalStateException(e);
     }
-
-    public void stop() throws InterruptedException {
-        logger.debug("Server shutting down");
-
-        Future bossWait = bossGroup.shutdownGracefully(0, SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS);
-        Future workWait = workGroup.shutdownGracefully(0, SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS);
-        
-        logger.debug("Server stopped");
+  }
+
+  public void setMessageListener(IMessageListener listener) {
+    messageListener = listener;
+  }
+
+  public boolean isSslEnable() {
+    return this.sslBuilder != null;
+  }
+
+  private class BeatsInitializer extends ChannelInitializer {
+    private final String SSL_HANDLER = "ssl-handler";
+    private final String IDLESTATE_HANDLER = "idlestate-handler";
+    private final String CONNECTION_HANDLER = "connection-handler";
+    private final String BEATS_ACKER = "beats-acker";
+
+    private final int DEFAULT_IDLESTATEHANDLER_THREAD = 4;
+    private final int IDLESTATE_WRITER_IDLE_TIME_SECONDS = 5;
+
+    private final EventExecutorGroup idleExecutorGroup;
+    private final EventExecutorGroup beatsHandlerExecutorGroup;
+    private final IMessageListener localMessageListener;
+    private final int localClientInactivityTimeoutSeconds;
+    private final boolean localEnableSSL;
+    private final BeatsHandler beatsHandler;
+
+    BeatsInitializer(
+        Boolean enableSSL,
+        IMessageListener messageListener,
+        int clientInactivityTimeoutSeconds,
+        int beatsHandlerThread) {
+      // Keeps a local copy of Server settings, so they can't be modified once it starts
+      // listening
+      this.localEnableSSL = enableSSL;
+      this.localMessageListener = messageListener;
+      this.localClientInactivityTimeoutSeconds = clientInactivityTimeoutSeconds;
+      this.beatsHandler = new BeatsHandler(localMessageListener);
+      idleExecutorGroup = new DefaultEventExecutorGroup(DEFAULT_IDLESTATEHANDLER_THREAD);
+      beatsHandlerExecutorGroup = new DefaultEventExecutorGroup(beatsHandlerThread);
     }
 
-    public void setMessageListener(IMessageListener listener) {
-        messageListener = listener;
+    public void initChannel(SocketChannel socket)
+        throws IOException, NoSuchAlgorithmException, CertificateException {
+      ChannelPipeline pipeline = socket.pipeline();
+
+      if (localEnableSSL) {
+        SslHandler sslHandler = sslBuilder.build(socket.alloc());
+        pipeline.addLast(SSL_HANDLER, sslHandler);
+      }
+      pipeline.addLast(
+          idleExecutorGroup,
+          IDLESTATE_HANDLER,
+          new IdleStateHandler(
+              localClientInactivityTimeoutSeconds,
+              IDLESTATE_WRITER_IDLE_TIME_SECONDS,
+              localClientInactivityTimeoutSeconds));
+      pipeline.addLast(BEATS_ACKER, new AckEncoder());
+      pipeline.addLast(CONNECTION_HANDLER, new ConnectionHandler());
+      pipeline.addLast(beatsHandlerExecutorGroup, new BeatsParser(), beatsHandler);
     }
 
-    public boolean isSslEnable() {
-        return this.sslBuilder != null;
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+      logger.log(Level.SEVERE, "Exception caught in channel initializer", cause);
+      try {
+        localMessageListener.onChannelInitializeException(ctx, cause);
+      } finally {
+        super.exceptionCaught(ctx, cause);
+      }
     }
 
-    private class BeatsInitializer extends ChannelInitializer {
-        private final String LOGGER_HANDLER = "logger";
-        private final String SSL_HANDLER = "ssl-handler";
-        private final String KEEP_ALIVE_HANDLER = "keep-alive-handler";
-        private final String BEATS_PARSER = "beats-parser";
-        private final String BEATS_HANDLER = "beats-handler";
-        private final String BEATS_ACKER = "beats-acker";
-
-        private final int DEFAULT_IDLESTATEHANDLER_THREAD = 4;
-        private final int IDLESTATE_WRITER_IDLE_TIME_SECONDS = 5;
-        private final int IDLESTATE_ALL_IDLE_TIME_SECONDS = 0;
-
-        private final EventExecutorGroup idleExecutorGroup;
-        private final BeatsHandler beatsHandler;
-        private final IMessageListener message;
-        private int clientInactivityTimeoutSeconds;
-        private final LoggingHandler loggingHandler = new LoggingHandler();
-
-
-        private boolean enableSSL = false;
-
-        public BeatsInitializer(Boolean secure, IMessageListener messageListener, int clientInactivityTimeoutSeconds) {
-            enableSSL = secure;
-            this.message = messageListener;
-            beatsHandler = new BeatsHandler(this.message);
-            this.clientInactivityTimeoutSeconds = clientInactivityTimeoutSeconds;
-            idleExecutorGroup = new DefaultEventExecutorGroup(DEFAULT_IDLESTATEHANDLER_THREAD);
-        }
-
-        public void initChannel(SocketChannel socket) throws IOException, NoSuchAlgorithmException, CertificateException {
-            ChannelPipeline pipeline = socket.pipeline();
-
-            pipeline.addLast(LOGGER_HANDLER, loggingHandler);
-
-            if(enableSSL) {
-                SslHandler sslHandler = sslBuilder.build(socket.alloc());
-                pipeline.addLast(SSL_HANDLER, sslHandler);
-            }
-
-            // We have set a specific executor for the idle check, because the `beatsHandler` can be
-            // blocked on the queue, this the idleStateHandler manage the `KeepAlive` signal.
-            pipeline.addLast(idleExecutorGroup, KEEP_ALIVE_HANDLER, new IdleStateHandler(clientInactivityTimeoutSeconds, IDLESTATE_WRITER_IDLE_TIME_SECONDS , IDLESTATE_ALL_IDLE_TIME_SECONDS));
-
-            pipeline.addLast(BEATS_PARSER, new BeatsParser());
-            pipeline.addLast(BEATS_ACKER, new AckEncoder());
-            pipeline.addLast(BEATS_HANDLER, beatsHandler);
-
-        }
-
-        @Override
-        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
-            this.message.onChannelInitializeException(ctx, cause);
-        }
-
-        public void shutdownEventExecutor() {
-            idleExecutorGroup.shutdownGracefully(0, SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS);
-        }
+    public void shutdownEventExecutor() {
+      try {
+        idleExecutorGroup.shutdownGracefully().sync();
+        beatsHandlerExecutorGroup.shutdownGracefully().sync();
+      } catch (InterruptedException e) {
+        throw new IllegalStateException(e);
+      }
     }
+  }
 }
diff --git a/proxy/src/main/java/org/logstash/beats/V1Batch.java b/proxy/src/main/java/org/logstash/beats/V1Batch.java
new file mode 100644
index 000000000..fa3761386
--- /dev/null
+++ b/proxy/src/main/java/org/logstash/beats/V1Batch.java
@@ -0,0 +1,76 @@
+package org.logstash.beats;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/** Implementation of {@link Batch} intended for batches constructed from v1 protocol */
+public class V1Batch implements Batch {
+
+  private int batchSize;
+  private List messages = new ArrayList<>();
+  private byte protocol = Protocol.VERSION_1;
+  private int highestSequence = -1;
+
+  @Override
+  public byte getProtocol() {
+    return protocol;
+  }
+
+  public void setProtocol(byte protocol) {
+    this.protocol = protocol;
+  }
+
+  /**
+   * Add Message to the batch
+   *
+   * @param message Message to add to the batch
+   */
+  void addMessage(Message message) {
+    message.setBatch(this);
+    messages.add(message);
+    if (message.getSequence() > highestSequence) {
+      highestSequence = message.getSequence();
+    }
+  }
+
+  @Override
+  public Iterator iterator() {
+    return messages.iterator();
+  }
+
+  @Override
+  public int getBatchSize() {
+    return batchSize;
+  }
+
+  @Override
+  public void setBatchSize(int batchSize) {
+    this.batchSize = batchSize;
+  }
+
+  @Override
+  public int size() {
+    return messages.size();
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return 0 == messages.size();
+  }
+
+  @Override
+  public int getHighestSequence() {
+    return highestSequence;
+  }
+
+  @Override
+  public boolean isComplete() {
+    return size() == getBatchSize();
+  }
+
+  @Override
+  public void release() {
+    // no-op
+  }
+}
diff --git a/proxy/src/main/java/org/logstash/beats/V2Batch.java b/proxy/src/main/java/org/logstash/beats/V2Batch.java
new file mode 100644
index 000000000..afd40728e
--- /dev/null
+++ b/proxy/src/main/java/org/logstash/beats/V2Batch.java
@@ -0,0 +1,107 @@
+package org.logstash.beats;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.PooledByteBufAllocator;
+import java.util.Iterator;
+
+/**
+ * Implementation of {@link Batch} for the v2 protocol backed by ByteBuf. *must* be released after
+ * use.
+ */
+public class V2Batch implements Batch {
+  private ByteBuf internalBuffer = PooledByteBufAllocator.DEFAULT.buffer();
+  private int written = 0;
+  private int read = 0;
+  private static final int SIZE_OF_INT = 4;
+  private int batchSize;
+  private int highestSequence = -1;
+
+  public void setProtocol(byte protocol) {
+    if (protocol != Protocol.VERSION_2) {
+      throw new IllegalArgumentException("Only version 2 protocol is supported");
+    }
+  }
+
+  @Override
+  public byte getProtocol() {
+    return Protocol.VERSION_2;
+  }
+
+  public Iterator iterator() {
+    internalBuffer.resetReaderIndex();
+    return new Iterator() {
+      @Override
+      public boolean hasNext() {
+        return read < written;
+      }
+
+      @Override
+      public Message next() {
+        int sequenceNumber = internalBuffer.readInt();
+        int readableBytes = internalBuffer.readInt();
+        Message message =
+            new Message(
+                sequenceNumber, internalBuffer.slice(internalBuffer.readerIndex(), readableBytes));
+        internalBuffer.readerIndex(internalBuffer.readerIndex() + readableBytes);
+        message.setBatch(V2Batch.this);
+        read++;
+        return message;
+      }
+    };
+  }
+
+  @Override
+  public int getBatchSize() {
+    return batchSize;
+  }
+
+  @Override
+  public void setBatchSize(final int batchSize) {
+    this.batchSize = batchSize;
+  }
+
+  @Override
+  public int size() {
+    return written;
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return written == 0;
+  }
+
+  @Override
+  public boolean isComplete() {
+    return written == batchSize;
+  }
+
+  @Override
+  public int getHighestSequence() {
+    return highestSequence;
+  }
+
+  /**
+   * Adds a message to the batch, which will be constructed into an actual {@link Message} lazily.
+   *
+   * @param sequenceNumber sequence number of the message within the batch
+   * @param buffer A ByteBuf pointing to serialized JSon
+   * @param size size of the serialized Json
+   */
+  void addMessage(int sequenceNumber, ByteBuf buffer, int size) {
+    written++;
+    if (internalBuffer.writableBytes() < size + (2 * SIZE_OF_INT)) {
+      internalBuffer.capacity(internalBuffer.capacity() + size + (2 * SIZE_OF_INT));
+    }
+    internalBuffer.writeInt(sequenceNumber);
+    internalBuffer.writeInt(size);
+    buffer.readBytes(internalBuffer, size);
+    if (sequenceNumber > highestSequence) {
+      highestSequence = sequenceNumber;
+    }
+  }
+
+  @Override
+  public void release() {
+    internalBuffer.release();
+  }
+}
diff --git a/proxy/src/main/java/org/logstash/netty/SslSimpleBuilder.java b/proxy/src/main/java/org/logstash/netty/SslSimpleBuilder.java
index c2ae7dd3d..a5ab44ba9 100644
--- a/proxy/src/main/java/org/logstash/netty/SslSimpleBuilder.java
+++ b/proxy/src/main/java/org/logstash/netty/SslSimpleBuilder.java
@@ -1,174 +1,193 @@
 package org.logstash.netty;
 
 import io.netty.buffer.ByteBufAllocator;
+import io.netty.handler.ssl.OpenSsl;
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslContextBuilder;
 import io.netty.handler.ssl.SslHandler;
-import org.apache.log4j.Logger;
-import org.logstash.beats.Server;
-
-import javax.net.ssl.SSLEngine;
 import java.io.*;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateException;
 import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
+import javax.net.ssl.SSLEngine;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
-/**
- * Created by ph on 2016-05-27.
- */
+/** Created by ph on 2016-05-27. */
 public class SslSimpleBuilder {
 
-
-    public static enum SslClientVerifyMode {
-        VERIFY_PEER,
-        FORCE_PEER,
-    }
-    private final static Logger logger = Logger.getLogger(SslSimpleBuilder.class);
-
-
-    private File sslKeyFile;
-    private File sslCertificateFile;
-    private SslClientVerifyMode verifyMode = SslClientVerifyMode.FORCE_PEER;
-
-    private long handshakeTimeoutMilliseconds = 10000;
-
-    /*
-    Mordern Ciphers List from
-    https://wiki.mozilla.org/Security/Server_Side_TLS
-    This list require the OpenSSl engine for netty.
-    */
-    public final static String[] DEFAULT_CIPHERS = new String[] {
-            "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA38",
-            "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
-            "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
-            "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
-            "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384",
-            "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384",
-            "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256"
-    };
-
-    private String[] ciphers = DEFAULT_CIPHERS;
-    private String[] protocols = new String[] { "TLSv1.2" };
-    private String[] certificateAuthorities;
-    private String passPhrase;
-
-    public SslSimpleBuilder(String sslCertificateFilePath, String sslKeyFilePath, String pass) throws FileNotFoundException {
-        sslCertificateFile = new File(sslCertificateFilePath);
-        sslKeyFile = new File(sslKeyFilePath);
-        passPhrase = pass;
-        ciphers = DEFAULT_CIPHERS;
-    }
-
-    public SslSimpleBuilder setProtocols(String[] protocols) {
-        protocols = protocols;
-        return this;
-    }
-
-    public SslSimpleBuilder setCipherSuites(String[] ciphersSuite) {
-        ciphers = ciphersSuite;
-        return this;
-    }
-
-    public SslSimpleBuilder setCertificateAuthorities(String[] cert) {
-        certificateAuthorities = cert;
-        return this;
+  public static enum SslClientVerifyMode {
+    VERIFY_PEER,
+    FORCE_PEER,
+  }
+
+  private static final Logger logger = Logger.getLogger(SslSimpleBuilder.class.getCanonicalName());
+
+  private File sslKeyFile;
+  private File sslCertificateFile;
+  private SslClientVerifyMode verifyMode = SslClientVerifyMode.FORCE_PEER;
+
+  private long handshakeTimeoutMilliseconds = 10000;
+
+  /*
+  Mordern Ciphers List from
+  https://wiki.mozilla.org/Security/Server_Side_TLS
+  This list require the OpenSSl engine for netty.
+  */
+  public static final String[] DEFAULT_CIPHERS =
+      new String[] {
+        "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+        "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+        "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+        "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+        "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384",
+        "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384",
+        "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",
+        "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256"
+      };
+
+  private String[] ciphers = DEFAULT_CIPHERS;
+  private String[] protocols = new String[] {"TLSv1.2"};
+  private String[] certificateAuthorities;
+  private String passPhrase;
+
+  public SslSimpleBuilder(String sslCertificateFilePath, String sslKeyFilePath, String pass)
+      throws FileNotFoundException {
+    sslCertificateFile = new File(sslCertificateFilePath);
+    sslKeyFile = new File(sslKeyFilePath);
+    passPhrase = pass;
+    ciphers = DEFAULT_CIPHERS;
+  }
+
+  public SslSimpleBuilder setProtocols(String[] protocols) {
+    this.protocols = protocols;
+    return this;
+  }
+
+  public SslSimpleBuilder setCipherSuites(String[] ciphersSuite) throws IllegalArgumentException {
+    for (String cipher : ciphersSuite) {
+      if (!OpenSsl.isCipherSuiteAvailable(cipher)) {
+        throw new IllegalArgumentException("Cipher `" + cipher + "` is not available");
+      } else {
+        logger.fine("Cipher is supported: " + cipher);
+      }
     }
 
-    public SslSimpleBuilder setHandshakeTimeoutMilliseconds(int timeout) {
-        handshakeTimeoutMilliseconds = timeout;
-        return this;
-    }
+    ciphers = ciphersSuite;
+    return this;
+  }
 
-    public SslSimpleBuilder setVerifyMode(SslClientVerifyMode mode) {
-        verifyMode = mode;
-        return this;
-    }
-
-    public File getSslKeyFile() {
-        return sslKeyFile;
-    }
+  public SslSimpleBuilder setCertificateAuthorities(String[] cert) {
+    certificateAuthorities = cert;
+    return this;
+  }
 
-    public File getSslCertificateFile() {
-        return sslCertificateFile;
-    }
+  public SslSimpleBuilder setHandshakeTimeoutMilliseconds(int timeout) {
+    handshakeTimeoutMilliseconds = timeout;
+    return this;
+  }
 
-    public SslHandler build(ByteBufAllocator bufferAllocator) throws IOException, NoSuchAlgorithmException, CertificateException {
-        SslContextBuilder builder = SslContextBuilder.forServer(sslCertificateFile, sslKeyFile, passPhrase);
+  public SslSimpleBuilder setVerifyMode(SslClientVerifyMode mode) {
+    verifyMode = mode;
+    return this;
+  }
 
-        if(logger.isDebugEnabled())
-            logger.debug("Ciphers:  " + ciphers.toString());
+  public File getSslKeyFile() {
+    return sslKeyFile;
+  }
 
-        builder.ciphers(Arrays.asList(ciphers));
+  public File getSslCertificateFile() {
+    return sslCertificateFile;
+  }
 
-        if(requireClientAuth()) {
-            if (logger.isDebugEnabled())
-                logger.debug("Certificate Authorities: " + certificateAuthorities.toString());
+  public SslHandler build(ByteBufAllocator bufferAllocator)
+      throws IOException, NoSuchAlgorithmException, CertificateException {
+    SslContextBuilder builder =
+        SslContextBuilder.forServer(sslCertificateFile, sslKeyFile, passPhrase);
 
-            builder.trustManager(loadCertificateCollection(certificateAuthorities));
-        }
+    if (logger.isLoggable(Level.FINE))
+      logger.fine(
+          "Available ciphers:" + Arrays.toString(OpenSsl.availableOpenSslCipherSuites().toArray()));
+    logger.fine("Ciphers:  " + Arrays.toString(ciphers));
 
-        SslContext context = builder.build();
-        SslHandler sslHandler = context.newHandler(bufferAllocator);
+    builder.ciphers(Arrays.asList(ciphers));
 
-        if(logger.isDebugEnabled())
-            logger.debug("TLS: " + protocols.toString());
+    if (requireClientAuth()) {
+      if (logger.isLoggable(Level.FINE))
+        logger.fine("Certificate Authorities: " + Arrays.toString(certificateAuthorities));
 
-        SSLEngine engine = sslHandler.engine();
-        engine.setEnabledProtocols(protocols);
+      builder.trustManager(loadCertificateCollection(certificateAuthorities));
+    }
 
+    SslContext context = builder.build();
+    SslHandler sslHandler = context.newHandler(bufferAllocator);
 
-        if(requireClientAuth()) {
-            // server is doing the handshake
-            engine.setUseClientMode(false);
+    if (logger.isLoggable(Level.FINE)) logger.fine("TLS: " + Arrays.toString(protocols));
 
-            if(verifyMode == SslClientVerifyMode.FORCE_PEER) {
-                // Explicitely require a client certificate
-                engine.setNeedClientAuth(true);
-            } else if(verifyMode == SslClientVerifyMode.VERIFY_PEER) {
-                // If the client supply a client certificate we will verify it.
-                engine.setWantClientAuth(true);
-            }
-        }
+    SSLEngine engine = sslHandler.engine();
+    engine.setEnabledProtocols(protocols);
 
-        sslHandler.setHandshakeTimeoutMillis(handshakeTimeoutMilliseconds);
+    if (requireClientAuth()) {
+      // server is doing the handshake
+      engine.setUseClientMode(false);
 
-        return sslHandler;
+      if (verifyMode == SslClientVerifyMode.FORCE_PEER) {
+        // Explicitely require a client certificate
+        engine.setNeedClientAuth(true);
+      } else if (verifyMode == SslClientVerifyMode.VERIFY_PEER) {
+        // If the client supply a client certificate we will verify it.
+        engine.setWantClientAuth(true);
+      }
     }
 
-    private X509Certificate[] loadCertificateCollection(String[] certificates) throws IOException, CertificateException {
-        CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509");
+    sslHandler.setHandshakeTimeoutMillis(handshakeTimeoutMilliseconds);
 
-        X509Certificate[] collections = new X509Certificate[certificates.length];
+    return sslHandler;
+  }
 
-        for(int i = 0; i < certificates.length; i++) {
-            String certificate = certificates[i];
+  private X509Certificate[] loadCertificateCollection(String[] certificates)
+      throws IOException, CertificateException {
+    logger.fine("Load certificates collection");
+    CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509");
 
-            InputStream in = null;
+    List collections = new ArrayList();
 
-            try {
-                in = new FileInputStream(certificate);
-                collections[i] = (X509Certificate) certificateFactory.generateCertificate(in);
-            } finally {
-                if(in != null) {
-                    in.close();
-                }
-            }
-        }
+    for (int i = 0; i < certificates.length; i++) {
+      String certificate = certificates[i];
 
-        return collections;
-    }
+      logger.fine("Loading certificates from file " + certificate);
 
-    private boolean requireClientAuth() {
-        if(certificateAuthorities != null) {
-            return true;
-        }
-
-        return false;
+      try (InputStream in = new FileInputStream(certificate)) {
+        List certificatesChains =
+            (List) certificateFactory.generateCertificates(in);
+        collections.addAll(certificatesChains);
+      }
     }
+    return collections.toArray(new X509Certificate[collections.size()]);
+  }
 
-    private FileInputStream createFileInputStream(String filepath) throws FileNotFoundException {
-        return new FileInputStream(filepath);
+  private boolean requireClientAuth() {
+    if (certificateAuthorities != null) {
+      return true;
     }
+
+    return false;
+  }
+
+  private FileInputStream createFileInputStream(String filepath) throws FileNotFoundException {
+    return new FileInputStream(filepath);
+  }
+
+  /**
+   * Get the supported protocols
+   *
+   * @return a defensive copy of the supported protocols
+   */
+  String[] getProtocols() {
+    return protocols.clone();
+  }
 }
diff --git a/proxy/src/main/resources/build/build.properties b/proxy/src/main/resources/build/build.properties
index 9782f6e27..76c4f5dd1 100644
--- a/proxy/src/main/resources/build/build.properties
+++ b/proxy/src/main/resources/build/build.properties
@@ -1,4 +1,5 @@
 build.version=${pom.version}
+build.package=jar
 build.commit=${build-commit}
 build.timestamp=${build-timestamp}
 build.hostname=${hostname}
diff --git a/proxy/src/test/java/com/wavefront/agent/HttpClientTest.java b/proxy/src/test/java/com/wavefront/agent/HttpClientTest.java
index 6e40e3742..118267bc6 100644
--- a/proxy/src/test/java/com/wavefront/agent/HttpClientTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/HttpClientTest.java
@@ -1,20 +1,6 @@
 package com.wavefront.agent;
 
-import org.apache.http.HttpHost;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.config.SocketConfig;
-import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.protocol.HttpContext;
-import org.jboss.resteasy.client.jaxrs.ResteasyClient;
-import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder;
-import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget;
-import org.jboss.resteasy.client.jaxrs.engines.ApacheHttpClient4Engine;
-import org.jboss.resteasy.plugins.providers.jackson.ResteasyJackson2Provider;
-import org.jboss.resteasy.spi.ResteasyProviderFactory;
-import org.junit.Test;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -22,16 +8,28 @@
 import java.net.Socket;
 import java.net.UnknownHostException;
 import java.util.concurrent.TimeUnit;
-
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.ProcessingException;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.MediaType;
-
-import static org.junit.Assert.assertTrue;
-
+import org.apache.http.HttpHost;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.SocketConfig;
+import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.protocol.HttpContext;
+import org.jboss.resteasy.client.jaxrs.ResteasyClient;
+import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget;
+import org.jboss.resteasy.client.jaxrs.engines.ApacheHttpClient43Engine;
+import org.jboss.resteasy.client.jaxrs.internal.LocalResteasyProviderFactory;
+import org.jboss.resteasy.client.jaxrs.internal.ResteasyClientBuilderImpl;
+import org.jboss.resteasy.plugins.providers.jackson.ResteasyJackson2Provider;
+import org.jboss.resteasy.spi.ResteasyProviderFactory;
+import org.junit.Test;
 
 public final class HttpClientTest {
 
@@ -64,53 +62,66 @@ public void run() {
     }
   }
 
-  @Test(expected=ProcessingException.class)
+  @Test(expected = ProcessingException.class)
   public void httpClientTimeoutsWork() throws Exception {
-    ResteasyProviderFactory factory = ResteasyProviderFactory.getInstance();
+    ResteasyProviderFactory factory =
+        new LocalResteasyProviderFactory(ResteasyProviderFactory.getInstance());
     factory.registerProvider(JsonNodeWriter.class);
     factory.registerProvider(ResteasyJackson2Provider.class);
 
-    HttpClient httpClient = HttpClientBuilder.create().
-        useSystemProperties().
-        setMaxConnTotal(200).
-        setMaxConnPerRoute(100).
-        setConnectionTimeToLive(1, TimeUnit.MINUTES).
-        setDefaultSocketConfig(
-            SocketConfig.custom().
-                setSoTimeout(100).build()).
-        setDefaultRequestConfig(
-            RequestConfig.custom().
-                setContentCompressionEnabled(true).
-                setRedirectsEnabled(true).
-                setConnectTimeout(5000).
-                setConnectionRequestTimeout(5000).
-                setSocketTimeout(60000).build()).
-        setSSLSocketFactory(
-            new LayeredConnectionSocketFactory() {
-              @Override
-              public Socket createLayeredSocket(Socket socket, String target, int port, HttpContext context)
-                  throws IOException, UnknownHostException {
-                return SSLConnectionSocketFactory.getSystemSocketFactory()
-                    .createLayeredSocket(socket, target, port, context);
-              }
-
-              @Override
-              public Socket createSocket(HttpContext context) throws IOException {
-                return SSLConnectionSocketFactory.getSystemSocketFactory()
-                    .createSocket(context);
-              }
-
-              @Override
-              public Socket connectSocket(int connectTimeout, Socket sock, HttpHost host, InetSocketAddress remoteAddress, InetSocketAddress localAddress, HttpContext context) throws IOException {
-                assertTrue("Non-zero timeout passed to connect socket is expected", connectTimeout > 0);
-                throw new ProcessingException("OK");
-              }
-            }).build();
-
-    ResteasyClient client = new ResteasyClientBuilder().
-        httpEngine(new ApacheHttpClient4Engine(httpClient, true)).
-        providerFactory(factory).
-        build();
+    HttpClient httpClient =
+        HttpClientBuilder.create()
+            .useSystemProperties()
+            .setMaxConnTotal(200)
+            .setMaxConnPerRoute(100)
+            .setConnectionTimeToLive(1, TimeUnit.MINUTES)
+            .setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(100).build())
+            .setDefaultRequestConfig(
+                RequestConfig.custom()
+                    .setContentCompressionEnabled(true)
+                    .setRedirectsEnabled(true)
+                    .setConnectTimeout(5000)
+                    .setConnectionRequestTimeout(5000)
+                    .setSocketTimeout(60000)
+                    .build())
+            .setSSLSocketFactory(
+                new LayeredConnectionSocketFactory() {
+                  @Override
+                  public Socket createLayeredSocket(
+                      Socket socket, String target, int port, HttpContext context)
+                      throws IOException, UnknownHostException {
+                    return SSLConnectionSocketFactory.getSystemSocketFactory()
+                        .createLayeredSocket(socket, target, port, context);
+                  }
+
+                  @Override
+                  public Socket createSocket(HttpContext context) throws IOException {
+                    return SSLConnectionSocketFactory.getSystemSocketFactory()
+                        .createSocket(context);
+                  }
+
+                  @Override
+                  public Socket connectSocket(
+                      int connectTimeout,
+                      Socket sock,
+                      HttpHost host,
+                      InetSocketAddress remoteAddress,
+                      InetSocketAddress localAddress,
+                      HttpContext context)
+                      throws IOException {
+                    assertTrue(
+                        "Non-zero timeout passed to connect socket is expected",
+                        connectTimeout > 0);
+                    throw new ProcessingException("OK");
+                  }
+                })
+            .build();
+
+    ResteasyClient client =
+        new ResteasyClientBuilderImpl()
+            .httpEngine(new ApacheHttpClient43Engine(httpClient, true))
+            .providerFactory(factory)
+            .build();
 
     SocketServerRunnable sr = new SocketServerRunnable();
     Thread serverThread = new Thread(sr);
@@ -119,7 +130,5 @@ public Socket connectSocket(int connectTimeout, Socket sock, HttpHost host, Inet
     ResteasyWebTarget target = client.target("https://localhost:" + sr.getPort());
     SimpleRESTEasyAPI proxy = target.proxy(SimpleRESTEasyAPI.class);
     proxy.search("resteasy");
-
   }
-
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/HttpEndToEndTest.java b/proxy/src/test/java/com/wavefront/agent/HttpEndToEndTest.java
new file mode 100644
index 000000000..fbdd9b439
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/HttpEndToEndTest.java
@@ -0,0 +1,939 @@
+package com.wavefront.agent;
+
+import static com.wavefront.agent.ProxyUtil.createInitializer;
+import static com.wavefront.agent.TestUtils.assertTrueWithTimeout;
+import static com.wavefront.agent.TestUtils.findAvailablePort;
+import static com.wavefront.agent.TestUtils.gzippedHttpPost;
+import static com.wavefront.agent.TestUtils.waitUntilListenerIsOnline;
+import static com.wavefront.agent.channel.ChannelUtils.makeResponse;
+import static com.wavefront.agent.channel.ChannelUtils.writeHttpResponse;
+import static com.wavefront.api.agent.Constants.PUSH_FORMAT_LOGS_JSON_ARR;
+import static com.wavefront.api.agent.Constants.PUSH_FORMAT_LOGS_JSON_LINES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.ImmutableSet;
+import com.wavefront.agent.auth.TokenAuthenticator;
+import com.wavefront.agent.channel.HealthCheckManager;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.SenderTaskFactoryImpl;
+import com.wavefront.agent.listeners.AbstractHttpOnlyHandler;
+import com.wavefront.agent.queueing.QueueingFactoryImpl;
+import com.wavefront.common.Clock;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.ingester.TcpIngester;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.util.CharsetUtil;
+import java.io.File;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/** @author vasily@wavefront.com */
+public class HttpEndToEndTest {
+  private static final Logger logger = Logger.getLogger("test");
+
+  private PushAgent proxy;
+  private MutableFunc server = new MutableFunc<>(x -> null);
+  private Thread thread;
+  private int backendPort;
+  private int proxyPort;
+
+  @Before
+  public void setup() throws Exception {
+    backendPort = findAvailablePort(8081);
+    ChannelHandler channelHandler =
+        new WrappingHttpHandler(null, null, String.valueOf(backendPort), server);
+    thread =
+        new Thread(
+            new TcpIngester(
+                createInitializer(
+                    channelHandler, backendPort, 32768, 16 * 1024 * 1024, 5, null, null),
+                backendPort));
+    thread.start();
+    waitUntilListenerIsOnline(backendPort);
+  }
+
+  @After
+  public void teardown() {
+    TokenManager.reset();
+    thread.interrupt();
+    proxy.stopListener(proxyPort);
+    proxy.shutdown();
+  }
+
+  @Test
+  public void testEndToEndMetrics() throws Exception {
+    AtomicInteger successfulSteps = new AtomicInteger(0);
+    AtomicInteger testCounter = new AtomicInteger(0);
+    long time = Clock.now() / 1000;
+    proxyPort = findAvailablePort(2898);
+    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
+    proxy = new PushAgent();
+    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
+    proxy.proxyConfig.token = UUID.randomUUID().toString();
+    proxy.proxyConfig.flushThreads = 1;
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(proxyPort);
+    proxy.proxyConfig.pushFlushInterval = 50;
+    proxy.proxyConfig.bufferFile = buffer;
+    proxy.proxyConfig.allowRegex = "^.*$";
+    proxy.proxyConfig.blockRegex = "^.*blocklist.*$";
+    proxy.proxyConfig.gzipCompression = false;
+    proxy.start(new String[] {});
+    waitUntilListenerIsOnline(proxyPort);
+    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl)) fail();
+    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl)) fail();
+
+    String payload =
+        "metric.name 1 "
+            + time
+            + " source=metric.source tagk1=tagv1\n"
+            + "metric.name 2 "
+            + time
+            + " source=metric.source tagk1=tagv2\n"
+            + "metric.name 3 "
+            + time
+            + " source=metric.source tagk1=tagv3\n"
+            + "metric.name 4 "
+            + time
+            + " source=metric.source tagk1=tagv4\n";
+    String expectedTest1part1 =
+        "\"metric.name\" 1.0 "
+            + time
+            + " source=\"metric.source\" \"tagk1\"=\"tagv1\"\n"
+            + "\"metric.name\" 2.0 "
+            + time
+            + " source=\"metric.source\" \"tagk1\"=\"tagv2\"";
+    String expectedTest1part2 =
+        "\"metric.name\" 3.0 "
+            + time
+            + " source=\"metric.source\" \"tagk1\"=\"tagv3\"\n"
+            + "\"metric.name\" 4.0 "
+            + time
+            + " source=\"metric.source\" \"tagk1\"=\"tagv4\"";
+
+    server.update(
+        req -> {
+          String content = req.content().toString(CharsetUtil.UTF_8);
+          logger.fine("Content received: " + content);
+          assertEquals(expectedTest1part1 + "\n" + expectedTest1part2, content);
+          successfulSteps.incrementAndGet();
+          return makeResponse(HttpResponseStatus.OK, "");
+        });
+    gzippedHttpPost("http://localhost:" + proxyPort + "/", payload);
+    HandlerKey key = HandlerKey.of(ReportableEntityType.POINT, String.valueOf(proxyPort));
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
+    assertEquals(1, successfulSteps.getAndSet(0));
+    AtomicBoolean part1 = new AtomicBoolean(false);
+    AtomicBoolean part2 = new AtomicBoolean(false);
+    server.update(
+        req -> {
+          String content = req.content().toString(CharsetUtil.UTF_8);
+          logger.fine("Content received: " + content);
+          switch (testCounter.incrementAndGet()) {
+            case 1:
+              assertEquals(expectedTest1part1 + "\n" + expectedTest1part2, content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.TOO_MANY_REQUESTS, "");
+            case 2:
+              assertEquals(expectedTest1part1 + "\n" + expectedTest1part2, content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+            case 3:
+              assertEquals(expectedTest1part1 + "\n" + expectedTest1part2, content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.valueOf(407), "");
+            case 4:
+              assertEquals(expectedTest1part1 + "\n" + expectedTest1part2, content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, "");
+            case 5:
+            case 6:
+              if (content.equals(expectedTest1part1)) part1.set(true);
+              if (content.equals(expectedTest1part2)) part2.set(true);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+          }
+          throw new IllegalStateException();
+        });
+    gzippedHttpPost("http://localhost:" + proxyPort + "/", payload);
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
+    ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
+    gzippedHttpPost("http://localhost:" + proxyPort + "/", payload);
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
+    for (int i = 0; i < 3; i++) ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
+    assertEquals(6, successfulSteps.getAndSet(0));
+    assertTrue(part1.get());
+    assertTrue(part2.get());
+  }
+
+  @Test
+  public void testEndToEndEvents() throws Exception {
+    AtomicInteger successfulSteps = new AtomicInteger(0);
+    AtomicInteger testCounter = new AtomicInteger(0);
+    long time = Clock.now() / 1000;
+    proxyPort = findAvailablePort(2898);
+    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
+    proxy = new PushAgent();
+    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
+    proxy.proxyConfig.token = UUID.randomUUID().toString();
+    proxy.proxyConfig.flushThreads = 1;
+    proxy.proxyConfig.flushThreadsEvents = 1;
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(proxyPort);
+    proxy.proxyConfig.pushFlushInterval = 10000;
+    proxy.proxyConfig.pushRateLimitEvents = 100;
+    proxy.proxyConfig.bufferFile = buffer;
+    proxy.start(new String[] {});
+    waitUntilListenerIsOnline(proxyPort);
+    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl)) fail();
+    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl)) fail();
+
+    String payloadEvents =
+        "@Event "
+            + time
+            + " \"Event name for testing\" host=host1 host=host2 tag=tag1 "
+            + "severity=INFO multi=bar multi=baz\n"
+            + "@Event "
+            + time
+            + " \"Another test event\" host=host3";
+    String expectedEvent1 =
+        "{\"name\":\"Event name for testing\",\"startTime\":"
+            + (time * 1000)
+            + ",\"endTime\":"
+            + (time * 1000 + 1)
+            + ",\"annotations\":{\"severity\":\"INFO\"},"
+            + "\"dimensions\":{\"multi\":[\"bar\",\"baz\"]},\"hosts\":[\"host1\",\"host2\"],"
+            + "\"tags\":[\"tag1\"]}";
+    String expectedEvent2 =
+        "{\"name\":\"Another test event\",\"startTime\":"
+            + (time * 1000)
+            + ",\"endTime\":"
+            + (time * 1000 + 1)
+            + ",\"annotations\":{},\"dimensions\":null,"
+            + "\"hosts\":[\"host3\"],\"tags\":null}";
+    server.update(
+        req -> {
+          String content = req.content().toString(CharsetUtil.UTF_8);
+          URI uri;
+          try {
+            uri = new URI(req.uri());
+          } catch (Exception e) {
+            throw new RuntimeException(e);
+          }
+          String path = uri.getPath();
+          logger.fine("Content received: " + content);
+          assertEquals(HttpMethod.POST, req.method());
+          assertEquals("/api/v2/wfproxy/event", path);
+          switch (testCounter.incrementAndGet()) {
+            case 1:
+              assertEquals("[" + expectedEvent1 + "," + expectedEvent2 + "]", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, "");
+            case 2:
+              assertEquals("[" + expectedEvent1 + "]", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+            case 3:
+              assertEquals("[" + expectedEvent2 + "]", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+            case 4:
+              assertEquals("[" + expectedEvent1 + "," + expectedEvent2 + "]", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.valueOf(407), "");
+            case 5:
+              assertEquals("[" + expectedEvent1 + "," + expectedEvent2 + "]", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.INTERNAL_SERVER_ERROR, "");
+            case 6:
+              assertEquals("[" + expectedEvent1 + "," + expectedEvent2 + "]", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+          }
+          logger.warning("Too many requests");
+          successfulSteps.incrementAndGet(); // this will force the assert to fail
+          return makeResponse(HttpResponseStatus.OK, "");
+        });
+    gzippedHttpPost("http://localhost:" + proxyPort + "/", payloadEvents);
+    HandlerKey key = HandlerKey.of(ReportableEntityType.EVENT, String.valueOf(proxyPort));
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
+    ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
+    gzippedHttpPost("http://localhost:" + proxyPort + "/", payloadEvents);
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
+    for (int i = 0; i < 2; i++) ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
+    assertEquals(6, successfulSteps.getAndSet(0));
+  }
+
+  @Test
+  public void testEndToEndSourceTags() throws Exception {
+    AtomicInteger successfulSteps = new AtomicInteger(0);
+    AtomicInteger testCounter = new AtomicInteger(0);
+    proxyPort = findAvailablePort(2898);
+    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
+    proxy = new PushAgent();
+    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
+    proxy.proxyConfig.token = UUID.randomUUID().toString();
+    proxy.proxyConfig.flushThreads = 1;
+    proxy.proxyConfig.flushThreadsSourceTags = 1;
+    proxy.proxyConfig.splitPushWhenRateLimited = true;
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(proxyPort);
+    proxy.proxyConfig.pushFlushInterval = 10000;
+    proxy.proxyConfig.pushRateLimitSourceTags = 100;
+    proxy.proxyConfig.bufferFile = buffer;
+    proxy.start(new String[] {});
+    waitUntilListenerIsOnline(proxyPort);
+    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl)) fail();
+    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl)) fail();
+
+    String payloadSourceTags =
+        "@SourceTag action=add source=testSource addTag1 addTag2 addTag3\n"
+            + "@SourceTag action=save source=testSource newtag1 newtag2\n"
+            + "@SourceTag action=delete source=testSource deleteTag\n"
+            + "@SourceDescription action=save source=testSource \"Long Description\"\n"
+            + "@SourceDescription action=delete source=testSource";
+
+    server.update(
+        req -> {
+          String content = req.content().toString(CharsetUtil.UTF_8);
+          URI uri;
+          try {
+            uri = new URI(req.uri());
+          } catch (Exception e) {
+            throw new RuntimeException(e);
+          }
+          String path = uri.getPath();
+          logger.fine("Content received: " + content);
+          switch (testCounter.incrementAndGet()) {
+            case 1:
+              assertEquals(HttpMethod.PUT, req.method());
+              assertEquals("/api/v2/source/testSource/tag/addTag1", path);
+              assertEquals("", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+            case 2:
+              assertEquals(HttpMethod.PUT, req.method());
+              assertEquals("/api/v2/source/testSource/tag/addTag2", path);
+              assertEquals("", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+            case 3:
+              assertEquals(HttpMethod.PUT, req.method());
+              assertEquals("/api/v2/source/testSource/tag/addTag3", path);
+              assertEquals("", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+            case 4:
+              assertEquals(HttpMethod.POST, req.method());
+              assertEquals("/api/v2/source/testSource/tag", path);
+              assertEquals("[\"newtag1\",\"newtag2\"]", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, "");
+            case 5:
+              assertEquals(HttpMethod.DELETE, req.method());
+              assertEquals("/api/v2/source/testSource/tag/deleteTag", path);
+              assertEquals("", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+            case 6:
+              assertEquals(HttpMethod.POST, req.method());
+              assertEquals("/api/v2/source/testSource/description", path);
+              assertEquals("Long Description", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.INTERNAL_SERVER_ERROR, "");
+            case 7:
+              assertEquals(HttpMethod.DELETE, req.method());
+              assertEquals("/api/v2/source/testSource/description", path);
+              assertEquals("", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.valueOf(407), "");
+            case 8:
+              assertEquals(HttpMethod.POST, req.method());
+              assertEquals("/api/v2/source/testSource/tag", path);
+              assertEquals("[\"newtag1\",\"newtag2\"]", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+            case 9:
+              assertEquals(HttpMethod.POST, req.method());
+              assertEquals("/api/v2/source/testSource/description", path);
+              assertEquals("Long Description", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+            case 10:
+              assertEquals(HttpMethod.DELETE, req.method());
+              assertEquals("/api/v2/source/testSource/description", path);
+              assertEquals("", content);
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+          }
+          logger.warning("Too many requests");
+          successfulSteps.incrementAndGet(); // this will force the assert to fail
+          return makeResponse(HttpResponseStatus.OK, "");
+        });
+    gzippedHttpPost("http://localhost:" + proxyPort + "/", payloadSourceTags);
+    HandlerKey key = HandlerKey.of(ReportableEntityType.SOURCE_TAG, String.valueOf(proxyPort));
+    for (int i = 0; i < 2; i++) ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
+    for (int i = 0; i < 4; i++) ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
+    assertEquals(10, successfulSteps.getAndSet(0));
+  }
+
+  @Test
+  public void testEndToEndHistograms() throws Exception {
+    AtomicInteger successfulSteps = new AtomicInteger(0);
+    AtomicInteger testCounter = new AtomicInteger(0);
+    long time = (Clock.now() / 1000) / 60 * 60 + 30;
+    AtomicLong digestTime = new AtomicLong(System.currentTimeMillis());
+    proxyPort = findAvailablePort(2898);
+    int histMinPort = findAvailablePort(40001);
+    int histHourPort = findAvailablePort(40002);
+    int histDayPort = findAvailablePort(40003);
+    int histDistPort = findAvailablePort(40000);
+    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
+    proxy = new PushAgent();
+    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
+    proxy.proxyConfig.token = UUID.randomUUID().toString();
+    proxy.proxyConfig.flushThreads = 1;
+    proxy.proxyConfig.histogramMinuteListenerPorts = String.valueOf(histMinPort);
+    proxy.proxyConfig.histogramHourListenerPorts = String.valueOf(histHourPort);
+    proxy.proxyConfig.histogramDayListenerPorts = String.valueOf(histDayPort);
+    proxy.proxyConfig.histogramDistListenerPorts = String.valueOf(histDistPort);
+    proxy.proxyConfig.histogramMinuteAccumulatorPersisted = false;
+    proxy.proxyConfig.histogramHourAccumulatorPersisted = false;
+    proxy.proxyConfig.histogramDayAccumulatorPersisted = false;
+    proxy.proxyConfig.histogramDistAccumulatorPersisted = false;
+    proxy.proxyConfig.histogramMinuteMemoryCache = false;
+    proxy.proxyConfig.histogramHourMemoryCache = false;
+    proxy.proxyConfig.histogramDayMemoryCache = false;
+    proxy.proxyConfig.histogramDistMemoryCache = false;
+    proxy.proxyConfig.histogramMinuteFlushSecs = 1;
+    proxy.proxyConfig.histogramHourFlushSecs = 1;
+    proxy.proxyConfig.histogramDayFlushSecs = 1;
+    proxy.proxyConfig.histogramDistFlushSecs = 1;
+    proxy.proxyConfig.histogramMinuteAccumulatorSize = 10L;
+    proxy.proxyConfig.histogramHourAccumulatorSize = 10L;
+    proxy.proxyConfig.histogramDayAccumulatorSize = 10L;
+    proxy.proxyConfig.histogramDistAccumulatorSize = 10L;
+    proxy.proxyConfig.histogramAccumulatorFlushInterval = 10000L;
+    proxy.proxyConfig.histogramAccumulatorResolveInterval = 10000L;
+    proxy.proxyConfig.splitPushWhenRateLimited = true;
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(proxyPort);
+    proxy.proxyConfig.pushFlushInterval = 10000;
+    proxy.proxyConfig.bufferFile = buffer;
+    proxy.proxyConfig.timeProvider = digestTime::get;
+    proxy.start(new String[] {});
+    waitUntilListenerIsOnline(histDistPort);
+    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl)) fail();
+    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl)) fail();
+
+    String payloadHistograms =
+        "metric.name 1 "
+            + time
+            + " source=metric.source tagk1=tagv1\n"
+            + "metric.name 1 "
+            + time
+            + " source=metric.source tagk1=tagv1\n"
+            + "metric.name 2 "
+            + (time + 1)
+            + " source=metric.source tagk1=tagv1\n"
+            + "metric.name 2 "
+            + (time + 2)
+            + " source=metric.source tagk1=tagv1\n"
+            + "metric.name 3 "
+            + time
+            + " source=metric.source tagk1=tagv2\n"
+            + "metric.name 4 "
+            + time
+            + " source=metric.source tagk1=tagv2\n"
+            + "metric.name 5 "
+            + (time + 60)
+            + " source=metric.source tagk1=tagv1\n"
+            + "metric.name 6 "
+            + (time + 60)
+            + " source=metric.source tagk1=tagv1\n";
+
+    long minuteBin = time / 60 * 60;
+    long hourBin = time / 3600 * 3600;
+    long dayBin = time / 86400 * 86400;
+    Set expectedHistograms =
+        ImmutableSet.of(
+            "!M "
+                + minuteBin
+                + " #2 1.0 #2 2.0 \"metric.name\" "
+                + "source=\"metric.source\" \"tagk1\"=\"tagv1\"",
+            "!M "
+                + minuteBin
+                + " #1 3.0 #1 4.0 \"metric.name\" source=\"metric.source\" "
+                + "\"tagk1\"=\"tagv2\"",
+            "!M "
+                + (minuteBin + 60)
+                + " #1 5.0 #1 6.0 \"metric.name\" source=\"metric.source\" "
+                + "\"tagk1\"=\"tagv1\"",
+            "!H "
+                + hourBin
+                + " #2 1.0 #2 2.0 #1 5.0 #1 6.0 \"metric.name\" "
+                + "source=\"metric.source\" \"tagk1\"=\"tagv1\"",
+            "!H "
+                + hourBin
+                + " #1 3.0 #1 4.0 \"metric.name\" source=\"metric.source\" "
+                + "\"tagk1\"=\"tagv2\"",
+            "!D "
+                + dayBin
+                + " #1 3.0 #1 4.0 \"metric.name\" source=\"metric.source\" "
+                + "\"tagk1\"=\"tagv2\"",
+            "!D "
+                + dayBin
+                + " #2 1.0 #2 2.0 #1 5.0 #1 6.0 \"metric.name\" "
+                + "source=\"metric.source\" \"tagk1\"=\"tagv1\"");
+
+    String distPayload =
+        "!M "
+            + minuteBin
+            + " #1 1.0 #1 1.0 #1 2.0 #1 2.0 \"metric.name\" "
+            + "source=\"metric.source\" \"tagk1\"=\"tagv1\"\n"
+            + "!M "
+            + minuteBin
+            + " #1 1.0 #1 1.0 #1 2.0 #1 2.0 \"metric.name\" "
+            + "source=\"metric.source\" \"tagk1\"=\"tagv1\"\n"
+            + "!H "
+            + minuteBin
+            + " #1 1.0 #1 1.0 #1 2.0 #1 2.0 \"metric.name\" "
+            + "source=\"metric.source\" \"tagk1\"=\"tagv1\"\n";
+
+    Set expectedDists =
+        ImmutableSet.of(
+            "!M "
+                + minuteBin
+                + " #4 1.0 #4 2.0 "
+                + "\"metric.name\" source=\"metric.source\" \"tagk1\"=\"tagv1\"",
+            "!H "
+                + hourBin
+                + " #2 1.0 #2 2.0 \"metric.name\" "
+                + "source=\"metric.source\" \"tagk1\"=\"tagv1\"");
+
+    server.update(
+        req -> {
+          String content = req.content().toString(CharsetUtil.UTF_8);
+          URI uri;
+          try {
+            uri = new URI(req.uri());
+          } catch (Exception e) {
+            throw new RuntimeException(e);
+          }
+          String path = uri.getPath();
+          assertEquals(HttpMethod.POST, req.method());
+          assertEquals("/api/v2/wfproxy/report", path);
+          logger.fine("Content received: " + content);
+          switch (testCounter.incrementAndGet()) {
+            case 1:
+              assertEquals(expectedHistograms, new HashSet<>(Arrays.asList(content.split("\n"))));
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+            case 2:
+              assertEquals(expectedDists, new HashSet<>(Arrays.asList(content.split("\n"))));
+              successfulSteps.incrementAndGet();
+              return makeResponse(HttpResponseStatus.OK, "");
+          }
+          return makeResponse(HttpResponseStatus.OK, "");
+        });
+    digestTime.set(System.currentTimeMillis() - 1001);
+    gzippedHttpPost("http://localhost:" + histMinPort + "/", payloadHistograms);
+    gzippedHttpPost("http://localhost:" + histHourPort + "/", payloadHistograms);
+    gzippedHttpPost("http://localhost:" + histDayPort + "/", payloadHistograms);
+    gzippedHttpPost("http://localhost:" + histDistPort + "/", payloadHistograms); // should reject
+    digestTime.set(System.currentTimeMillis());
+    proxy.histogramFlushRunnables.forEach(Runnable::run);
+    HandlerKey key = HandlerKey.of(ReportableEntityType.HISTOGRAM, "histogram_ports");
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
+
+    digestTime.set(System.currentTimeMillis() - 1001);
+    gzippedHttpPost("http://localhost:" + histDistPort + "/", distPayload);
+    digestTime.set(System.currentTimeMillis());
+    proxy.histogramFlushRunnables.forEach(Runnable::run);
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
+    assertEquals(2, successfulSteps.getAndSet(0));
+  }
+
+  @Test
+  public void testEndToEndSpans() throws Exception {
+    long time = Clock.now() / 1000;
+    proxyPort = findAvailablePort(2898);
+    proxyPort = findAvailablePort(2898);
+    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
+    proxy = new PushAgent();
+    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
+    proxy.proxyConfig.token = UUID.randomUUID().toString();
+    proxy.proxyConfig.flushThreads = 1;
+    proxy.proxyConfig.traceListenerPorts = String.valueOf(proxyPort);
+    proxy.proxyConfig.pushFlushInterval = 50;
+    proxy.proxyConfig.bufferFile = buffer;
+    proxy.proxyConfig.trafficShaping = true;
+    proxy.start(new String[] {});
+    waitUntilListenerIsOnline(proxyPort);
+    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl)) fail();
+    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl)) fail();
+
+    String traceId = UUID.randomUUID().toString();
+    long timestamp1 = time * 1000000 + 12345;
+    long timestamp2 = time * 1000000 + 23456;
+    String payload =
+        "testSpanName parent=parent1 source=testsource spanId=testspanid "
+            + "traceId=\""
+            + traceId
+            + "\" parent=parent2 "
+            + time
+            + " "
+            + (time + 1)
+            + "\n"
+            + "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}]}\n";
+    String expectedSpan =
+        "\"testSpanName\" source=\"testsource\" spanId=\"testspanid\" "
+            + "traceId=\""
+            + traceId
+            + "\" \"parent\"=\"parent1\" \"parent\"=\"parent2\" "
+            + (time * 1000)
+            + " 1000";
+    String expectedSpanLog =
+        "{\"customer\":\"dummy\",\"traceId\":\""
+            + traceId
+            + "\",\"spanId"
+            + "\":\"testspanid\",\"spanSecondaryId\":null,\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ","
+            + "\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ","
+            + "\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}],\"span\":\"_sampledByPolicy=NONE\"}";
+    AtomicBoolean gotSpan = new AtomicBoolean(false);
+    AtomicBoolean gotSpanLog = new AtomicBoolean(false);
+    server.update(
+        req -> {
+          String content = req.content().toString(CharsetUtil.UTF_8);
+          logger.fine("Content received: " + content);
+          if (content.equals(expectedSpan)) gotSpan.set(true);
+          if (content.equals(expectedSpanLog)) gotSpanLog.set(true);
+          return makeResponse(HttpResponseStatus.OK, "");
+        });
+    gzippedHttpPost("http://localhost:" + proxyPort + "/", payload);
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory)
+        .flushNow(HandlerKey.of(ReportableEntityType.TRACE, String.valueOf(proxyPort)));
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory)
+        .flushNow(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, String.valueOf(proxyPort)));
+    assertTrueWithTimeout(50, gotSpan::get);
+    assertTrueWithTimeout(50, gotSpanLog::get);
+  }
+
+  @Test
+  public void testEndToEndSpans_SpanLogsWithSpanField() throws Exception {
+    long time = Clock.now() / 1000;
+    proxyPort = findAvailablePort(2898);
+    proxyPort = findAvailablePort(2898);
+    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
+    proxy = new PushAgent();
+    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
+    proxy.proxyConfig.token = UUID.randomUUID().toString();
+    proxy.proxyConfig.flushThreads = 1;
+    proxy.proxyConfig.traceListenerPorts = String.valueOf(proxyPort);
+    proxy.proxyConfig.pushFlushInterval = 50;
+    proxy.proxyConfig.bufferFile = buffer;
+    proxy.start(new String[] {});
+    waitUntilListenerIsOnline(proxyPort);
+    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl)) fail();
+    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl)) fail();
+
+    String traceId = UUID.randomUUID().toString();
+    long timestamp1 = time * 1000000 + 12345;
+    long timestamp2 = time * 1000000 + 23456;
+    String payload =
+        "testSpanName parent=parent1 source=testsource spanId=testspanid "
+            + "traceId=\""
+            + traceId
+            + "\" parent=parent2 "
+            + time
+            + " "
+            + (time + 1)
+            + "\n"
+            + "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}],\"span\":\""
+            + "testSpanName parent=parent1 source=testsource spanId=testspanid traceId=\\\""
+            + traceId
+            + "\\\" parent=parent2 "
+            + time
+            + " "
+            + (time + 1)
+            + "\\n\"}\n";
+    String expectedSpan =
+        "\"testSpanName\" source=\"testsource\" spanId=\"testspanid\" "
+            + "traceId=\""
+            + traceId
+            + "\" \"parent\"=\"parent1\" \"parent\"=\"parent2\" "
+            + (time * 1000)
+            + " 1000";
+    String expectedSpanLog =
+        "{\"customer\":\"dummy\",\"traceId\":\""
+            + traceId
+            + "\",\"spanId"
+            + "\":\"testspanid\",\"spanSecondaryId\":null,\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ","
+            + "\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ","
+            + "\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}],\"span\":\"_sampledByPolicy=NONE\"}";
+    AtomicBoolean gotSpan = new AtomicBoolean(false);
+    AtomicBoolean gotSpanLog = new AtomicBoolean(false);
+    server.update(
+        req -> {
+          String content = req.content().toString(CharsetUtil.UTF_8);
+          logger.fine("Content received: " + content);
+          if (content.equals(expectedSpan)) gotSpan.set(true);
+          if (content.equals(expectedSpanLog)) gotSpanLog.set(true);
+          return makeResponse(HttpResponseStatus.OK, "");
+        });
+    gzippedHttpPost("http://localhost:" + proxyPort + "/", payload);
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory)
+        .flushNow(HandlerKey.of(ReportableEntityType.TRACE, String.valueOf(proxyPort)));
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory)
+        .flushNow(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, String.valueOf(proxyPort)));
+    assertTrueWithTimeout(50, gotSpan::get);
+    assertTrueWithTimeout(50, gotSpanLog::get);
+  }
+
+  @Test
+  public void testEndToEndLogArray() throws Exception {
+    long time = Clock.now() / 1000;
+    proxyPort = findAvailablePort(2898);
+    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
+    proxy = new PushAgent();
+    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
+    proxy.proxyConfig.token = UUID.randomUUID().toString();
+    proxy.proxyConfig.flushThreads = 1;
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(proxyPort);
+    proxy.proxyConfig.bufferFile = buffer;
+    proxy.proxyConfig.pushRateLimitLogs = 1024;
+    proxy.proxyConfig.pushFlushIntervalLogs = 50;
+
+    proxy.start(new String[] {});
+    waitUntilListenerIsOnline(proxyPort);
+    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl)) fail();
+    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl)) fail();
+
+    long timestamp = time * 1000 + 12345;
+    String payload = "[{\"source\": \"myHost\",\n \"timestamp\": \"" + timestamp + "\"" + "}]";
+    String expectedLog =
+        "[{\"source\":\"myHost\",\"timestamp\":" + timestamp + ",\"text\":\"\"" + "}]";
+    AtomicBoolean gotLog = new AtomicBoolean(false);
+    server.update(
+        req -> {
+          String content = req.content().toString(CharsetUtil.UTF_8);
+          logger.fine("Content received: " + content);
+          if (content.equals(expectedLog)) gotLog.set(true);
+          return makeResponse(HttpResponseStatus.OK, "");
+        });
+    gzippedHttpPost("http://localhost:" + proxyPort + "/?f=" + PUSH_FORMAT_LOGS_JSON_ARR, payload);
+    HandlerKey key = HandlerKey.of(ReportableEntityType.LOGS, String.valueOf(proxyPort));
+    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
+    ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
+    assertTrueWithTimeout(50, gotLog::get);
+  }
+
+  @Ignore
+  @Test
+  public void testEndToEndLogLines() throws Exception {
+    long time = Clock.now() / 1000;
+    proxyPort = findAvailablePort(2898);
+    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
+    proxy = new PushAgent();
+    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
+    proxy.proxyConfig.token = UUID.randomUUID().toString();
+    proxy.proxyConfig.flushThreads = 1;
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(proxyPort);
+    proxy.proxyConfig.bufferFile = buffer;
+    proxy.proxyConfig.pushRateLimitLogs = 1024;
+    proxy.proxyConfig.pushFlushIntervalLogs = 50;
+
+    proxy.start(new String[] {});
+    waitUntilListenerIsOnline(proxyPort);
+    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl)) fail();
+    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl)) fail();
+
+    long timestamp = time * 1000 + 12345;
+    String payload =
+        "{\"source\": \"myHost1\",\n \"timestamp\": \""
+            + timestamp
+            + "\""
+            + "}\n{\"source\": \"myHost2\",\n \"timestamp\": \""
+            + timestamp
+            + "\""
+            + "}";
+    String expectedLog1 =
+        "[{\"source\":\"myHost1\",\"timestamp\":" + timestamp + ",\"text\":\"\"" + "}]";
+    String expectedLog2 =
+        "[{\"source\":\"myHost2\",\"timestamp\":" + timestamp + ",\"text\":\"\"" + "}]";
+    AtomicBoolean gotLog = new AtomicBoolean(false);
+    Set actualLogs = new HashSet<>();
+    server.update(
+        req -> {
+          String content = req.content().toString(CharsetUtil.UTF_8);
+          logger.fine("Content received: " + content);
+          actualLogs.add(content);
+          return makeResponse(HttpResponseStatus.OK, "");
+        });
+    gzippedHttpPost(
+        "http://localhost:" + proxyPort + "/?f=" + PUSH_FORMAT_LOGS_JSON_LINES, payload);
+    HandlerKey key = HandlerKey.of(ReportableEntityType.LOGS, String.valueOf(proxyPort));
+    proxy.senderTaskFactory.flushNow(key);
+    ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
+    assertEquals(2, actualLogs.size());
+    if (actualLogs.contains(expectedLog1) && actualLogs.contains(expectedLog2)) gotLog.set(true);
+    assertTrueWithTimeout(50, gotLog::get);
+  }
+
+  @Ignore
+  @Test
+  public void testEndToEndLogCloudwatch() throws Exception {
+    long time = Clock.now() / 1000;
+    proxyPort = findAvailablePort(2898);
+    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
+    proxy = new PushAgent();
+    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
+    proxy.proxyConfig.token = UUID.randomUUID().toString();
+    proxy.proxyConfig.flushThreads = 1;
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(proxyPort);
+    proxy.proxyConfig.bufferFile = buffer;
+    proxy.proxyConfig.pushRateLimitLogs = 1024;
+    proxy.proxyConfig.pushFlushIntervalLogs = 50;
+
+    proxy.start(new String[] {});
+    waitUntilListenerIsOnline(proxyPort);
+    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl)) fail();
+    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl)) fail();
+
+    long timestamp = time * 1000 + 12345;
+    String payload =
+        "{\"someKey\": \"someVal\", "
+            + "\"logEvents\": [{\"source\": \"myHost1\", \"timestamp\": \""
+            + timestamp
+            + "\"}, "
+            + "{\"source\": \"myHost2\", \"timestamp\": \""
+            + timestamp
+            + "\"}]}";
+
+    String expectedLog1 =
+        "[{\"source\":\"myHost1\",\"timestamp\":" + timestamp + ",\"text\":\"\"" + "}]";
+    String expectedLog2 =
+        "[{\"source\":\"myHost2\",\"timestamp\":" + timestamp + ",\"text\":\"\"" + "}]";
+
+    AtomicBoolean gotLog = new AtomicBoolean(false);
+    Set actualLogs = new HashSet<>();
+    server.update(
+        req -> {
+          String content = req.content().toString(CharsetUtil.UTF_8);
+          logger.fine("Content received: " + content);
+          actualLogs.add(content);
+          return makeResponse(HttpResponseStatus.OK, "");
+        });
+    gzippedHttpPost("http://localhost:" + proxyPort + "/?f=" + "logs_json_cloudwatch", payload);
+    HandlerKey key = HandlerKey.of(ReportableEntityType.LOGS, String.valueOf(proxyPort));
+    proxy.senderTaskFactory.flushNow(key);
+    ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
+    assertEquals(2, actualLogs.size());
+    if (actualLogs.contains(expectedLog1) && actualLogs.contains(expectedLog2)) gotLog.set(true);
+    assertTrueWithTimeout(50, gotLog::get);
+  }
+
+  private static class WrappingHttpHandler extends AbstractHttpOnlyHandler {
+    private final Function func;
+
+    public WrappingHttpHandler(
+        @Nullable TokenAuthenticator tokenAuthenticator,
+        @Nullable HealthCheckManager healthCheckManager,
+        @Nullable String handle,
+        @Nonnull Function func) {
+      super(tokenAuthenticator, healthCheckManager, handle);
+      this.func = func;
+    }
+
+    @Override
+    protected void handleHttpMessage(ChannelHandlerContext ctx, FullHttpRequest request) {
+      URI uri;
+      try {
+        uri = new URI(request.uri());
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+      String path = uri.getPath();
+      logger.fine("Incoming HTTP request: " + uri.getPath());
+      if (path.endsWith("/checkin")
+          && (path.startsWith("/api/daemon") || path.contains("wfproxy"))) {
+        // simulate checkin response for proxy chaining
+        ObjectNode jsonResponse = JsonNodeFactory.instance.objectNode();
+        jsonResponse.put("currentTime", Clock.now());
+        jsonResponse.put("allowAnyHostKeys", true);
+        jsonResponse.put("logServerEndpointUrl", "http://localhost:" + handle + "/api/");
+        jsonResponse.put("logServerToken", "12345");
+        writeHttpResponse(ctx, HttpResponseStatus.OK, jsonResponse, request);
+        return;
+      } else if (path.endsWith("/config/processed")) {
+        writeHttpResponse(ctx, HttpResponseStatus.OK, "", request);
+        return;
+      } else if (path.endsWith("/wfproxy/saveConfig")) {
+        writeHttpResponse(ctx, HttpResponseStatus.OK, "", request);
+        return;
+      } else if (path.endsWith("/wfproxy/savePreprocessorRules")) {
+        writeHttpResponse(ctx, HttpResponseStatus.OK, "", request);
+        return;
+      }
+      HttpResponse response = func.apply(request);
+      logger.fine("Responding with HTTP " + response.status());
+      writeHttpResponse(ctx, response, request);
+    }
+  }
+
+  private static class MutableFunc implements Function {
+    private Function delegate;
+
+    public MutableFunc(Function delegate) {
+      this.delegate = delegate;
+    }
+
+    public void update(Function delegate) {
+      this.delegate = delegate;
+    }
+
+    @Override
+    public R apply(T t) {
+      return delegate.apply(t);
+    }
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/PointMatchers.java b/proxy/src/test/java/com/wavefront/agent/PointMatchers.java
index e3719cbb6..b6823728f 100644
--- a/proxy/src/test/java/com/wavefront/agent/PointMatchers.java
+++ b/proxy/src/test/java/com/wavefront/agent/PointMatchers.java
@@ -1,20 +1,13 @@
 package com.wavefront.agent;
 
-import com.yammer.metrics.core.WavefrontHistogram;
-
+import java.util.Map;
 import org.hamcrest.BaseMatcher;
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
+import wavefront.report.Histogram;
 import wavefront.report.ReportPoint;
 
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public class PointMatchers {
 
   private static String mapToString(Map map) {
@@ -40,27 +33,61 @@ private static  boolean mapsEqual(Map m1, Map m2) {
     return isSubMap(m1, m2) && isSubMap(m2, m1);
   }
 
-  public static Matcher matches(Object value, String metricName, Map tags) {
+  public static Matcher matches(
+      Object value, String metricName, Map tags) {
     return new BaseMatcher() {
 
       @Override
       public boolean matches(Object o) {
         ReportPoint me = (ReportPoint) o;
-        return me.getValue().equals(value) && me.getMetric().equals(metricName)
+        return me.getValue().equals(value)
+            && me.getMetric().equals(metricName)
             && mapsEqual(me.getAnnotations(), tags);
       }
 
       @Override
       public void describeTo(Description description) {
         description.appendText(
-            "Value should equal " + value.toString() + " and have metric name " + metricName + " and tags "
+            "Value should equal "
+                + value.toString()
+                + " and have metric name "
+                + metricName
+                + " and tags "
                 + mapToString(tags));
+      }
+    };
+  }
 
+  public static Matcher matches(
+      Object value, String metricName, String hostName, Map tags) {
+    return new BaseMatcher() {
+
+      @Override
+      public boolean matches(Object o) {
+        ReportPoint me = (ReportPoint) o;
+        return me.getValue().equals(value)
+            && me.getMetric().equals(metricName)
+            && me.getHost().equals(hostName)
+            && mapsEqual(me.getAnnotations(), tags);
+      }
+
+      @Override
+      public void describeTo(Description description) {
+        description.appendText(
+            "Value should equal "
+                + value.toString()
+                + " and have metric name "
+                + metricName
+                + ", host "
+                + hostName
+                + ", and tags "
+                + mapToString(tags));
       }
     };
   }
 
-  public static Matcher almostMatches(double value, String metricName, Map tags) {
+  public static Matcher almostMatches(
+      double value, String metricName, Map tags) {
     return new BaseMatcher() {
 
       @Override
@@ -75,11 +102,36 @@ public boolean matches(Object o) {
       @Override
       public void describeTo(Description description) {
         description.appendText(
-            "Value should approximately equal " + value + " and have metric name " + metricName + " and tags "
+            "Value should approximately equal "
+                + value
+                + " and have metric name "
+                + metricName
+                + " and tags "
                 + mapToString(tags));
-
       }
     };
   }
 
+  public static Matcher histogramMatches(int samples, double weight) {
+    return new BaseMatcher() {
+
+      @Override
+      public boolean matches(Object o) {
+        ReportPoint point = (ReportPoint) o;
+        if (!(point.getValue() instanceof Histogram)) return false;
+        Histogram value = (Histogram) point.getValue();
+        double sum = 0;
+        for (int i = 0; i < value.getBins().size(); i++) {
+          sum += value.getBins().get(i) * value.getCounts().get(i);
+        }
+        return sum == weight && value.getCounts().stream().reduce(Integer::sum).get() == samples;
+      }
+
+      @Override
+      public void describeTo(Description description) {
+        description.appendText(
+            "Total histogram weight should be " + weight + ", and total samples = " + samples);
+      }
+    };
+  }
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/ProxyCheckInSchedulerTest.java b/proxy/src/test/java/com/wavefront/agent/ProxyCheckInSchedulerTest.java
new file mode 100644
index 000000000..dc17d6617
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/ProxyCheckInSchedulerTest.java
@@ -0,0 +1,795 @@
+package com.wavefront.agent;
+
+import static com.wavefront.agent.api.APIContainer.CENTRAL_TENANT_NAME;
+import static com.wavefront.common.Utils.getBuildVersion;
+import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.api.ProxyV2API;
+import com.wavefront.api.agent.AgentConfiguration;
+import com.wavefront.api.agent.ValidationConfiguration;
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+import java.util.Collections;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.ws.rs.ClientErrorException;
+import javax.ws.rs.ProcessingException;
+import javax.ws.rs.ServerErrorException;
+import javax.ws.rs.core.Response;
+import org.easymock.EasyMock;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/** @author vasily@wavefront.com */
+public class ProxyCheckInSchedulerTest {
+
+  @Test
+  public void testNormalCheckin() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    apiContainer.updateLogServerEndpointURLandToken(anyObject(), anyObject());
+    expectLastCall().anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    returnConfig.setPointsPerBatch(1234567L);
+    returnConfig.currentTime = System.currentTimeMillis();
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andReturn(returnConfig)
+        .once();
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    proxyV2API.proxySaveConfig(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    replay(proxyV2API, apiContainer);
+    ProxyCheckInScheduler scheduler =
+        new ProxyCheckInScheduler(
+            proxyId,
+            proxyConfig,
+            apiContainer,
+            (tenantName, config) -> assertEquals(1234567L, config.getPointsPerBatch().longValue()),
+            () -> {},
+            () -> {});
+    scheduler.scheduleCheckins();
+    verify(proxyConfig, proxyV2API, apiContainer);
+    assertEquals(1, scheduler.getSuccessfulCheckinCount());
+    scheduler.shutdown();
+  }
+
+  @Ignore
+  @Test
+  public void testNormalCheckinWithRemoteShutdown() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    TenantInfo token = new TokenWorkerWF("abcde12345", "https://acme.corp/api");
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, token);
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    apiContainer.updateLogServerEndpointURLandToken(anyObject(), anyObject());
+    expectLastCall().anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    returnConfig.setShutOffAgents(true);
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andReturn(returnConfig)
+        .anyTimes();
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    proxyV2API.proxySaveConfig(eq(proxyId), anyObject());
+    proxyV2API.proxySavePreprocessorRules(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    replay(proxyV2API, apiContainer);
+    AtomicBoolean shutdown = new AtomicBoolean(false);
+    ProxyCheckInScheduler.preprocessorRulesNeedUpdate.set(true);
+    ProxyCheckInScheduler scheduler =
+        new ProxyCheckInScheduler(
+            proxyId,
+            proxyConfig,
+            apiContainer,
+            (tenantName, config) -> {},
+            () -> shutdown.set(true),
+            () -> {});
+    scheduler.updateProxyMetrics();
+    scheduler.updateConfiguration();
+    verify(proxyConfig, proxyV2API, apiContainer);
+    assertTrue(shutdown.get());
+  }
+
+  @Test
+  public void testNormalCheckinWithBadConsumer() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    TenantInfo token = new TokenWorkerWF("abcde12345", "https://acme.corp/api");
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, token);
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    expect(proxyConfig.getLogServerIngestionURL()).andReturn(null);
+    expect(proxyConfig.getLogServerIngestionToken()).andReturn(null);
+    apiContainer.updateLogServerEndpointURLandToken(anyObject(), anyObject());
+    proxyConfig.setEnableHyperlogsConvergedCsp(true);
+    proxyConfig.setReceivedLogServerDetails(false);
+    expectLastCall().anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andReturn(returnConfig)
+        .anyTimes();
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    proxyV2API.proxySaveConfig(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    replay(proxyV2API, apiContainer);
+    try {
+      ProxyCheckInScheduler scheduler =
+          new ProxyCheckInScheduler(
+              proxyId,
+              proxyConfig,
+              apiContainer,
+              (tenantName, config) -> {
+                throw new NullPointerException("gotcha!");
+              },
+              () -> {},
+              () -> {});
+      scheduler.updateProxyMetrics();
+      scheduler.updateConfiguration();
+      verify(proxyConfig, proxyV2API, apiContainer);
+      fail("We're not supposed to get here");
+    } catch (NullPointerException e) {
+      // NPE caught, we're good
+    }
+  }
+
+  @Test
+  public void testNetworkErrors() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    TenantInfo token = new TokenWorkerWF("abcde12345", "https://acme.corp/zzz");
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, token);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    returnConfig.setPointsPerBatch(1234567L);
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    proxyV2API.proxySavePreprocessorRules(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ProcessingException(new UnknownHostException()))
+        .once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ProcessingException(new SocketTimeoutException()))
+        .once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ProcessingException(new ConnectException()))
+        .once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ProcessingException(new NullPointerException()))
+        .once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new NullPointerException())
+        .once();
+    proxyV2API.proxySaveConfig(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    replay(proxyV2API, apiContainer);
+    ProxyCheckInScheduler scheduler =
+        new ProxyCheckInScheduler(
+            proxyId,
+            proxyConfig,
+            apiContainer,
+            (tenantName, config) -> fail("We are not supposed to get here"),
+            () -> {},
+            () -> {});
+    scheduler.updateConfiguration();
+    scheduler.updateConfiguration();
+    scheduler.updateConfiguration();
+    scheduler.updateConfiguration();
+    verify(proxyConfig, proxyV2API, apiContainer);
+  }
+
+  @Test
+  public void testHttpErrors() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    TenantInfo token = new TokenWorkerWF("abcde12345", "https://acme.corp/zzz");
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, token);
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    apiContainer.updateLogServerEndpointURLandToken(anyObject(), anyObject());
+    expectLastCall().anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    // we need to allow 1 successful checking to prevent early termination
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andReturn(returnConfig)
+        .once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ClientErrorException(Response.status(401).build()))
+        .once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ClientErrorException(Response.status(403).build()))
+        .once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ClientErrorException(Response.status(407).build()))
+        .once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ClientErrorException(Response.status(408).build()))
+        .once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ClientErrorException(Response.status(429).build()))
+        .once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ServerErrorException(Response.status(500).build()))
+        .once();
+    proxyV2API.proxySavePreprocessorRules(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ServerErrorException(Response.status(502).build()))
+        .once();
+    proxyV2API.proxySaveConfig(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    replay(proxyV2API, apiContainer);
+    ProxyCheckInScheduler scheduler =
+        new ProxyCheckInScheduler(
+            proxyId,
+            proxyConfig,
+            apiContainer,
+            (tenantName, config) -> assertNull(config.getPointsPerBatch()),
+            () -> {},
+            () -> {});
+    scheduler.updateProxyMetrics();
+    scheduler.updateConfiguration();
+    scheduler.updateProxyMetrics();
+    scheduler.updateConfiguration();
+    scheduler.updateProxyMetrics();
+    scheduler.updateConfiguration();
+    scheduler.updateProxyMetrics();
+    scheduler.updateConfiguration();
+    scheduler.updateProxyMetrics();
+    scheduler.updateConfiguration();
+    scheduler.updateProxyMetrics();
+    scheduler.updateConfiguration();
+    scheduler.updateProxyMetrics();
+    scheduler.updateConfiguration();
+    verify(proxyConfig, proxyV2API, apiContainer);
+  }
+
+  @Test
+  public void testRetryCheckinOnMisconfiguredUrl() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    TenantInfo token = new TokenWorkerWF("abcde12345", "https://acme.corp/zzz");
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, token);
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    apiContainer.updateLogServerEndpointURLandToken(anyObject(), anyObject());
+    expectLastCall().anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    expect(proxyConfig.getPreprocessorConfigFile()).andReturn("testFilepath").anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    returnConfig.setPointsPerBatch(1234567L);
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ClientErrorException(Response.status(404).build()))
+        .once();
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    apiContainer.updateServerEndpointURL(CENTRAL_TENANT_NAME, "https://acme.corp/zzz/api/");
+    expectLastCall().once();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andReturn(returnConfig)
+        .once();
+    proxyV2API.proxySaveConfig(eq(proxyId), anyObject());
+    proxyV2API.proxySavePreprocessorRules(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    replay(proxyV2API, apiContainer);
+    ProxyCheckInScheduler.preprocessorRulesNeedUpdate.set(true);
+    ProxyCheckInScheduler scheduler =
+        new ProxyCheckInScheduler(
+            proxyId,
+            proxyConfig,
+            apiContainer,
+            (tenantName, config) -> assertEquals(1234567L, config.getPointsPerBatch().longValue()),
+            () -> {},
+            () -> {});
+    verify(proxyConfig, proxyV2API, apiContainer);
+  }
+
+  @Test
+  public void testRetryCheckinOnMisconfiguredUrlFailsTwiceTerminates() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    TenantInfo token = new TokenWorkerWF("abcde12345", "https://acme.corp/zzz");
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, token);
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    returnConfig.setPointsPerBatch(1234567L);
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ClientErrorException(Response.status(404).build()))
+        .times(2);
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    proxyV2API.proxySaveConfig(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    apiContainer.updateServerEndpointURL(CENTRAL_TENANT_NAME, "https://acme.corp/zzz/api/");
+    expectLastCall().once();
+    replay(proxyV2API, apiContainer);
+    try {
+      ProxyCheckInScheduler scheduler =
+          new ProxyCheckInScheduler(
+              proxyId,
+              proxyConfig,
+              apiContainer,
+              (tenantName, config) -> fail("We are not supposed to get here"),
+              () -> {},
+              () -> {});
+      fail();
+    } catch (RuntimeException e) {
+      //
+    }
+    verify(proxyConfig, proxyV2API, apiContainer);
+  }
+
+  @Test
+  public void testDontRetryCheckinOnMisconfiguredUrlThatEndsWithApi() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    TenantInfo token = new TokenWorkerWF("abcde12345", "https://acme.corp/api");
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, token);
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    returnConfig.setPointsPerBatch(1234567L);
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ClientErrorException(Response.status(404).build()))
+        .once();
+    replay(proxyV2API, apiContainer);
+    try {
+      ProxyCheckInScheduler scheduler =
+          new ProxyCheckInScheduler(
+              proxyId,
+              proxyConfig,
+              apiContainer,
+              (tenantName, config) -> fail("We are not supposed to get here"),
+              () -> {},
+              () -> {});
+      fail();
+    } catch (RuntimeException e) {
+      //
+    }
+    verify(proxyConfig, proxyV2API, apiContainer);
+  }
+
+  @Test
+  public void testDontRetryCheckinOnBadCredentials() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    TenantInfo token = new TokenWorkerWF("abcde12345", "https://acme.corp/api");
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, token);
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    returnConfig.setPointsPerBatch(1234567L);
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andThrow(new ClientErrorException(Response.status(401).build()))
+        .once();
+    replay(proxyV2API, apiContainer);
+    try {
+      ProxyCheckInScheduler scheduler =
+          new ProxyCheckInScheduler(
+              proxyId,
+              proxyConfig,
+              apiContainer,
+              (tenantName, config) -> fail("We are not supposed to get here"),
+              () -> {},
+              () -> {});
+      fail("We're not supposed to get here");
+    } catch (RuntimeException e) {
+      //
+    }
+    verify(proxyConfig, proxyV2API, apiContainer);
+  }
+
+  @Test
+  public void testCheckinConvergedCSPWithLogServerConfiguration() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    TenantInfo token = new TokenWorkerWF("abcde12345", "https://acme.corp/api");
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, token);
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    expect(proxyConfig.getLogServerIngestionURL()).andReturn("vRLIC-URL");
+    expect(proxyConfig.getLogServerIngestionToken()).andReturn("vRLIC-token");
+    apiContainer.updateLogServerEndpointURLandToken(anyObject(), anyObject());
+    proxyConfig.setEnableHyperlogsConvergedCsp(true);
+    expectLastCall().anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    returnConfig.setPointsPerBatch(1234567L);
+    returnConfig.currentTime = System.currentTimeMillis();
+    ValidationConfiguration validationConfiguration = new ValidationConfiguration();
+    validationConfiguration.setEnableHyperlogsConvergedCsp(true);
+    returnConfig.setValidationConfiguration(validationConfiguration);
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andReturn(returnConfig)
+        .once();
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    proxyV2API.proxySaveConfig(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    replay(proxyV2API, apiContainer);
+    ProxyCheckInScheduler scheduler =
+        new ProxyCheckInScheduler(
+            proxyId,
+            proxyConfig,
+            apiContainer,
+            (tenantName, config) -> assertEquals(1234567L, config.getPointsPerBatch().longValue()),
+            () -> {},
+            () -> {});
+    scheduler.scheduleCheckins();
+    verify(proxyConfig, proxyV2API, apiContainer);
+    assertEquals(1, scheduler.getSuccessfulCheckinCount());
+    scheduler.shutdown();
+  }
+
+  @Test
+  public void testCheckinConvergedCSPWithoutLogServerConfiguration() {
+    ProxyConfig proxyConfig = EasyMock.createMock(ProxyConfig.class);
+    ProxyV2API proxyV2API = EasyMock.createMock(ProxyV2API.class);
+    APIContainer apiContainer = EasyMock.createMock(APIContainer.class);
+    TenantInfo token = new TokenWorkerWF("abcde12345", "https://acme.corp/api");
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, token);
+    reset(proxyConfig, proxyV2API, proxyConfig);
+    expect(proxyConfig.getHostname()).andReturn("proxyHost").anyTimes();
+    expect(proxyConfig.isEphemeral()).andReturn(true).anyTimes();
+    expect(proxyConfig.getAgentMetricsPointTags()).andReturn(Collections.emptyMap()).anyTimes();
+    expect(proxyConfig.getProxyname()).andReturn("proxyName").anyTimes();
+    expect(proxyConfig.getJsonConfig()).andReturn(null).anyTimes();
+    expect(proxyConfig.getLogServerIngestionURL()).andReturn(null);
+    expect(proxyConfig.getLogServerIngestionToken()).andReturn(null);
+    apiContainer.updateLogServerEndpointURLandToken(anyObject(), anyObject());
+    proxyConfig.setEnableHyperlogsConvergedCsp(true);
+    proxyConfig.setReceivedLogServerDetails(false);
+    expectLastCall().anyTimes();
+    String authHeader = "Bearer abcde12345";
+    AgentConfiguration returnConfig = new AgentConfiguration();
+    returnConfig.setPointsPerBatch(1234567L);
+    returnConfig.currentTime = System.currentTimeMillis();
+    ValidationConfiguration validationConfiguration = new ValidationConfiguration();
+    validationConfiguration.setEnableHyperlogsConvergedCsp(true);
+    returnConfig.setValidationConfiguration(validationConfiguration);
+    replay(proxyConfig);
+    UUID proxyId = ProxyUtil.getOrCreateProxyId(proxyConfig);
+    expect(
+            proxyV2API.proxyCheckin(
+                eq(proxyId),
+                eq(authHeader),
+                eq("proxyHost"),
+                eq("proxyName"),
+                eq(getBuildVersion()),
+                anyLong(),
+                anyObject(),
+                eq(true)))
+        .andReturn(returnConfig)
+        .once();
+    expect(apiContainer.getProxyV2APIForTenant(EasyMock.anyObject(String.class)))
+        .andReturn(proxyV2API)
+        .anyTimes();
+    proxyV2API.proxySaveConfig(eq(proxyId), anyObject());
+    expectLastCall().anyTimes();
+    replay(proxyV2API, apiContainer);
+    ProxyCheckInScheduler scheduler =
+        new ProxyCheckInScheduler(
+            proxyId,
+            proxyConfig,
+            apiContainer,
+            (tenantName, config) -> assertEquals(1234567L, config.getPointsPerBatch().longValue()),
+            () -> {},
+            () -> {});
+    scheduler.scheduleCheckins();
+    verify(proxyConfig, proxyV2API, apiContainer);
+    assertEquals(1, scheduler.getSuccessfulCheckinCount());
+    scheduler.shutdown();
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/ProxyConfigTest.java b/proxy/src/test/java/com/wavefront/agent/ProxyConfigTest.java
new file mode 100644
index 000000000..60d0d29e2
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/ProxyConfigTest.java
@@ -0,0 +1,298 @@
+package com.wavefront.agent;
+
+import static com.wavefront.agent.api.APIContainer.CENTRAL_TENANT_NAME;
+import static org.junit.Assert.*;
+
+import com.beust.jcommander.ParameterException;
+import com.wavefront.agent.auth.TokenValidationMethod;
+import com.wavefront.agent.data.TaskQueueLevel;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.UUID;
+import org.junit.Test;
+
+public class ProxyConfigTest {
+
+  @Test
+  public void testArgsAndFile() throws IOException {
+    File cfgFile = File.createTempFile("proxy", ".cfg");
+    cfgFile.deleteOnExit();
+
+    Properties props = new Properties();
+    props.setProperty("pushListenerPorts", "1234");
+
+    FileOutputStream out = new FileOutputStream(cfgFile);
+    props.store(out, "");
+    out.close();
+
+    String[] args =
+        new String[] {
+          "-f",
+          cfgFile.getAbsolutePath(),
+          "--pushListenerPorts",
+          "4321",
+          "--proxyname",
+          "proxyname",
+          "--token",
+          UUID.randomUUID().toString()
+        };
+
+    ProxyConfig cfg = new ProxyConfig();
+    assertTrue(cfg.parseArguments(args, ""));
+    assertEquals(cfg.getProxyname(), "proxyname");
+    assertEquals(cfg.getPushListenerPorts(), "1234");
+  }
+
+  @Test
+  public void testBadConfig() {
+    String[] args =
+        new String[] {
+          "--token", UUID.randomUUID().toString(),
+          "--cspAppId", UUID.randomUUID().toString()
+        };
+    assertThrows(IllegalArgumentException.class, () -> new ProxyConfig().parseArguments(args, ""));
+
+    String[] args2 =
+        new String[] {
+          "--token", UUID.randomUUID().toString(),
+          "--cspAppSecret", UUID.randomUUID().toString()
+        };
+    assertThrows(IllegalArgumentException.class, () -> new ProxyConfig().parseArguments(args2, ""));
+
+    String[] args3 =
+        new String[] {
+          "--token", UUID.randomUUID().toString(),
+          "--cspAppId", UUID.randomUUID().toString(),
+          "--cspAppSecret", UUID.randomUUID().toString()
+        };
+    assertThrows(IllegalArgumentException.class, () -> new ProxyConfig().parseArguments(args3, ""));
+
+    String[] args4 =
+        new String[] {
+          "--cspAPIToken", UUID.randomUUID().toString(),
+          "--cspAppId", UUID.randomUUID().toString(),
+          "--cspAppSecret", UUID.randomUUID().toString()
+        };
+
+    assertThrows(IllegalArgumentException.class, () -> new ProxyConfig().parseArguments(args4, ""));
+
+    String[] args5 =
+        new String[] {
+          "--token", UUID.randomUUID().toString(),
+          "--cspAPIToken", UUID.randomUUID().toString()
+        };
+    assertThrows(IllegalArgumentException.class, () -> new ProxyConfig().parseArguments(args5, ""));
+  }
+
+  @Test
+  public void testBadCSPOAuthConfig() {
+    String[] args = new String[] {"--cspAppId", UUID.randomUUID().toString()};
+    assertThrows(IllegalArgumentException.class, () -> new ProxyConfig().parseArguments(args, ""));
+
+    String[] args2 = new String[] {"--cspAppSecret", UUID.randomUUID().toString()};
+    assertThrows(IllegalArgumentException.class, () -> new ProxyConfig().parseArguments(args2, ""));
+
+    String[] args3 =
+        new String[] {
+          "--token", UUID.randomUUID().toString(),
+          "--cspAppId", UUID.randomUUID().toString(),
+          "--cspAppSecret", UUID.randomUUID().toString()
+        };
+
+    assertThrows(IllegalArgumentException.class, () -> new ProxyConfig().parseArguments(args3, ""));
+  }
+
+  @Test
+  public void testGoodCSPOAuthConfig() {
+    String[] args =
+        new String[] {
+          "--cspAppId", UUID.randomUUID().toString(),
+          "--cspAppSecret", UUID.randomUUID().toString()
+        };
+
+    assertTrue(new ProxyConfig().parseArguments(args, ""));
+  }
+
+  @Test
+  public void testGoodCSPUserConfig() {
+    String[] args = new String[] {"--cspAPIToken", UUID.randomUUID().toString()};
+
+    assertTrue(new ProxyConfig().parseArguments(args, ""));
+  }
+
+  @Test
+  public void testGoodWfTokenConfig() {
+    String[] args = new String[] {"--token", UUID.randomUUID().toString()};
+
+    assertTrue(new ProxyConfig().parseArguments(args, ""));
+  }
+
+  @Test
+  public void testMultiTennat() throws IOException {
+    File cfgFile = File.createTempFile("proxy", ".cfg");
+    cfgFile.deleteOnExit();
+
+    Properties props = new Properties();
+    props.setProperty("pushListenerPorts", "1234");
+
+    props.setProperty("multicastingTenants", "2");
+
+    props.setProperty("multicastingTenantName_1", "name1");
+    props.setProperty("multicastingServer_1", "server1");
+    props.setProperty("multicastingToken_1", "token1");
+
+    props.setProperty("multicastingTenantName_2", "name2");
+    props.setProperty("multicastingServer_2", "server2");
+    props.setProperty("multicastingToken_2", "token2");
+
+    FileOutputStream out = new FileOutputStream(cfgFile);
+    props.store(out, "");
+    out.close();
+
+    String token = UUID.randomUUID().toString();
+    String[] args =
+        new String[] {
+          "-f",
+          cfgFile.getAbsolutePath(),
+          "--pushListenerPorts",
+          "4321",
+          "--proxyname",
+          "proxyname",
+          "--token",
+          token
+        };
+
+    ProxyConfig cfg = new ProxyConfig();
+    assertTrue(cfg.parseArguments(args, ""));
+
+    // default values
+    TenantInfo info = TokenManager.getMulticastingTenantList().get(CENTRAL_TENANT_NAME);
+    assertNotNull(info);
+    assertEquals("http://localhost:8080/api/", info.getWFServer());
+    assertEquals(token, info.getBearerToken());
+
+    info = TokenManager.getMulticastingTenantList().get("name1");
+    assertNotNull(info);
+    assertEquals("server1", info.getWFServer());
+    assertEquals("token1", info.getBearerToken());
+
+    info = TokenManager.getMulticastingTenantList().get("name2");
+    assertNotNull(info);
+    assertEquals("server2", info.getWFServer());
+    assertEquals("token2", info.getBearerToken());
+
+    assertNull(TokenManager.getMulticastingTenantList().get("fake"));
+  }
+
+  @Test
+  public void testVersionOrHelpReturnFalse() {
+    assertFalse(new ProxyConfig().parseArguments(new String[] {"--version"}, "PushAgentTest"));
+    assertFalse(new ProxyConfig().parseArguments(new String[] {"--help"}, "PushAgentTest"));
+    assertTrue(
+        new ProxyConfig()
+            .parseArguments(
+                new String[] {"--token", UUID.randomUUID().toString()}, "PushAgentTest"));
+  }
+
+  @Test
+  public void testTokenValidationMethodParsing() {
+    ProxyConfig proxyConfig = new ProxyConfig();
+    proxyConfig.parseArguments(
+        new String[] {"--token", UUID.randomUUID().toString()}, "PushAgentTest");
+
+    proxyConfig.parseArguments(new String[] {"--authMethod", "NONE"}, "PushAgentTest");
+    assertEquals(proxyConfig.authMethod, TokenValidationMethod.NONE);
+
+    proxyConfig.parseArguments(new String[] {"--authMethod", "STATIC_TOKEN"}, "PushAgentTest");
+    assertEquals(proxyConfig.authMethod, TokenValidationMethod.STATIC_TOKEN);
+
+    proxyConfig.parseArguments(new String[] {"--authMethod", "HTTP_GET"}, "PushAgentTest");
+    assertEquals(proxyConfig.authMethod, TokenValidationMethod.HTTP_GET);
+
+    proxyConfig.parseArguments(new String[] {"--authMethod", "OAUTH2"}, "PushAgentTest");
+    assertEquals(proxyConfig.authMethod, TokenValidationMethod.OAUTH2);
+
+    try {
+      proxyConfig.parseArguments(new String[] {"--authMethod", "OTHER"}, "PushAgentTest");
+      fail();
+    } catch (ParameterException e) {
+      // noop
+    }
+
+    try {
+      proxyConfig.parseArguments(new String[] {"--authMethod", ""}, "PushAgentTest");
+      fail();
+    } catch (ParameterException e) {
+      // noop
+    }
+  }
+
+  @Test
+  public void testTaskQueueLevelParsing() {
+    ProxyConfig proxyConfig = new ProxyConfig();
+    proxyConfig.parseArguments(
+        new String[] {"--token", UUID.randomUUID().toString()}, "PushAgentTest");
+
+    proxyConfig.parseArguments(new String[] {"--taskQueueLevel", "NEVER"}, "PushAgentTest");
+    assertEquals(proxyConfig.taskQueueLevel, TaskQueueLevel.NEVER);
+
+    proxyConfig.parseArguments(new String[] {"--taskQueueLevel", "MEMORY"}, "PushAgentTest");
+    assertEquals(proxyConfig.taskQueueLevel, TaskQueueLevel.MEMORY);
+
+    proxyConfig.parseArguments(new String[] {"--taskQueueLevel", "PUSHBACK"}, "PushAgentTest");
+    assertEquals(proxyConfig.taskQueueLevel, TaskQueueLevel.PUSHBACK);
+
+    proxyConfig.parseArguments(new String[] {"--taskQueueLevel", "ANY_ERROR"}, "PushAgentTest");
+    assertEquals(proxyConfig.taskQueueLevel, TaskQueueLevel.ANY_ERROR);
+
+    proxyConfig.parseArguments(new String[] {"--taskQueueLevel", "ALWAYS"}, "PushAgentTest");
+    assertEquals(proxyConfig.taskQueueLevel, TaskQueueLevel.ALWAYS);
+
+    try {
+      proxyConfig.parseArguments(new String[] {"--taskQueueLevel", "OTHER"}, "PushAgentTest");
+      fail();
+    } catch (ParameterException e) {
+      // noop
+    }
+
+    try {
+      proxyConfig.parseArguments(new String[] {"--taskQueueLevel", ""}, "PushAgentTest");
+      fail();
+    } catch (ParameterException e) {
+      // noop
+    }
+  }
+
+  @Test
+  public void testOtlpResourceAttrsOnMetricsIncluded() {
+    ProxyConfig config = new ProxyConfig();
+    config.parseArguments(new String[] {"--token", UUID.randomUUID().toString()}, "PushAgentTest");
+
+    // do not include OTLP resource attributes by default on metrics
+    // TODO: find link from OTel GH PR where this choice was made
+    assertFalse(config.isOtlpResourceAttrsOnMetricsIncluded());
+
+    // include OTLP resource attributes
+    config.parseArguments(
+        new String[] {"--otlpResourceAttrsOnMetricsIncluded", String.valueOf(true)},
+        "PushAgentTest");
+    assertTrue(config.isOtlpResourceAttrsOnMetricsIncluded());
+  }
+
+  @Test
+  public void testOtlpAppTagsOnMetricsIncluded() {
+    ProxyConfig config = new ProxyConfig();
+    config.parseArguments(new String[] {"--token", UUID.randomUUID().toString()}, "PushAgentTest");
+
+    // include application, shard, cluster, service.name resource attributes by default on
+    // metrics
+    assertTrue(config.isOtlpAppTagsOnMetricsIncluded());
+
+    // do not include the above-mentioned resource attributes
+    config.parseArguments(
+        new String[] {"--otlpAppTagsOnMetricsIncluded", String.valueOf(false)}, "PushAgentTest");
+    assertFalse(config.isOtlpAppTagsOnMetricsIncluded());
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/ProxyUtilTest.java b/proxy/src/test/java/com/wavefront/agent/ProxyUtilTest.java
new file mode 100644
index 000000000..4c673cb82
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/ProxyUtilTest.java
@@ -0,0 +1,26 @@
+package com.wavefront.agent;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import java.io.File;
+import java.util.UUID;
+import org.junit.Test;
+
+/** @author vasily@wavefront.com */
+public class ProxyUtilTest {
+
+  @Test
+  public void testLoadProxyIdFromFile() throws Exception {
+    UUID proxyId = UUID.randomUUID();
+    String path = File.createTempFile("proxyTestIdFile", null).getPath();
+    Files.asCharSink(new File(path), Charsets.UTF_8).write(proxyId.toString());
+    UUID uuid = ProxyUtil.getOrCreateProxyIdFromFile(path);
+    assertEquals(proxyId, uuid);
+
+    path = File.createTempFile("proxyTestIdFile", null).getPath() + ".id";
+    uuid = ProxyUtil.getOrCreateProxyIdFromFile(path);
+    assertEquals(uuid, ProxyUtil.getOrCreateProxyIdFromFile(path));
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/PushAgentTest.java b/proxy/src/test/java/com/wavefront/agent/PushAgentTest.java
index 33f48e7b5..a31021944 100644
--- a/proxy/src/test/java/com/wavefront/agent/PushAgentTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/PushAgentTest.java
@@ -1,116 +1,197 @@
 package com.wavefront.agent;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.io.Resources;
+import static com.wavefront.agent.TestUtils.findAvailablePort;
+import static com.wavefront.agent.TestUtils.getResource;
+import static com.wavefront.agent.TestUtils.gzippedHttpPost;
+import static com.wavefront.agent.TestUtils.httpGet;
+import static com.wavefront.agent.TestUtils.httpPost;
+import static com.wavefront.agent.TestUtils.verifyWithTimeout;
+import static com.wavefront.agent.TestUtils.waitUntilListenerIsOnline;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.HEART_BEAT_METRIC;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.startsWith;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.agent.channel.HealthCheckManagerImpl;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl;
+import com.wavefront.agent.handlers.HandlerKey;
 import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
 import com.wavefront.agent.handlers.ReportableEntityHandler;
 import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
-
-import junit.framework.AssertionFailedError;
-
-import net.jcip.annotations.NotThreadSafe;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.StatusLine;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.easymock.EasyMock;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
+import com.wavefront.agent.handlers.SenderTask;
+import com.wavefront.agent.handlers.SenderTaskFactory;
+import com.wavefront.agent.listeners.otlp.OtlpTestHelpers;
+import com.wavefront.api.agent.preprocessor.PreprocessorRuleMetrics;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.preprocessor.SpanAddAnnotationIfNotExistsTransformer;
+import com.wavefront.api.agent.preprocessor.SpanReplaceRegexTransformer;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.agent.tls.NaiveTrustManager;
+import com.wavefront.api.agent.AgentConfiguration;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.dto.Event;
+import com.wavefront.dto.SourceTag;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.entities.tracing.sampling.DurationSampler;
+import com.wavefront.sdk.entities.tracing.sampling.RateSampler;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.metrics.v1.Gauge;
+import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
+import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
+import io.opentelemetry.proto.metrics.v1.ScopeMetrics;
 import java.io.BufferedOutputStream;
-import java.io.BufferedWriter;
 import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.net.HttpURLConnection;
-import java.net.ServerSocket;
 import java.net.Socket;
-import java.net.URL;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 import java.util.zip.GZIPOutputStream;
-
+import javax.annotation.Nonnull;
+import javax.annotation.concurrent.NotThreadSafe;
 import javax.net.SocketFactory;
-
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManager;
+import junit.framework.AssertionFailedError;
+import org.apache.http.HttpResponse;
+import org.apache.http.StatusLine;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.easymock.Capture;
+import org.easymock.CaptureType;
+import org.easymock.EasyMock;
+import org.junit.*;
+import org.junit.rules.Timeout;
 import wavefront.report.Annotation;
 import wavefront.report.Histogram;
 import wavefront.report.HistogramType;
+import wavefront.report.ReportEvent;
 import wavefront.report.ReportPoint;
 import wavefront.report.ReportSourceTag;
+import wavefront.report.SourceOperationType;
+import wavefront.report.SourceTagAction;
 import wavefront.report.Span;
-
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
-import static org.easymock.EasyMock.verify;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
 
 @NotThreadSafe
 public class PushAgentTest {
-  protected static final Logger logger = Logger.getLogger(PushAgentTest.class.getCanonicalName());
-
+  private static SSLSocketFactory sslSocketFactory;
+  // Derived RED metrics related.
+  private final String PREPROCESSED_APPLICATION_TAG_VALUE = "preprocessedApplication";
+  private final String PREPROCESSED_SERVICE_TAG_VALUE = "preprocessedService";
+  private final String PREPROCESSED_CLUSTER_TAG_VALUE = "preprocessedCluster";
+  private final String PREPROCESSED_SHARD_TAG_VALUE = "preprocessedShard";
+  private final String PREPROCESSED_SOURCE_VALUE = "preprocessedSource";
+  private final long alignedStartTimeEpochSeconds = System.currentTimeMillis() / 1000 / 60 * 60;
   private PushAgent proxy;
-  private long startTime = System.currentTimeMillis() / 1000 / 60 * 60;
   private int port;
   private int tracePort;
+  private int customTracePort;
   private int ddPort;
-  private ReportableEntityHandler mockPointHandler =
+  private int deltaPort;
+  private ReportableEntityHandler mockPointHandler =
       MockReportableEntityHandlerFactory.getMockReportPointHandler();
-  private ReportableEntityHandler mockSourceTagHandler =
+  private ReportableEntityHandler mockSourceTagHandler =
       MockReportableEntityHandlerFactory.getMockSourceTagHandler();
-  private ReportableEntityHandler mockHistogramHandler =
+  private ReportableEntityHandler mockHistogramHandler =
       MockReportableEntityHandlerFactory.getMockHistogramHandler();
-  private ReportableEntityHandler mockTraceHandler =
+  private ReportableEntityHandler mockTraceHandler =
       MockReportableEntityHandlerFactory.getMockTraceHandler();
+  private ReportableEntityHandler mockTraceSpanLogsHandler =
+      MockReportableEntityHandlerFactory.getMockTraceSpanLogsHandler();
+  private ReportableEntityHandler mockEventHandler =
+      MockReportableEntityHandlerFactory.getMockEventHandlerImpl();
+  private WavefrontSender mockWavefrontSender = EasyMock.createMock(WavefrontSender.class);
+  private SenderTask mockSenderTask = EasyMock.createNiceMock(SenderTask.class);
+  private Map>> mockSenderTaskMap =
+      ImmutableMap.of(APIContainer.CENTRAL_TENANT_NAME, ImmutableList.of(mockSenderTask));
+
+  private SenderTaskFactory mockSenderTaskFactory =
+      new SenderTaskFactory() {
+        @SuppressWarnings("unchecked")
+        @Override
+        public Map>> createSenderTasks(
+            @Nonnull HandlerKey handlerKey) {
+          return mockSenderTaskMap;
+        }
+
+        @Override
+        public void shutdown() {}
+
+        @Override
+        public void shutdown(@Nonnull String handle) {}
+
+        @Override
+        public void drainBuffersToQueue(QueueingReason reason) {}
+
+        @Override
+        public void truncateBuffers() {}
+      };
+
   private ReportableEntityHandlerFactory mockHandlerFactory =
-      MockReportableEntityHandlerFactory.createMockHandlerFactory(mockPointHandler, mockSourceTagHandler,
-          mockHistogramHandler, mockTraceHandler);
+      MockReportableEntityHandlerFactory.createMockHandlerFactory(
+          mockPointHandler,
+          mockSourceTagHandler,
+          mockHistogramHandler,
+          mockTraceHandler,
+          mockTraceSpanLogsHandler,
+          mockEventHandler);
   private HttpClient mockHttpClient = EasyMock.createMock(HttpClient.class);
 
-  private static int findAvailablePort(int startingPortNumber) {
-    int portNum = startingPortNumber;
-    ServerSocket socket;
-    while (portNum < startingPortNumber + 1000) {
-      try {
-        socket = new ServerSocket(portNum);
-        socket.close();
-        logger.log(Level.INFO, "Found available port: " + portNum);
-        return portNum;
-      } catch (IOException exc) {
-        logger.log(Level.WARNING, "Port " + portNum + " is not available:" + exc.getMessage());
-      }
-      portNum++;
-    }
-    throw new RuntimeException("Unable to find an available port in the [" + startingPortNumber + ";" +
-        (startingPortNumber + 1000) + ") range");
+  @Rule public Timeout globalTimeout = Timeout.seconds(5);
+
+  @BeforeClass
+  public static void init() throws Exception {
+    TrustManager[] tm = new TrustManager[] {new NaiveTrustManager()};
+    SSLContext context = SSLContext.getInstance("SSL");
+    context.init(new KeyManager[0], tm, new SecureRandom());
+    sslSocketFactory = context.getSocketFactory();
+    HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
+    HttpsURLConnection.setDefaultHostnameVerifier((h, s) -> h.equals("localhost"));
   }
 
   @Before
   public void setup() throws Exception {
-    port = findAvailablePort(2888);
-    tracePort = findAvailablePort(3888);
-    ddPort = findAvailablePort(4888);
     proxy = new PushAgent();
-    proxy.flushThreads = 2;
-    proxy.retryThreads = 1;
-    proxy.pushListenerPorts = String.valueOf(port);
-    proxy.traceListenerPorts = String.valueOf(tracePort);
-    proxy.dataDogJsonPorts = String.valueOf(ddPort);
-    proxy.dataDogProcessSystemMetrics = false;
-    proxy.dataDogProcessServiceChecks = true;
-    proxy.startGraphiteListener(proxy.pushListenerPorts, mockHandlerFactory, null);
-    proxy.startTraceListener(proxy.traceListenerPorts, mockHandlerFactory);
-    proxy.startDataDogListener(proxy.dataDogJsonPorts, mockHandlerFactory, mockHttpClient);
-    TimeUnit.MILLISECONDS.sleep(500);
+    proxy.proxyConfig.flushThreads = 2;
+    proxy.proxyConfig.dataBackfillCutoffHours = 100000000;
+    proxy.proxyConfig.dataDogRequestRelaySyncMode = true;
+    proxy.proxyConfig.dataDogProcessSystemMetrics = false;
+    proxy.proxyConfig.dataDogProcessServiceChecks = true;
+    assertEquals(2, proxy.proxyConfig.getFlushThreads());
+    assertFalse(proxy.proxyConfig.isDataDogProcessSystemMetrics());
+    assertTrue(proxy.proxyConfig.isDataDogProcessServiceChecks());
   }
 
   @After
@@ -118,43 +199,229 @@ public void teardown() {
     proxy.shutdown();
   }
 
+  @Test
+  public void testSecureAll() throws Exception {
+    int securePort1 = findAvailablePort(2888);
+    int securePort2 = findAvailablePort(2889);
+    proxy.proxyConfig.privateCertPath =
+        getClass().getClassLoader().getResource("demo.cert").getPath();
+    proxy.proxyConfig.privateKeyPath =
+        getClass().getClassLoader().getResource("demo.key").getPath();
+    proxy.proxyConfig.tlsPorts = "*";
+    proxy.initSslContext();
+    proxy.proxyConfig.pushListenerPorts = securePort1 + "," + securePort2;
+    SpanSampler sampler = new SpanSampler(new RateSampler(1.0D), () -> null);
+    proxy.startGraphiteListener(String.valueOf(securePort1), mockHandlerFactory, null, sampler);
+    proxy.startGraphiteListener(String.valueOf(securePort2), mockHandlerFactory, null, sampler);
+    waitUntilListenerIsOnline(securePort1);
+    waitUntilListenerIsOnline(securePort2);
+    reset(mockPointHandler);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
+    expectLastCall();
+    replay(mockPointHandler);
+
+    // try plaintext over tcp first
+    Socket socket = sslSocketFactory.createSocket("localhost", securePort1);
+    BufferedOutputStream stream = new BufferedOutputStream(socket.getOutputStream());
+    String payloadStr =
+        "metric.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test1\n"
+            + "metric.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test2\n";
+    stream.write(payloadStr.getBytes());
+    stream.flush();
+    socket.close();
+    verifyWithTimeout(500, mockPointHandler);
+
+    reset(mockPointHandler);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test")
+            .setHost("test3")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test")
+            .setHost("test4")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
+    expectLastCall();
+    replay(mockPointHandler);
+
+    // secure test
+    socket = sslSocketFactory.createSocket("localhost", securePort2);
+    stream = new BufferedOutputStream(socket.getOutputStream());
+    payloadStr =
+        "metric.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test3\n"
+            + "metric.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test4\n";
+    stream.write(payloadStr.getBytes());
+    stream.flush();
+    socket.close();
+    verifyWithTimeout(500, mockPointHandler);
+  }
+
   @Test
   public void testWavefrontUnifiedPortHandlerPlaintextUncompressed() throws Exception {
+    port = findAvailablePort(2888);
+    int securePort = findAvailablePort(2889);
+    proxy.proxyConfig.privateCertPath =
+        getClass().getClassLoader().getResource("demo.cert").getPath();
+    proxy.proxyConfig.privateKeyPath =
+        getClass().getClassLoader().getResource("demo.key").getPath();
+    proxy.proxyConfig.tlsPorts = "1,23 , 4,   , " + securePort + "  ,6";
+    proxy.initSslContext();
+    proxy.proxyConfig.pushListenerPorts = port + "," + securePort;
+    SpanSampler sampler = new SpanSampler(new RateSampler(1.0D), () -> null);
+    proxy.startGraphiteListener(String.valueOf(port), mockHandlerFactory, null, sampler);
+    proxy.startGraphiteListener(String.valueOf(securePort), mockHandlerFactory, null, sampler);
+    waitUntilListenerIsOnline(port);
+    waitUntilListenerIsOnline(securePort);
     reset(mockPointHandler);
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric.test").setHost("test1").setTimestamp(startTime * 1000).setValue(0.0d).build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
     expectLastCall();
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric.test").setHost("test2").setTimestamp((startTime + 1) * 1000).setValue(1.0d).build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
     expectLastCall();
     replay(mockPointHandler);
 
     // try plaintext over tcp first
     Socket socket = SocketFactory.getDefault().createSocket("localhost", port);
     BufferedOutputStream stream = new BufferedOutputStream(socket.getOutputStream());
-    String payloadStr = "metric.test 0 " + startTime + " source=test1\n" +
-        "metric.test 1 " + (startTime + 1) + " source=test2\n";
+    String payloadStr =
+        "metric.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test1\n"
+            + "metric.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test2\n";
     stream.write(payloadStr.getBytes());
     stream.flush();
     socket.close();
-    TimeUnit.MILLISECONDS.sleep(500);
-    verify(mockPointHandler);
+    verifyWithTimeout(500, mockPointHandler);
+
+    reset(mockPointHandler);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test")
+            .setHost("test3")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test")
+            .setHost("test4")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
+    expectLastCall();
+    replay(mockPointHandler);
+
+    // secure test
+    socket = sslSocketFactory.createSocket("localhost", securePort);
+    stream = new BufferedOutputStream(socket.getOutputStream());
+    payloadStr =
+        "metric.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test3\n"
+            + "metric.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test4\n";
+    stream.write(payloadStr.getBytes());
+    stream.flush();
+    socket.close();
+    verifyWithTimeout(500, mockPointHandler);
   }
 
   @Test
   public void testWavefrontUnifiedPortHandlerGzippedPlaintextStream() throws Exception {
+    port = findAvailablePort(2888);
+    int securePort = findAvailablePort(2889);
+    proxy.proxyConfig.privateCertPath =
+        getClass().getClassLoader().getResource("demo.cert").getPath();
+    proxy.proxyConfig.privateKeyPath =
+        getClass().getClassLoader().getResource("demo.key").getPath();
+    proxy.proxyConfig.tlsPorts = "1,23 , 4,   , " + securePort + "  ,6";
+    proxy.initSslContext();
+    proxy.proxyConfig.pushListenerPorts = port + "," + securePort;
+    SpanSampler sampler = new SpanSampler(new RateSampler(1.0D), () -> null);
+    proxy.startGraphiteListener(String.valueOf(port), mockHandlerFactory, null, sampler);
+    proxy.startGraphiteListener(String.valueOf(securePort), mockHandlerFactory, null, sampler);
+    waitUntilListenerIsOnline(port);
+    waitUntilListenerIsOnline(securePort);
     reset(mockPointHandler);
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric2.test").setHost("test1").setTimestamp(startTime * 1000).setValue(0.0d).build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric2.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
     expectLastCall();
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric2.test").setHost("test2").setTimestamp((startTime + 1) * 1000).setValue(1.0d).build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric2.test")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
     expectLastCall();
     replay(mockPointHandler);
 
     // try gzipped plaintext stream over tcp
-    String payloadStr = "metric2.test 0 " + startTime + " source=test1\n" +
-        "metric2.test 1 " + (startTime + 1) + " source=test2\n";
+    String payloadStr =
+        "metric2.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test1\n"
+            + "metric2.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test2\n";
     Socket socket = SocketFactory.getDefault().createSocket("localhost", port);
     ByteArrayOutputStream baos = new ByteArrayOutputStream(payloadStr.length());
     GZIPOutputStream gzip = new GZIPOutputStream(baos);
@@ -163,176 +430,1327 @@ public void testWavefrontUnifiedPortHandlerGzippedPlaintextStream() throws Excep
     socket.getOutputStream().write(baos.toByteArray());
     socket.getOutputStream().flush();
     socket.close();
-    TimeUnit.MILLISECONDS.sleep(500);
-    verify(mockPointHandler);
+    verifyWithTimeout(500, mockPointHandler);
+
+    // secure test
+    reset(mockPointHandler);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric2.test")
+            .setHost("test3")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric2.test")
+            .setHost("test4")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
+    expectLastCall();
+    replay(mockPointHandler);
+
+    // try gzipped plaintext stream over tcp
+    payloadStr =
+        "metric2.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test3\n"
+            + "metric2.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test4\n";
+    socket = sslSocketFactory.createSocket("localhost", securePort);
+    baos = new ByteArrayOutputStream(payloadStr.length());
+    gzip = new GZIPOutputStream(baos);
+    gzip.write(payloadStr.getBytes("UTF-8"));
+    gzip.close();
+    socket.getOutputStream().write(baos.toByteArray());
+    socket.getOutputStream().flush();
+    socket.close();
+    verifyWithTimeout(500, mockPointHandler);
   }
 
   @Test
   public void testWavefrontUnifiedPortHandlerPlaintextOverHttp() throws Exception {
+    port = findAvailablePort(2888);
+    int securePort = findAvailablePort(2889);
+    int healthCheckPort = findAvailablePort(8881);
+    proxy.proxyConfig.privateCertPath =
+        getClass().getClassLoader().getResource("demo.cert").getPath();
+    proxy.proxyConfig.privateKeyPath =
+        getClass().getClassLoader().getResource("demo.key").getPath();
+    proxy.proxyConfig.tlsPorts = "1,23 , 4,   , " + securePort + "  ,6";
+    proxy.initSslContext();
+    proxy.proxyConfig.pushListenerPorts = port + "," + securePort;
+    proxy.proxyConfig.httpHealthCheckPath = "/health";
+    proxy.proxyConfig.httpHealthCheckPorts = String.valueOf(healthCheckPort);
+    proxy.proxyConfig.httpHealthCheckAllPorts = true;
+    proxy.healthCheckManager = new HealthCheckManagerImpl(proxy.proxyConfig);
+    SpanSampler sampler = new SpanSampler(new RateSampler(1.0D), () -> null);
+    proxy.startGraphiteListener(String.valueOf(port), mockHandlerFactory, null, sampler);
+    proxy.startGraphiteListener(String.valueOf(securePort), mockHandlerFactory, null, sampler);
+    proxy.startHealthCheckListener(healthCheckPort);
+    waitUntilListenerIsOnline(port);
+    waitUntilListenerIsOnline(securePort);
+    reset(mockPointHandler);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric3.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric3.test")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric3.test")
+            .setHost("test3")
+            .setTimestamp((alignedStartTimeEpochSeconds + 2) * 1000)
+            .setValue(2.0d)
+            .build());
+    expectLastCall();
+    replay(mockPointHandler);
+
+    // try http connection
+    String payloadStr =
+        "metric3.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test1\n"
+            + "metric3.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test2\n"
+            + "metric3.test 2 "
+            + (alignedStartTimeEpochSeconds + 2)
+            + " source=test3"; // note the lack of newline at the end!
+    assertEquals(202, httpPost("http://localhost:" + port, payloadStr));
+    assertEquals(200, httpGet("http://localhost:" + port + "/health"));
+    assertEquals(202, httpGet("http://localhost:" + port + "/health2"));
+    assertEquals(200, httpGet("http://localhost:" + healthCheckPort + "/health"));
+    assertEquals(404, httpGet("http://localhost:" + healthCheckPort + "/health2"));
+    verify(mockPointHandler);
+
+    // secure test
     reset(mockPointHandler);
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric3.test").setHost("test1").setTimestamp(startTime * 1000).setValue(0.0d).build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric3.test")
+            .setHost("test4")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
     expectLastCall();
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric3.test").setHost("test2").setTimestamp((startTime + 1) * 1000).setValue(1.0d).build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric3.test")
+            .setHost("test5")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
     expectLastCall();
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric3.test").setHost("test3").setTimestamp((startTime + 2) * 1000).setValue(2.0d).build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric3.test")
+            .setHost("test6")
+            .setTimestamp((alignedStartTimeEpochSeconds + 2) * 1000)
+            .setValue(2.0d)
+            .build());
     expectLastCall();
     replay(mockPointHandler);
 
     // try http connection
-    String payloadStr = "metric3.test 0 " + startTime + " source=test1\n" +
-        "metric3.test 1 " + (startTime + 1) + " source=test2\n" +
-        "metric3.test 2 " + (startTime + 2) + " source=test3"; // note the lack of newline at the end!
-    URL url = new URL("http://localhost:" + port);
-    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
-    connection.setRequestMethod("POST");
-    connection.setDoOutput(true);
-    connection.setDoInput(true);
-    BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(connection.getOutputStream(), "UTF-8"));
-    writer.write(payloadStr);
-    writer.flush();
-    writer.close();
-    logger.info("HTTP response code (plaintext content): " + connection.getResponseCode());
+    payloadStr =
+        "metric3.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test4\n"
+            + "metric3.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test5\n"
+            + "metric3.test 2 "
+            + (alignedStartTimeEpochSeconds + 2)
+            + " source=test6"; // note the lack of newline at the end!
+    assertEquals(202, httpPost("https://localhost:" + securePort, payloadStr));
     verify(mockPointHandler);
   }
 
   @Test
   public void testWavefrontUnifiedPortHandlerHttpGzipped() throws Exception {
+    port = findAvailablePort(2888);
+    int securePort = findAvailablePort(2889);
+    proxy.proxyConfig.privateCertPath =
+        getClass().getClassLoader().getResource("demo.cert").getPath();
+    proxy.proxyConfig.privateKeyPath =
+        getClass().getClassLoader().getResource("demo.key").getPath();
+    proxy.proxyConfig.tlsPorts = "1,23 , 4,   , " + securePort + "  ,6";
+    proxy.initSslContext();
+    proxy.proxyConfig.pushListenerPorts = port + "," + securePort;
+    SpanSampler sampler = new SpanSampler(new RateSampler(1.0D), () -> null);
+    proxy.startGraphiteListener(String.valueOf(port), mockHandlerFactory, null, sampler);
+    proxy.startGraphiteListener(String.valueOf(securePort), mockHandlerFactory, null, sampler);
+    waitUntilListenerIsOnline(port);
+    waitUntilListenerIsOnline(securePort);
     reset(mockPointHandler);
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric4.test").setHost("test1").setTimestamp(startTime * 1000).setValue(0.0d).build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
     expectLastCall();
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric4.test").setHost("test2").setTimestamp((startTime + 1) * 1000).setValue(1.0d).build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
     expectLastCall();
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric4.test").setHost("test3").setTimestamp((startTime + 2) * 1000).setValue(2.0d).build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test3")
+            .setTimestamp((alignedStartTimeEpochSeconds + 2) * 1000)
+            .setValue(2.0d)
+            .build());
     expectLastCall();
     replay(mockPointHandler);
 
     // try http connection with gzip
-    String payloadStr = "metric4.test 0 " + startTime + " source=test1\n" +
-        "metric4.test 1 " + (startTime + 1) + " source=test2\n" +
-        "metric4.test 2 " + (startTime + 2) + " source=test3"; // note the lack of newline at the end!
+    String payloadStr =
+        "metric4.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test1\n"
+            + "metric4.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test2\n"
+            + "metric4.test 2 "
+            + (alignedStartTimeEpochSeconds + 2)
+            + " source=test3"; // note the lack of newline at the end!
     gzippedHttpPost("http://localhost:" + port, payloadStr);
     verify(mockPointHandler);
+
+    reset(mockPointHandler);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric_4.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric_4.test")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric_4.test")
+            .setHost("test3")
+            .setTimestamp((alignedStartTimeEpochSeconds + 2) * 1000)
+            .setValue(2.0d)
+            .build());
+    expectLastCall();
+    replay(mockPointHandler);
+
+    // try secure http connection with gzip
+    payloadStr =
+        "metric_4.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test1\n"
+            + "metric_4.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test2\n"
+            + "metric_4.test 2 "
+            + (alignedStartTimeEpochSeconds + 2)
+            + " source=test3"; // note the lack of newline at the end!
+    gzippedHttpPost("https://localhost:" + securePort, payloadStr);
+    verify(mockPointHandler);
   }
 
   // test that histograms received on Wavefront port get routed to the correct handler
   @Test
-  public void testHistogramDataOnWavefrontUnifiedPortHandlerPlaintextUncompressed() throws Exception {
+  public void testHistogramDataOnWavefrontUnifiedPortHandlerPlaintextUncompressed()
+      throws Exception {
+    port = findAvailablePort(2888);
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(port);
+    proxy.startGraphiteListener(
+        proxy.proxyConfig.getPushListenerPorts(),
+        mockHandlerFactory,
+        null,
+        new SpanSampler(new RateSampler(1.0D), () -> null));
+    waitUntilListenerIsOnline(port);
     reset(mockHistogramHandler);
-    mockHistogramHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric.test.histo").setHost("test1").setTimestamp(startTime * 1000).setValue(
-            Histogram.newBuilder()
-                .setType(HistogramType.TDIGEST)
-                .setDuration(60000)
-                .setBins(ImmutableList.of(10.0d, 100.0d))
-                .setCounts(ImmutableList.of(5, 10))
-                .build())
-        .build());
-    mockHistogramHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric.test.histo").setHost("test2").setTimestamp((startTime + 60) * 1000).setValue(
-            Histogram.newBuilder()
-                .setType(HistogramType.TDIGEST)
-                .setDuration(60000)
-                .setBins(ImmutableList.of(20.0d, 30.0d, 40.0d))
-                .setCounts(ImmutableList.of(5, 6, 7))
-                .build())
-        .build());
+    mockHistogramHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.histo")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(
+                Histogram.newBuilder()
+                    .setType(HistogramType.TDIGEST)
+                    .setDuration(60000)
+                    .setBins(ImmutableList.of(10.0d, 100.0d))
+                    .setCounts(ImmutableList.of(5, 10))
+                    .build())
+            .build());
+    mockHistogramHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.histo")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 60) * 1000)
+            .setValue(
+                Histogram.newBuilder()
+                    .setType(HistogramType.TDIGEST)
+                    .setDuration(60000)
+                    .setBins(ImmutableList.of(20.0d, 30.0d, 40.0d))
+                    .setCounts(ImmutableList.of(5, 6, 7))
+                    .build())
+            .build());
     expectLastCall();
     replay(mockHistogramHandler);
 
     Socket socket = SocketFactory.getDefault().createSocket("localhost", port);
     BufferedOutputStream stream = new BufferedOutputStream(socket.getOutputStream());
-    String payloadStr = "!M " + startTime + " #5 10.0 #10 100.0 metric.test.histo source=test1\n" +
-        "!M " + (startTime + 60) + " #5 20.0 #6 30.0 #7 40.0 metric.test.histo source=test2\n";
+    String payloadStr =
+        "!M "
+            + alignedStartTimeEpochSeconds
+            + " #5 10.0 #10 100.0 metric.test.histo source=test1\n"
+            + "!M "
+            + (alignedStartTimeEpochSeconds + 60)
+            + " #5 20.0 #6 30.0 #7 40.0 metric.test.histo source=test2\n";
     stream.write(payloadStr.getBytes());
     stream.flush();
     socket.close();
-    TimeUnit.MILLISECONDS.sleep(500);
-    verify(mockHistogramHandler);
+    verifyWithTimeout(500, mockHistogramHandler);
   }
 
   // test Wavefront port handler with mixed payload: metrics, histograms, source tags
   @Test
-  public void testWavefrontUnifiedPortHandlerPlaintextUncompressedMixedDataPayload() throws Exception {
+  public void testWavefrontUnifiedPortHandlerPlaintextUncompressedMixedDataPayload()
+      throws Exception {
+    port = findAvailablePort(2888);
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(port);
+    proxy.startGraphiteListener(
+        proxy.proxyConfig.getPushListenerPorts(),
+        mockHandlerFactory,
+        null,
+        new SpanSampler(new RateSampler(1.0D), () -> null));
+    waitUntilListenerIsOnline(port);
     reset(mockHistogramHandler);
     reset(mockPointHandler);
     reset(mockSourceTagHandler);
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric.test.mixed").setHost("test2").setTimestamp((startTime + 1) * 1000).setValue(10d).build());
-    mockHistogramHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric.test.mixed").setHost("test1").setTimestamp(startTime * 1000).setValue(
-        Histogram.newBuilder()
-            .setType(HistogramType.TDIGEST)
-            .setDuration(60000)
-            .setBins(ImmutableList.of(10.0d, 100.0d))
-            .setCounts(ImmutableList.of(5, 10))
-            .build())
-        .build());
-    mockPointHandler.report(ReportPoint.newBuilder().setTable("dummy").
-        setMetric("metric.test.mixed").setHost("test2").setTimestamp((startTime + 1) * 1000).setValue(9d).build());
-    mockSourceTagHandler.report(ReportSourceTag.newBuilder().setSourceTagLiteral("SourceTag").setAction("save")
-        .setSource("testSource").setAnnotations(ImmutableList.of("newtag1", "newtag2")).setDescription("").build());
+    reset(mockEventHandler);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.mixed")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(10d)
+            .build());
+    mockHistogramHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.mixed")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(
+                Histogram.newBuilder()
+                    .setType(HistogramType.TDIGEST)
+                    .setDuration(60000)
+                    .setBins(ImmutableList.of(10.0d, 100.0d))
+                    .setCounts(ImmutableList.of(5, 10))
+                    .build())
+            .build());
+    mockEventHandler.report(
+        ReportEvent.newBuilder()
+            .setStartTime(alignedStartTimeEpochSeconds * 1000)
+            .setEndTime(alignedStartTimeEpochSeconds * 1000 + 1)
+            .setName("Event name for testing")
+            .setHosts(ImmutableList.of("host1", "host2"))
+            .setDimensions(ImmutableMap.of("multi", ImmutableList.of("bar", "baz")))
+            .setAnnotations(ImmutableMap.of("severity", "INFO"))
+            .setTags(ImmutableList.of("tag1"))
+            .build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.mixed")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(9d)
+            .build());
+    mockSourceTagHandler.report(
+        ReportSourceTag.newBuilder()
+            .setOperation(SourceOperationType.SOURCE_TAG)
+            .setAction(SourceTagAction.SAVE)
+            .setSource("testSource")
+            .setAnnotations(ImmutableList.of("newtag1", "newtag2"))
+            .build());
     expectLastCall();
     replay(mockPointHandler);
     replay(mockHistogramHandler);
+    replay(mockEventHandler);
 
     Socket socket = SocketFactory.getDefault().createSocket("localhost", port);
     BufferedOutputStream stream = new BufferedOutputStream(socket.getOutputStream());
-    String payloadStr = "metric.test.mixed 10.0 " + (startTime + 1) + " source=test2\n" +
-        "!M " + startTime + " #5 10.0 #10 100.0 metric.test.mixed source=test1\n" +
-        "@SourceTag action=save source=testSource newtag1 newtag2\n" +
-        "metric.test.mixed 9.0 " + (startTime + 1) + " source=test2\n";
+    String payloadStr =
+        "metric.test.mixed 10.0 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test2\n"
+            + "!M "
+            + alignedStartTimeEpochSeconds
+            + " #5 10.0 #10 100.0 metric.test.mixed source=test1\n"
+            + "@SourceTag action=save source=testSource newtag1 newtag2\n"
+            + "metric.test.mixed 9.0 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test2\n"
+            + "@Event "
+            + alignedStartTimeEpochSeconds
+            + " \"Event name for testing\" host=host1 host=host2 tag=tag1 "
+            + "severity=INFO multi=bar multi=baz\n";
     stream.write(payloadStr.getBytes());
     stream.flush();
     socket.close();
-    TimeUnit.MILLISECONDS.sleep(500);
-    verify(mockPointHandler);
-    verify(mockHistogramHandler);
+    verifyWithTimeout(500, mockPointHandler, mockHistogramHandler, mockEventHandler);
+  }
+
+  @Test
+  public void testWavefrontHandlerAsDDIEndpoint() throws Exception {
+    port = findAvailablePort(2978);
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(port);
+    proxy.proxyConfig.dataBackfillCutoffHours = 8640;
+    proxy.startGraphiteListener(
+        proxy.proxyConfig.getPushListenerPorts(),
+        mockHandlerFactory,
+        null,
+        new SpanSampler(new DurationSampler(5000), () -> null));
+    waitUntilListenerIsOnline(port);
+    String traceId = UUID.randomUUID().toString();
+    long timestamp1 = alignedStartTimeEpochSeconds * 1000000 + 12345;
+    long timestamp2 = alignedStartTimeEpochSeconds * 1000000 + 23456;
+
+    String payloadStr =
+        "metric4.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test1\n"
+            + "metric4.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test2\n"
+            + "metric4.test 2 "
+            + (alignedStartTimeEpochSeconds + 2)
+            + " source=test3"; // note the lack of newline at the end!
+    String histoData =
+        "!M "
+            + alignedStartTimeEpochSeconds
+            + " #5 10.0 #10 100.0 metric.test.histo source=test1\n"
+            + "!M "
+            + (alignedStartTimeEpochSeconds + 60)
+            + " #5 20.0 #6 30.0 #7 40.0 metric.test.histo source=test2";
+    String spanData =
+        "testSpanName parent=parent1 source=testsource spanId=testspanid "
+            + "traceId=\""
+            + traceId
+            + "\" parent=parent2 "
+            + alignedStartTimeEpochSeconds
+            + " "
+            + (alignedStartTimeEpochSeconds + 10);
+    String spanDataToDiscard =
+        "testSpanName parent=parent1 source=testsource spanId=testspanid "
+            + "traceId=\""
+            + traceId
+            + "\" parent=parent2 "
+            + alignedStartTimeEpochSeconds
+            + " "
+            + (alignedStartTimeEpochSeconds + 1);
+    String spanLogData =
+        "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}]}\n";
+    String spanLogDataWithSpanField =
+        "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\"}}],"
+            + "\"span\":\""
+            + escapeSpanData(spanData)
+            + "\"}\n";
+    String spanLogDataWithSpanFieldToDiscard =
+        "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}}],"
+            + "\"span\":\""
+            + escapeSpanData(spanDataToDiscard)
+            + "\"}\n";
+    String mixedData =
+        "@SourceTag action=save source=testSource newtag1 newtag2\n"
+            + "@Event "
+            + alignedStartTimeEpochSeconds
+            + " \"Event name for testing\" host=host1 host=host2 tag=tag1 "
+            + "severity=INFO multi=bar multi=baz\n"
+            + "!M "
+            + (alignedStartTimeEpochSeconds + 60)
+            + " #5 20.0 #6 30.0 #7 40.0 metric.test.histo source=test2\n"
+            + "metric4.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test1\n"
+            + spanLogData
+            + spanLogDataWithSpanField;
+
+    String invalidData =
+        "{\"spanId\"}\n@SourceTag\n@Event\n!M #5\nmetric.name\n"
+            + "metric5.test 0 1234567890 source=test1\n";
+
+    reset(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    expectLastCall().times(2);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
+    expectLastCall().times(2);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test3")
+            .setTimestamp((alignedStartTimeEpochSeconds + 2) * 1000)
+            .setValue(2.0d)
+            .build());
+    expectLastCall().times(2);
+    replay(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+
+    assertEquals(202, gzippedHttpPost("http://localhost:" + port + "/report", payloadStr));
+    assertEquals(
+        202, gzippedHttpPost("http://localhost:" + port + "/report?format=wavefront", payloadStr));
+    verify(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+
+    reset(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+    mockHistogramHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.histo")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(
+                Histogram.newBuilder()
+                    .setType(HistogramType.TDIGEST)
+                    .setDuration(60000)
+                    .setBins(ImmutableList.of(10.0d, 100.0d))
+                    .setCounts(ImmutableList.of(5, 10))
+                    .build())
+            .build());
+    expectLastCall();
+    mockHistogramHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.histo")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 60) * 1000)
+            .setValue(
+                Histogram.newBuilder()
+                    .setType(HistogramType.TDIGEST)
+                    .setDuration(60000)
+                    .setBins(ImmutableList.of(20.0d, 30.0d, 40.0d))
+                    .setCounts(ImmutableList.of(5, 6, 7))
+                    .build())
+            .build());
+    expectLastCall();
+    replay(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+
+    assertEquals(
+        202, gzippedHttpPost("http://localhost:" + port + "/report?format=histogram", histoData));
+    verify(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+
+    reset(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId(traceId)
+            .setSpanId("testspanid")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp1)
+                        .setFields(ImmutableMap.of("key", "value", "key2", "value2"))
+                        .build(),
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp2)
+                        .setFields(ImmutableMap.of("key3", "value3", "key4", "value4"))
+                        .build()))
+            .build());
+    expectLastCall();
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId(traceId)
+            .setSpanId("testspanid")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp1)
+                        .setFields(ImmutableMap.of("key", "value", "key2", "value2"))
+                        .build(),
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp2)
+                        .setFields(ImmutableMap.of("key3", "value3"))
+                        .build()))
+            .build());
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(alignedStartTimeEpochSeconds * 1000)
+            .setDuration(10000)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("parent", "parent1"), new Annotation("parent", "parent2")))
+            .build());
+    expectLastCall();
+    replay(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+
+    assertEquals(
+        202, gzippedHttpPost("http://localhost:" + port + "/report?format=trace", spanData));
+    assertEquals(
+        202, gzippedHttpPost("http://localhost:" + port + "/report?format=spanLogs", spanLogData));
+    assertEquals(
+        202,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/report?format=spanLogs", spanLogDataWithSpanField));
+    assertEquals(
+        202,
+        gzippedHttpPost("http://localhost:" + port + "/report?format=trace", spanDataToDiscard));
+    assertEquals(
+        202,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/report?format=spanLogs",
+            spanLogDataWithSpanFieldToDiscard));
+    verify(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+
+    reset(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+    mockSourceTagHandler.report(
+        ReportSourceTag.newBuilder()
+            .setOperation(SourceOperationType.SOURCE_TAG)
+            .setAction(SourceTagAction.SAVE)
+            .setSource("testSource")
+            .setAnnotations(ImmutableList.of("newtag1", "newtag2"))
+            .build());
+    expectLastCall();
+    mockEventHandler.report(
+        ReportEvent.newBuilder()
+            .setStartTime(alignedStartTimeEpochSeconds * 1000)
+            .setEndTime(alignedStartTimeEpochSeconds * 1000 + 1)
+            .setName("Event name for testing")
+            .setHosts(ImmutableList.of("host1", "host2"))
+            .setTags(ImmutableList.of("tag1"))
+            .setAnnotations(ImmutableMap.of("severity", "INFO"))
+            .setDimensions(ImmutableMap.of("multi", ImmutableList.of("bar", "baz")))
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    expectLastCall();
+    replay(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+
+    proxy
+        .entityPropertiesFactoryMap
+        .get("central")
+        .get(ReportableEntityType.HISTOGRAM)
+        .setFeatureDisabled(true);
+    assertEquals(
+        403, gzippedHttpPost("http://localhost:" + port + "/report?format=histogram", histoData));
+    proxy
+        .entityPropertiesFactoryMap
+        .get("central")
+        .get(ReportableEntityType.TRACE)
+        .setFeatureDisabled(true);
+    assertEquals(
+        403, gzippedHttpPost("http://localhost:" + port + "/report?format=trace", spanData));
+    proxy
+        .entityPropertiesFactoryMap
+        .get("central")
+        .get(ReportableEntityType.TRACE_SPAN_LOGS)
+        .setFeatureDisabled(true);
+    assertEquals(
+        403, gzippedHttpPost("http://localhost:" + port + "/report?format=spanLogs", spanLogData));
+    assertEquals(
+        403,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/report?format=spanLogs", spanLogDataWithSpanField));
+    assertEquals(202, gzippedHttpPost("http://localhost:" + port + "/report", mixedData));
+    verify(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+
+    reset(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+    mockSourceTagHandler.report(
+        ReportSourceTag.newBuilder()
+            .setOperation(SourceOperationType.SOURCE_TAG)
+            .setAction(SourceTagAction.SAVE)
+            .setSource("testSource")
+            .setAnnotations(ImmutableList.of("newtag1", "newtag2"))
+            .build());
+    expectLastCall();
+    mockEventHandler.report(
+        ReportEvent.newBuilder()
+            .setStartTime(alignedStartTimeEpochSeconds * 1000)
+            .setEndTime(alignedStartTimeEpochSeconds * 1000 + 1)
+            .setName("Event name for testing")
+            .setHosts(ImmutableList.of("host1", "host2"))
+            .setTags(ImmutableList.of("tag1"))
+            .setAnnotations(ImmutableMap.of("severity", "INFO"))
+            .setDimensions(ImmutableMap.of("multi", ImmutableList.of("bar", "baz")))
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    expectLastCall();
+    mockSourceTagHandler.reject(eq("@SourceTag"), anyString());
+    expectLastCall();
+    mockEventHandler.reject(eq("@Event"), anyString());
+    expectLastCall();
+    mockPointHandler.reject(eq("metric.name"), anyString());
+    expectLastCall();
+    mockPointHandler.reject(
+        eq(
+            ReportPoint.newBuilder()
+                .setTable("dummy")
+                .setMetric("metric5.test")
+                .setHost("test1")
+                .setTimestamp(1234567890000L)
+                .setValue(0.0d)
+                .build()),
+        startsWith("WF-402: Point outside of reasonable timeframe"));
+    expectLastCall();
+    replay(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+
+    assertEquals(
+        202,
+        gzippedHttpPost("http://localhost:" + port + "/report", mixedData + "\n" + invalidData));
+
+    verify(
+        mockPointHandler,
+        mockHistogramHandler,
+        mockTraceHandler,
+        mockTraceSpanLogsHandler,
+        mockSourceTagHandler,
+        mockEventHandler);
+  }
+
+  @Test
+  public void testTraceUnifiedPortHandlerPlaintextDebugSampling() throws Exception {
+    tracePort = findAvailablePort(3888);
+    proxy.proxyConfig.traceListenerPorts = String.valueOf(tracePort);
+    proxy.startTraceListener(
+        proxy.proxyConfig.getTraceListenerPorts(),
+        mockHandlerFactory,
+        new SpanSampler(new RateSampler(0.0D), () -> null));
+    waitUntilListenerIsOnline(tracePort);
+    reset(mockTraceHandler);
+    reset(mockTraceSpanLogsHandler);
+    String traceId = UUID.randomUUID().toString();
+    long timestamp1 = alignedStartTimeEpochSeconds * 1000000 + 12345;
+    long timestamp2 = alignedStartTimeEpochSeconds * 1000000 + 23456;
+    String spanData =
+        "testSpanName parent=parent1 source=testsource spanId=testspanid "
+            + "traceId=\""
+            + traceId
+            + "\" debug=true "
+            + alignedStartTimeEpochSeconds
+            + " "
+            + (alignedStartTimeEpochSeconds + 1)
+            + "\n";
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId(traceId)
+            .setSpanId("testspanid")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp1)
+                        .setFields(ImmutableMap.of("key", "value", "key2", "value2"))
+                        .build(),
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp2)
+                        .setFields(ImmutableMap.of("key3", "value3", "key4", "value4"))
+                        .build()))
+            .build());
+    expectLastCall();
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId(traceId)
+            .setSpanId("testspanid")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp1)
+                        .setFields(ImmutableMap.of("key", "value", "key2", "value2"))
+                        .build(),
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp2)
+                        .setFields(ImmutableMap.of("key3", "value3", "key4", "value4"))
+                        .build()))
+            .build());
+    expectLastCall();
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(alignedStartTimeEpochSeconds * 1000)
+            .setDuration(1000)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("parent", "parent1"), new Annotation("debug", "true")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler);
+    replay(mockTraceSpanLogsHandler);
+
+    Socket socket = SocketFactory.getDefault().createSocket("localhost", tracePort);
+    BufferedOutputStream stream = new BufferedOutputStream(socket.getOutputStream());
+    String payloadStr =
+        spanData
+            + "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}]}\n"
+            + "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}],"
+            + "\"span\":\""
+            + escapeSpanData(spanData)
+            + "\"}\n";
+    stream.write(payloadStr.getBytes());
+    stream.flush();
+    socket.close();
+    verifyWithTimeout(500, mockTraceHandler, mockTraceSpanLogsHandler);
   }
 
   @Test
   public void testTraceUnifiedPortHandlerPlaintext() throws Exception {
+    tracePort = findAvailablePort(3888);
+    proxy.proxyConfig.traceListenerPorts = String.valueOf(tracePort);
+    proxy.startTraceListener(
+        proxy.proxyConfig.getTraceListenerPorts(),
+        mockHandlerFactory,
+        new SpanSampler(new RateSampler(1.0D), () -> null));
+    waitUntilListenerIsOnline(tracePort);
     reset(mockTraceHandler);
+    reset(mockTraceSpanLogsHandler);
     String traceId = UUID.randomUUID().toString();
-    mockTraceHandler.report(Span.newBuilder().setCustomer("dummy").setStartMillis(startTime * 1000)
-        .setDuration(1000)
-        .setName("testSpanName")
-        .setSource("testsource")
-        .setSpanId("testspanid")
-        .setTraceId(traceId)
-        .setAnnotations(ImmutableList.of(new Annotation("parent", "parent1"), new Annotation("parent", "parent2")))
-        .build());
+    long timestamp1 = alignedStartTimeEpochSeconds * 1000000 + 12345;
+    long timestamp2 = alignedStartTimeEpochSeconds * 1000000 + 23456;
+    String spanData =
+        "testSpanName parent=parent1 source=testsource spanId=testspanid "
+            + "traceId=\""
+            + traceId
+            + "\" parent=parent2 "
+            + alignedStartTimeEpochSeconds
+            + " "
+            + (alignedStartTimeEpochSeconds + 1)
+            + "\n";
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId(traceId)
+            .setSpanId("testspanid")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp1)
+                        .setFields(ImmutableMap.of("key", "value", "key2", "value2"))
+                        .build(),
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp2)
+                        .setFields(ImmutableMap.of("key3", "value3", "key4", "value4"))
+                        .build()))
+            .build());
+    expectLastCall();
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId(traceId)
+            .setSpanId("testspanid")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp1)
+                        .setFields(ImmutableMap.of("key", "value", "key2", "value2"))
+                        .build(),
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp2)
+                        .setFields(ImmutableMap.of("key3", "value3", "key4", "value4"))
+                        .build()))
+            .build());
+    expectLastCall();
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(alignedStartTimeEpochSeconds * 1000)
+            .setDuration(1000)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("parent", "parent1"), new Annotation("parent", "parent2")))
+            .build());
     expectLastCall();
     replay(mockTraceHandler);
+    replay(mockTraceSpanLogsHandler);
 
     Socket socket = SocketFactory.getDefault().createSocket("localhost", tracePort);
     BufferedOutputStream stream = new BufferedOutputStream(socket.getOutputStream());
-    String payloadStr = "testSpanName parent=parent1 source=testsource spanId=testspanid " +
-        "traceId=" + traceId + " parent=parent2 " + startTime + " " + (startTime + 1) + "\n";
+    String payloadStr =
+        spanData
+            + "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}]}\n"
+            + "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}],"
+            + "\"span\":\""
+            + escapeSpanData(spanData)
+            + "\"}\n";
     stream.write(payloadStr.getBytes());
     stream.flush();
     socket.close();
-    TimeUnit.MILLISECONDS.sleep(500);
-    verify(mockTraceHandler);
+    verifyWithTimeout(500, mockTraceHandler, mockTraceSpanLogsHandler);
+  }
+
+  @Ignore
+  @Test
+  public void testCustomTraceUnifiedPortHandlerDerivedMetrics() throws Exception {
+    customTracePort = findAvailablePort(51233);
+    proxy.proxyConfig.customTracingListenerPorts = String.valueOf(customTracePort);
+    setUserPreprocessorForTraceDerivedREDMetrics(customTracePort);
+    proxy.startCustomTracingListener(
+        proxy.proxyConfig.getCustomTracingListenerPorts(),
+        mockHandlerFactory,
+        mockWavefrontSender,
+        new SpanSampler(new RateSampler(1.0D), () -> null));
+    waitUntilListenerIsOnline(customTracePort);
+    reset(mockTraceHandler);
+    reset(mockWavefrontSender);
+
+    String traceId = UUID.randomUUID().toString();
+    String spanData =
+        "testSpanName source=testsource spanId=testspanid "
+            + "traceId=\""
+            + traceId
+            + "\" "
+            + alignedStartTimeEpochSeconds
+            + " "
+            + (alignedStartTimeEpochSeconds + 1)
+            + "\n";
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(alignedStartTimeEpochSeconds * 1000)
+            .setDuration(1000)
+            .setName("testSpanName")
+            .setSource(PREPROCESSED_SOURCE_VALUE)
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("application", PREPROCESSED_APPLICATION_TAG_VALUE),
+                    new Annotation("service", PREPROCESSED_SERVICE_TAG_VALUE),
+                    new Annotation("cluster", PREPROCESSED_CLUSTER_TAG_VALUE),
+                    new Annotation("shard", PREPROCESSED_SHARD_TAG_VALUE)))
+            .build());
+    expectLastCall();
+
+    Capture> tagsCapture = EasyMock.newCapture();
+    mockWavefrontSender.sendMetric(
+        eq(HEART_BEAT_METRIC),
+        eq(1.0),
+        anyLong(),
+        eq(PREPROCESSED_SOURCE_VALUE),
+        EasyMock.capture(tagsCapture));
+    expectLastCall().anyTimes();
+    replay(mockTraceHandler, mockWavefrontSender);
+
+    Socket socket = SocketFactory.getDefault().createSocket("localhost", customTracePort);
+    BufferedOutputStream stream = new BufferedOutputStream(socket.getOutputStream());
+    stream.write(spanData.getBytes());
+    stream.flush();
+    socket.close();
+    // sleep to get around "Nothing captured yet" issue with heartbeat metric call.
+    Thread.sleep(100);
+    verifyWithTimeout(500, mockTraceHandler, mockWavefrontSender);
+    HashMap tagsReturned = tagsCapture.getValue();
+    assertEquals(PREPROCESSED_APPLICATION_TAG_VALUE, tagsReturned.get(APPLICATION_TAG_KEY));
+    assertEquals(PREPROCESSED_SERVICE_TAG_VALUE, tagsReturned.get(SERVICE_TAG_KEY));
+    assertEquals(PREPROCESSED_CLUSTER_TAG_VALUE, tagsReturned.get(CLUSTER_TAG_KEY));
+    assertEquals(PREPROCESSED_SHARD_TAG_VALUE, tagsReturned.get(SHARD_TAG_KEY));
+  }
+
+  private void setUserPreprocessorForTraceDerivedREDMetrics(int port) {
+    ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+    PreprocessorRuleMetrics preprocessorRuleMetrics = new PreprocessorRuleMetrics(null, null, null);
+    preprocessor
+        .forSpan()
+        .addTransformer(
+            new SpanAddAnnotationIfNotExistsTransformer(
+                "application",
+                PREPROCESSED_APPLICATION_TAG_VALUE,
+                x -> true,
+                preprocessorRuleMetrics));
+    preprocessor
+        .forSpan()
+        .addTransformer(
+            new SpanAddAnnotationIfNotExistsTransformer(
+                "service", PREPROCESSED_SERVICE_TAG_VALUE, x -> true, preprocessorRuleMetrics));
+    preprocessor
+        .forSpan()
+        .addTransformer(
+            new SpanAddAnnotationIfNotExistsTransformer(
+                "cluster", PREPROCESSED_CLUSTER_TAG_VALUE, x -> true, preprocessorRuleMetrics));
+    preprocessor
+        .forSpan()
+        .addTransformer(
+            new SpanAddAnnotationIfNotExistsTransformer(
+                "shard", PREPROCESSED_SHARD_TAG_VALUE, x -> true, preprocessorRuleMetrics));
+    preprocessor
+        .forSpan()
+        .addTransformer(
+            new SpanReplaceRegexTransformer(
+                "sourceName",
+                "^test.*",
+                PREPROCESSED_SOURCE_VALUE,
+                null,
+                null,
+                false,
+                x -> true,
+                preprocessorRuleMetrics));
+    Map userPreprocessorMap = new HashMap<>();
+    userPreprocessorMap.put(String.valueOf(port), preprocessor);
+    proxy.preprocessors.userPreprocessors = userPreprocessorMap;
+  }
+
+  @Test
+  public void testCustomTraceUnifiedPortHandlerPlaintext() throws Exception {
+    customTracePort = findAvailablePort(50000);
+    proxy.proxyConfig.customTracingListenerPorts = String.valueOf(customTracePort);
+    proxy.startCustomTracingListener(
+        proxy.proxyConfig.getCustomTracingListenerPorts(),
+        mockHandlerFactory,
+        mockWavefrontSender,
+        new SpanSampler(new RateSampler(1.0D), () -> null));
+    waitUntilListenerIsOnline(customTracePort);
+    reset(mockTraceHandler);
+    reset(mockTraceSpanLogsHandler);
+    reset(mockWavefrontSender);
+    String traceId = UUID.randomUUID().toString();
+    long timestamp1 = alignedStartTimeEpochSeconds * 1000000 + 12345;
+    long timestamp2 = alignedStartTimeEpochSeconds * 1000000 + 23456;
+    String spanData =
+        "testSpanName source=testsource spanId=testspanid "
+            + "traceId=\""
+            + traceId
+            + "\" application=application1 service=service1 "
+            + alignedStartTimeEpochSeconds
+            + " "
+            + (alignedStartTimeEpochSeconds + 1)
+            + "\n";
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId(traceId)
+            .setSpanId("testspanid")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp1)
+                        .setFields(ImmutableMap.of("key", "value", "key2", "value2"))
+                        .build(),
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp2)
+                        .setFields(ImmutableMap.of("key3", "value3", "key4", "value4"))
+                        .build()))
+            .build());
+    expectLastCall();
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId(traceId)
+            .setSpanId("testspanid")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp1)
+                        .setFields(ImmutableMap.of("key", "value", "key2", "value2"))
+                        .build(),
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp2)
+                        .setFields(ImmutableMap.of("key3", "value3", "key4", "value4"))
+                        .build()))
+            .build());
+    expectLastCall();
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(alignedStartTimeEpochSeconds * 1000)
+            .setDuration(1000)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("application", "application1"),
+                    new Annotation("service", "service1")))
+            .build());
+    expectLastCall();
+    Capture> tagsCapture = EasyMock.newCapture();
+    mockWavefrontSender.sendMetric(
+        eq(HEART_BEAT_METRIC), eq(1.0), anyLong(), eq("testsource"), EasyMock.capture(tagsCapture));
+    EasyMock.expectLastCall().anyTimes();
+    replay(mockTraceHandler, mockTraceSpanLogsHandler, mockWavefrontSender);
+    Socket socket = SocketFactory.getDefault().createSocket("localhost", customTracePort);
+    BufferedOutputStream stream = new BufferedOutputStream(socket.getOutputStream());
+    String payloadStr =
+        spanData
+            + "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}]}\n"
+            + "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}],"
+            + "\"span\":\""
+            + escapeSpanData(spanData)
+            + "\"}\n";
+    stream.write(payloadStr.getBytes());
+    stream.flush();
+    socket.close();
+    verifyWithTimeout(500, mockTraceHandler, mockTraceSpanLogsHandler, mockWavefrontSender);
+    HashMap tagsReturned = tagsCapture.getValue();
+    assertEquals("application1", tagsReturned.get(APPLICATION_TAG_KEY));
+    assertEquals("service1", tagsReturned.get(SERVICE_TAG_KEY));
+    assertEquals("none", tagsReturned.get(CLUSTER_TAG_KEY));
+    assertEquals("none", tagsReturned.get(SHARD_TAG_KEY));
   }
 
   @Test(timeout = 30000)
   public void testDataDogUnifiedPortHandler() throws Exception {
+    ddPort = findAvailablePort(4888);
+    proxy.proxyConfig.dataDogJsonPorts = String.valueOf(ddPort);
+    proxy.startDataDogListener(
+        proxy.proxyConfig.getDataDogJsonPorts(), mockHandlerFactory, mockHttpClient);
     int ddPort2 = findAvailablePort(4988);
     PushAgent proxy2 = new PushAgent();
-    proxy2.flushThreads = 2;
-    proxy2.retryThreads = 1;
-    proxy2.dataDogJsonPorts = String.valueOf(ddPort2);
-    proxy2.dataDogProcessSystemMetrics = true;
-    proxy2.dataDogProcessServiceChecks = false;
-    proxy2.dataDogRequestRelayTarget = "http://relay-to:1234";
-    proxy2.startDataDogListener(proxy2.dataDogJsonPorts, mockHandlerFactory, mockHttpClient);
-    TimeUnit.MILLISECONDS.sleep(500);
+    proxy2.proxyConfig.flushThreads = 2;
+    proxy2.proxyConfig.dataBackfillCutoffHours = 100000000;
+    proxy2.proxyConfig.dataDogJsonPorts = String.valueOf(ddPort2);
+    proxy2.proxyConfig.dataDogRequestRelaySyncMode = true;
+    proxy2.proxyConfig.dataDogProcessSystemMetrics = true;
+    proxy2.proxyConfig.dataDogProcessServiceChecks = false;
+    proxy2.proxyConfig.dataDogRequestRelayTarget = "http://relay-to:1234";
+    assertEquals(2, proxy2.proxyConfig.getFlushThreads());
+    assertTrue(proxy2.proxyConfig.isDataDogProcessSystemMetrics());
+    assertFalse(proxy2.proxyConfig.isDataDogProcessServiceChecks());
+
+    proxy2.startDataDogListener(
+        proxy2.proxyConfig.getDataDogJsonPorts(), mockHandlerFactory, mockHttpClient);
+    waitUntilListenerIsOnline(ddPort2);
+
+    int ddPort3 = findAvailablePort(4990);
+    PushAgent proxy3 = new PushAgent();
+    proxy3.proxyConfig.dataBackfillCutoffHours = 100000000;
+    proxy3.proxyConfig.dataDogJsonPorts = String.valueOf(ddPort3);
+    proxy3.proxyConfig.dataDogProcessSystemMetrics = true;
+    proxy3.proxyConfig.dataDogProcessServiceChecks = true;
+    assertTrue(proxy3.proxyConfig.isDataDogProcessSystemMetrics());
+    assertTrue(proxy3.proxyConfig.isDataDogProcessServiceChecks());
+
+    proxy3.startDataDogListener(
+        proxy3.proxyConfig.getDataDogJsonPorts(), mockHandlerFactory, mockHttpClient);
+    waitUntilListenerIsOnline(ddPort3);
 
     // test 1: post to /intake with system metrics enabled and http relay enabled
     HttpResponse mockHttpResponse = EasyMock.createMock(HttpResponse.class);
@@ -356,14 +1774,17 @@ public void testDataDogUnifiedPortHandler() throws Exception {
     gzippedHttpPost("http://localhost:" + ddPort + "/intake", getResource("ddTestSystem.json"));
     verify(mockPointHandler);
 
-    // test 3: post to /intake with system metrics enabled and http relay enabled, but remote unavailable
+    // test 3: post to /intake with system metrics enabled and http relay enabled, but remote
+    // unavailable
     reset(mockPointHandler, mockHttpClient, mockHttpResponse, mockStatusLine);
     expect(mockStatusLine.getStatusCode()).andReturn(404); // remote returns a error http code
     expect(mockHttpResponse.getStatusLine()).andReturn(mockStatusLine);
     expect(mockHttpResponse.getEntity()).andReturn(new StringEntity(""));
     expect(mockHttpClient.execute(anyObject(HttpPost.class))).andReturn(mockHttpResponse);
     mockPointHandler.report(anyObject());
-    expectLastCall().andThrow(new AssertionFailedError()).anyTimes(); // we are not supposed to actually process data!
+    expectLastCall()
+        .andThrow(new AssertionFailedError())
+        .anyTimes(); // we are not supposed to actually process data!
     replay(mockHttpClient, mockHttpResponse, mockStatusLine, mockPointHandler);
     gzippedHttpPost("http://localhost:" + ddPort2 + "/intake", getResource("ddTestSystem.json"));
     verify(mockHttpClient, mockPointHandler);
@@ -375,99 +1796,977 @@ public void testDataDogUnifiedPortHandler() throws Exception {
     expect(mockHttpResponse.getEntity()).andReturn(new StringEntity(""));
     expect(mockHttpClient.execute(anyObject(HttpPost.class))).andReturn(mockHttpResponse);
     mockPointHandler.report(anyObject());
-    expectLastCall().andThrow(new AssertionFailedError()).anyTimes(); // we are not supposed to actually process data!
+    expectLastCall()
+        .andThrow(new AssertionFailedError())
+        .anyTimes(); // we are not supposed to actually process data!
     replay(mockHttpClient, mockHttpResponse, mockStatusLine, mockPointHandler);
-    gzippedHttpPost("http://localhost:" + ddPort2 + "/api/v1/check_run", getResource("ddTestServiceCheck.json"));
+    gzippedHttpPost(
+        "http://localhost:" + ddPort2 + "/api/v1/check_run",
+        getResource("ddTestServiceCheck.json"));
     verify(mockHttpClient, mockPointHandler);
 
     // test 5: post to /api/v1/check_run with service checks enabled
     reset(mockPointHandler);
-    mockPointHandler.report(ReportPoint.newBuilder().
-        setTable("dummy").
-        setMetric("testApp.status").
-        setHost("testhost").
-        setTimestamp(1536719228000L).
-        setValue(3.0d).
-        build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("testApp.status")
+            .setHost("testhost")
+            .setTimestamp(1536719228000L)
+            .setValue(3.0d)
+            .build());
     expectLastCall().once();
     replay(mockPointHandler);
-    gzippedHttpPost("http://localhost:" + ddPort + "/api/v1/check_run", getResource("ddTestServiceCheck.json"));
+    gzippedHttpPost(
+        "http://localhost:" + ddPort + "/api/v1/check_run", getResource("ddTestServiceCheck.json"));
     verify(mockPointHandler);
 
-    // test 6: post to /api/v1/series
+    // test 6: post to /api/v1/series including a /api/v1/intake call to ensure system host-tags
+    // are
+    // propogated
     reset(mockPointHandler);
-    mockPointHandler.report(ReportPoint.newBuilder().
-        setTable("dummy").
-        setMetric("system.net.tcp.retrans_segs").
-        setHost("testhost").
-        setTimestamp(1531176936000L).
-        setValue(0.0d).
-        build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("system.net.tcp.retrans_segs")
+            .setHost("testhost")
+            .setTimestamp(1531176936000L)
+            .setValue(0.0d)
+            .setAnnotations(ImmutableMap.of("app", "closedstack", "role", "control"))
+            .build());
+    expectLastCall().once();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("system.net.tcp.listen_drops")
+            .setHost("testhost")
+            .setTimestamp(1531176936000L)
+            .setValue(0.0d)
+            .setAnnotations(
+                ImmutableMap.of(
+                    "_source", "Launcher", "env", "prod", "app", "openstack", "role", "control"))
+            .build());
     expectLastCall().once();
-    mockPointHandler.report(ReportPoint.newBuilder().
-        setTable("dummy").
-        setMetric("system.net.tcp.listen_drops").
-        setHost("testhost").
-        setTimestamp(1531176936000L).
-        setValue(0.0d).
-        build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("system.net.packets_in.count")
+            .setHost("testhost")
+            .setTimestamp(1531176936000L)
+            .setValue(12.052631578947368d)
+            .setAnnotations(
+                ImmutableMap.of("device", "eth0", "app", "closedstack", "role", "control"))
+            .build());
     expectLastCall().once();
-    mockPointHandler.report(ReportPoint.newBuilder().
-        setTable("dummy").
-        setMetric("system.net.packets_in.count").
-        setHost("testhost").
-        setTimestamp(1531176936000L).
-        setValue(12.052631578947368d).
-        build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("test.metric")
+            .setHost("testhost")
+            .setTimestamp(1531176936000L)
+            .setValue(400.0d)
+            .setAnnotations(ImmutableMap.of("app", "closedstack", "role", "control"))
+            .build());
     expectLastCall().once();
     replay(mockPointHandler);
-    gzippedHttpPost("http://localhost:" + ddPort + "/api/v1/series", getResource("ddTestTimeseries.json"));
+    gzippedHttpPost(
+        "http://localhost:" + ddPort3 + "/intake", getResource("ddTestSystemMetadataOnly.json"));
+    gzippedHttpPost(
+        "http://localhost:" + ddPort3 + "/api/v1/series", getResource("ddTestTimeseries.json"));
     verify(mockPointHandler);
 
     // test 7: post multiple checks to /api/v1/check_run with service checks enabled
     reset(mockPointHandler);
-    mockPointHandler.report(ReportPoint.newBuilder().
-        setTable("dummy").
-        setMetric("testApp.status").
-        setHost("testhost").
-        setTimestamp(1536719228000L).
-        setValue(3.0d).
-        build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("testApp.status")
+            .setHost("testhost")
+            .setTimestamp(1536719228000L)
+            .setValue(3.0d)
+            .build());
     expectLastCall().once();
-    mockPointHandler.report(ReportPoint.newBuilder().
-        setTable("dummy").
-        setMetric("testApp2.status").
-        setHost("testhost2").
-        setTimestamp(1536719228000L).
-        setValue(2.0d).
-        build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("testApp2.status")
+            .setHost("testhost2")
+            .setTimestamp(1536719228000L)
+            .setValue(2.0d)
+            .build());
     expectLastCall().once();
     replay(mockPointHandler);
-    gzippedHttpPost("http://localhost:" + ddPort + "/api/v1/check_run",
+    gzippedHttpPost(
+        "http://localhost:" + ddPort + "/api/v1/check_run",
         getResource("ddTestMultipleServiceChecks.json"));
     verify(mockPointHandler);
+  }
+
+  @Test
+  public void testDeltaCounterHandlerMixedData() throws Exception {
+    deltaPort = findAvailablePort(5888);
+    proxy.proxyConfig.deltaCountersAggregationListenerPorts = String.valueOf(deltaPort);
+    proxy.proxyConfig.deltaCountersAggregationIntervalSeconds = 10;
+    proxy.proxyConfig.pushFlushInterval = 100;
+    proxy.startDeltaCounterListener(
+        proxy.proxyConfig.getDeltaCountersAggregationListenerPorts(),
+        null,
+        mockSenderTaskFactory,
+        new SpanSampler(new RateSampler(1.0D), () -> null));
+    waitUntilListenerIsOnline(deltaPort);
+    reset(mockSenderTask);
+    Capture capturedArgument = Capture.newInstance(CaptureType.ALL);
+    mockSenderTask.add(EasyMock.capture(capturedArgument));
+    expectLastCall().atLeastOnce();
+    replay(mockSenderTask);
 
+    String payloadStr1 = "∆test.mixed1 1.0 source=test1\n";
+    String payloadStr2 = "∆test.mixed2 2.0 source=test1\n";
+    String payloadStr3 = "test.mixed3 3.0 source=test1\n";
+    String payloadStr4 = "∆test.mixed3 3.0 source=test1\n";
+    assertEquals(
+        202,
+        httpPost(
+            "http://localhost:" + deltaPort,
+            payloadStr1 + payloadStr2 + payloadStr2 + payloadStr3 + payloadStr4));
+    ReportableEntityHandler handler =
+        proxy.deltaCounterHandlerFactory.getHandler(
+            HandlerKey.of(ReportableEntityType.POINT, String.valueOf(deltaPort)));
+    if (handler instanceof DeltaCounterAccumulationHandlerImpl) {
+      ((DeltaCounterAccumulationHandlerImpl) handler).flushDeltaCounters();
+    }
+    verify(mockSenderTask);
+    assertEquals(3, capturedArgument.getValues().size());
+    assertTrue(capturedArgument.getValues().get(0).startsWith("\"∆test.mixed1\" 1.0"));
+    assertTrue(capturedArgument.getValues().get(1).startsWith("\"∆test.mixed2\" 4.0"));
+    assertTrue(capturedArgument.getValues().get(2).startsWith("\"∆test.mixed3\" 3.0"));
   }
 
-  private String getResource(String resourceName) throws Exception {
-    URL url = Resources.getResource("com.wavefront.agent/" + resourceName);
-    File myFile = new File(url.toURI());
-    return FileUtils.readFileToString(myFile, "UTF-8");
+  @Test
+  public void testDeltaCounterHandlerDataStream() throws Exception {
+    deltaPort = findAvailablePort(5888);
+    proxy.proxyConfig.deltaCountersAggregationListenerPorts = String.valueOf(deltaPort);
+    proxy.proxyConfig.deltaCountersAggregationIntervalSeconds = 10;
+    proxy.startDeltaCounterListener(
+        proxy.proxyConfig.getDeltaCountersAggregationListenerPorts(),
+        null,
+        mockSenderTaskFactory,
+        new SpanSampler(new RateSampler(1.0D), () -> null));
+    waitUntilListenerIsOnline(deltaPort);
+    reset(mockSenderTask);
+    Capture capturedArgument = Capture.newInstance(CaptureType.ALL);
+    mockSenderTask.add(EasyMock.capture(capturedArgument));
+    expectLastCall().atLeastOnce();
+    replay(mockSenderTask);
+
+    String payloadStr = "∆test.mixed 1.0 " + alignedStartTimeEpochSeconds + " source=test1\n";
+    assertEquals(202, httpPost("http://localhost:" + deltaPort, payloadStr + payloadStr));
+    ReportableEntityHandler handler =
+        proxy.deltaCounterHandlerFactory.getHandler(
+            HandlerKey.of(ReportableEntityType.POINT, String.valueOf(deltaPort)));
+    if (!(handler instanceof DeltaCounterAccumulationHandlerImpl)) fail();
+    ((DeltaCounterAccumulationHandlerImpl) handler).flushDeltaCounters();
+
+    assertEquals(202, httpPost("http://localhost:" + deltaPort, payloadStr));
+    assertEquals(202, httpPost("http://localhost:" + deltaPort, payloadStr + payloadStr));
+    ((DeltaCounterAccumulationHandlerImpl) handler).flushDeltaCounters();
+    verify(mockSenderTask);
+    assertEquals(2, capturedArgument.getValues().size());
+    assertTrue(capturedArgument.getValues().get(0).startsWith("\"∆test.mixed\" 2.0"));
+    assertTrue(capturedArgument.getValues().get(1).startsWith("\"∆test.mixed\" 3.0"));
   }
 
-  private void gzippedHttpPost(String postUrl, String payload) throws Exception {
-    URL url = new URL(postUrl);
-    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
-    connection.setRequestMethod("POST");
-    connection.setDoOutput(true);
-    connection.setDoInput(true);
-    connection.setRequestProperty("Content-Encoding", "gzip");
-    ByteArrayOutputStream baos = new ByteArrayOutputStream(payload.length());
-    GZIPOutputStream gzip = new GZIPOutputStream(baos);
-    gzip.write(payload.getBytes("UTF-8"));
-    gzip.close();
-    connection.getOutputStream().write(baos.toByteArray());
-    connection.getOutputStream().flush();
-    logger.info("HTTP response code (gzipped content): " + connection.getResponseCode());
+  @Test
+  public void testOpenTSDBPortHandler() throws Exception {
+    port = findAvailablePort(4242);
+    proxy.proxyConfig.opentsdbPorts = String.valueOf(port);
+    proxy.startOpenTsdbListener(proxy.proxyConfig.getOpentsdbPorts(), mockHandlerFactory);
+    waitUntilListenerIsOnline(port);
+    reset(mockPointHandler);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setAnnotations(ImmutableMap.of("env", "prod"))
+            .setValue(0.0d)
+            .build());
+    expectLastCall().times(2);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setAnnotations(ImmutableMap.of("env", "prod"))
+            .setValue(1.0d)
+            .build());
+    expectLastCall().times(2);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test3")
+            .setTimestamp((alignedStartTimeEpochSeconds + 2) * 1000)
+            .setAnnotations(ImmutableMap.of("env", "prod"))
+            .setValue(2.0d)
+            .build());
+    expectLastCall().times(2);
+    mockPointHandler.reject((ReportPoint) eq(null), anyString());
+    expectLastCall().once();
+    replay(mockPointHandler);
+
+    String payloadStr =
+        "[\n"
+            + "  {\n"
+            + "    \"metric\": \"metric4.test\",\n"
+            + "    \"timestamp\": "
+            + alignedStartTimeEpochSeconds
+            + ",\n"
+            + "    \"value\": 0.0,\n"
+            + "    \"tags\": {\n"
+            + "      \"host\": \"test1\",\n"
+            + "      \"env\": \"prod\"\n"
+            + "    }\n"
+            + "  },\n"
+            + "  {\n"
+            + "    \"metric\": \"metric4.test\",\n"
+            + "    \"timestamp\": "
+            + (alignedStartTimeEpochSeconds + 1)
+            + ",\n"
+            + "    \"value\": 1.0,\n"
+            + "    \"tags\": {\n"
+            + "      \"host\": \"test2\",\n"
+            + "      \"env\": \"prod\"\n"
+            + "    }\n"
+            + "  }\n"
+            + "]\n";
+    String payloadStr2 =
+        "[\n"
+            + "  {\n"
+            + "    \"metric\": \"metric4.test\",\n"
+            + "    \"timestamp\": "
+            + (alignedStartTimeEpochSeconds + 2)
+            + ",\n"
+            + "    \"value\": 2.0,\n"
+            + "    \"tags\": {\n"
+            + "      \"host\": \"test3\",\n"
+            + "      \"env\": \"prod\"\n"
+            + "    }\n"
+            + "  },\n"
+            + "  {\n"
+            + "    \"metric\": \"metric4.test\",\n"
+            + "    \"timestamp\": "
+            + alignedStartTimeEpochSeconds
+            + ",\n"
+            + "    \"tags\": {\n"
+            + "      \"host\": \"test4\",\n"
+            + "      \"env\": \"prod\"\n"
+            + "    }\n"
+            + "  }]\n";
+
+    Socket socket = SocketFactory.getDefault().createSocket("localhost", port);
+    BufferedOutputStream stream = new BufferedOutputStream(socket.getOutputStream());
+    String points =
+        "version\n"
+            + "put metric4.test "
+            + alignedStartTimeEpochSeconds
+            + " 0 host=test1 env=prod\n"
+            + "put metric4.test "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " 1 host=test2 env=prod\n"
+            + "put metric4.test "
+            + (alignedStartTimeEpochSeconds + 2)
+            + " 2 host=test3 env=prod\n";
+    stream.write(points.getBytes());
+    stream.flush();
+    socket.close();
+
+    // nonexistent path should return 400
+    assertEquals(400, gzippedHttpPost("http://localhost:" + port + "/api/nonexistent", ""));
+    assertEquals(200, gzippedHttpPost("http://localhost:" + port + "/api/version", ""));
+    // malformed json should return 400
+    assertEquals(400, gzippedHttpPost("http://localhost:" + port + "/api/put", "{]"));
+    assertEquals(204, gzippedHttpPost("http://localhost:" + port + "/api/put", payloadStr));
+    // 1 good, 1 invalid point - should return 400, but good point should still go through
+    assertEquals(400, gzippedHttpPost("http://localhost:" + port + "/api/put", payloadStr2));
+
+    verify(mockPointHandler);
+  }
+
+  @Test
+  public void testJsonMetricsPortHandler() throws Exception {
+    port = findAvailablePort(3878);
+    proxy.proxyConfig.jsonListenerPorts = String.valueOf(port);
+    proxy.startJsonListener(proxy.proxyConfig.jsonListenerPorts, mockHandlerFactory);
+    waitUntilListenerIsOnline(port);
+    reset(mockPointHandler);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test")
+            .setHost("testSource")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setAnnotations(ImmutableMap.of("env", "prod", "dc", "test1"))
+            .setValue(1.0d)
+            .build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.cpu.usage.idle")
+            .setHost("testSource")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(99.0d)
+            .build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.cpu.usage.user")
+            .setHost("testSource")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.5d)
+            .build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.cpu.usage.system")
+            .setHost("testSource")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.7d)
+            .build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.disk.free")
+            .setHost("testSource")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.mem.used")
+            .setHost("testSource")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(50.0d)
+            .build());
+    replay(mockPointHandler);
+
+    String payloadStr =
+        "{\n"
+            + "  \"value\": 1.0,\n"
+            + "  \"tags\": {\n"
+            + "    \"dc\": \"test1\",\n"
+            + "    \"env\": \"prod\"\n"
+            + "  }\n"
+            + "}\n";
+    String payloadStr2 =
+        "{\n"
+            + "  \"cpu.usage\": {\n"
+            + "    \"idle\": 99.0,\n"
+            + "    \"user\": 0.5,\n"
+            + "    \"system\": 0.7\n"
+            + "  },\n"
+            + "  \"disk.free\": 0.0,\n"
+            + "  \"mem\": {\n"
+            + "    \"used\": 50.0\n"
+            + "  }\n"
+            + "}\n";
+    assertEquals(
+        200,
+        gzippedHttpPost(
+            "http://localhost:"
+                + port
+                + "/?h=testSource&p=metric.test&"
+                + "d="
+                + alignedStartTimeEpochSeconds * 1000,
+            payloadStr));
+    assertEquals(
+        200,
+        gzippedHttpPost(
+            "http://localhost:"
+                + port
+                + "/?h=testSource&p=metric.test&"
+                + "d="
+                + alignedStartTimeEpochSeconds * 1000,
+            payloadStr2));
+    verify(mockPointHandler);
+  }
+
+  @Test
+  public void testOtlpHttpPortHandlerTraces() throws Exception {
+    port = findAvailablePort(4318);
+    proxy.proxyConfig.hostname = "defaultLocalHost";
+    SpanSampler mockSampler = EasyMock.createMock(SpanSampler.class);
+    proxy.startOtlpHttpListener(
+        String.valueOf(port), mockHandlerFactory, mockWavefrontSender, mockSampler);
+    waitUntilListenerIsOnline(port);
+
+    reset(mockSampler, mockTraceHandler, mockTraceSpanLogsHandler, mockWavefrontSender);
+    expect(mockSampler.sample(anyObject(), anyObject())).andReturn(true);
+    Capture actualSpan = EasyMock.newCapture();
+    Capture actualLogs = EasyMock.newCapture();
+    mockTraceHandler.report(capture(actualSpan));
+    mockTraceSpanLogsHandler.report(capture(actualLogs));
+
+    replay(mockSampler, mockTraceHandler, mockTraceSpanLogsHandler);
+
+    io.opentelemetry.proto.trace.v1.Span.Event otlpEvent = OtlpTestHelpers.otlpSpanEvent(0);
+    io.opentelemetry.proto.trace.v1.Span otlpSpan =
+        OtlpTestHelpers.otlpSpanGenerator().addEvents(otlpEvent).build();
+    ExportTraceServiceRequest otlpRequest = OtlpTestHelpers.otlpTraceRequest(otlpSpan);
+
+    String validUrl = "http://localhost:" + port + "/v1/traces";
+    assertEquals(200, httpPost(validUrl, otlpRequest.toByteArray(), "application/x-protobuf"));
+    assertEquals(400, httpPost(validUrl, "junk".getBytes(), "application/x-protobuf"));
+    assertEquals(
+        400,
+        httpPost(
+            "http://localhost:" + port + "/unknown",
+            otlpRequest.toByteArray(),
+            "application/x-protobuf"));
+    verify(mockSampler, mockTraceHandler, mockTraceSpanLogsHandler);
+
+    Span expectedSpan =
+        OtlpTestHelpers.wfSpanGenerator(Arrays.asList(new Annotation("_spanLogs", "true")))
+            .setSource("defaultLocalHost")
+            .build();
+    SpanLogs expectedLogs =
+        OtlpTestHelpers.wfSpanLogsGenerator(expectedSpan, 0, "_sampledByPolicy=NONE").build();
+
+    OtlpTestHelpers.assertWFSpanEquals(expectedSpan, actualSpan.getValue());
+    assertEquals(expectedLogs, actualLogs.getValue());
+  }
+
+  @Test
+  public void testOtlpHttpPortHandlerMetrics() throws Exception {
+    port = findAvailablePort(4318);
+    proxy.proxyConfig.hostname = "defaultLocalHost";
+    proxy.startOtlpHttpListener(String.valueOf(port), mockHandlerFactory, null, null);
+    waitUntilListenerIsOnline(port);
+
+    reset(mockPointHandler);
+
+    mockPointHandler.report(
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-gauge")
+            .setTimestamp(TimeUnit.SECONDS.toMillis(alignedStartTimeEpochSeconds))
+            .setValue(2.3)
+            .setHost("defaultLocalHost")
+            .build());
+    expectLastCall();
+
+    replay(mockPointHandler);
+    io.opentelemetry.proto.metrics.v1.Metric simpleGauge =
+        OtlpTestHelpers.otlpMetricGenerator()
+            .setName("test-gauge")
+            .setGauge(
+                Gauge.newBuilder()
+                    .addDataPoints(
+                        NumberDataPoint.newBuilder()
+                            .setAsDouble(2.3)
+                            .setTimeUnixNano(TimeUnit.SECONDS.toNanos(alignedStartTimeEpochSeconds))
+                            .build()))
+            .build();
+    ExportMetricsServiceRequest payload =
+        ExportMetricsServiceRequest.newBuilder()
+            .addResourceMetrics(
+                ResourceMetrics.newBuilder()
+                    .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(simpleGauge).build())
+                    .build())
+            .build();
+
+    String validUrl = "http://localhost:" + port + "/v1/metrics";
+    String invalidUrl = "http://localhost:" + port + "/blah";
+    assertEquals(400, httpPost(validUrl, "invalid payload".getBytes(), "application/x-protobuf"));
+    assertEquals(200, httpPost(validUrl, payload.toByteArray(), "application/x-protobuf"));
+    assertEquals(400, httpPost(invalidUrl, payload.toByteArray(), "application/x-protobuf"));
+    verify(mockPointHandler);
+  }
+
+  @Test
+  public void testOtlpGrpcHandlerCanListen() throws Exception {
+    port = findAvailablePort(4317);
+    SpanSampler mockSampler = EasyMock.createMock(SpanSampler.class);
+    proxy.startOtlpGrpcListener(
+        String.valueOf(port), mockHandlerFactory, mockWavefrontSender, mockSampler);
+    waitUntilListenerIsOnline(port);
+  }
+
+  @Test
+  public void testJaegerGrpcHandlerCanListen() throws Exception {
+    port = findAvailablePort(14250);
+    SpanSampler mockSampler = EasyMock.createMock(SpanSampler.class);
+    proxy.startTraceJaegerGrpcListener(
+        String.valueOf(port), mockHandlerFactory, mockWavefrontSender, mockSampler);
+    waitUntilListenerIsOnline(port);
+  }
+
+  @Test
+  public void testWriteHttpJsonMetricsPortHandler() throws Exception {
+    port = findAvailablePort(4878);
+    proxy.proxyConfig.writeHttpJsonListenerPorts = String.valueOf(port);
+    proxy.proxyConfig.hostname = "defaultLocalHost";
+    proxy.startWriteHttpJsonListener(
+        proxy.proxyConfig.writeHttpJsonListenerPorts, mockHandlerFactory);
+    waitUntilListenerIsOnline(port);
+    reset(mockPointHandler);
+    mockPointHandler.reject((ReportPoint) eq(null), anyString());
+    expectLastCall().times(2);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("disk.sda.disk_octets.read")
+            .setHost("testSource")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(197141504)
+            .build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("disk.sda.disk_octets.write")
+            .setHost("testSource")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(175136768)
+            .build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("disk.sda.disk_octets.read")
+            .setHost("defaultLocalHost")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(297141504)
+            .build());
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("disk.sda.disk_octets.write")
+            .setHost("defaultLocalHost")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(275136768)
+            .build());
+    replay(mockPointHandler);
+    String payloadStr =
+        "[{\n"
+            + "\"values\": [197141504, 175136768],\n"
+            + "\"dstypes\": [\"counter\", \"counter\"],\n"
+            + "\"dsnames\": [\"read\", \"write\"],\n"
+            + "\"time\": "
+            + alignedStartTimeEpochSeconds
+            + ",\n"
+            + "\"interval\": 10,\n"
+            + "\"host\": \"testSource\",\n"
+            + "\"plugin\": \"disk\",\n"
+            + "\"plugin_instance\": \"sda\",\n"
+            + "\"type\": \"disk_octets\",\n"
+            + "\"type_instance\": \"\"\n"
+            + "},{\n"
+            + "\"values\": [297141504, 275136768],\n"
+            + "\"dstypes\": [\"counter\", \"counter\"],\n"
+            + "\"dsnames\": [\"read\", \"write\"],\n"
+            + "\"time\": "
+            + alignedStartTimeEpochSeconds
+            + ",\n"
+            + "\"interval\": 10,\n"
+            + "\"plugin\": \"disk\",\n"
+            + "\"plugin_instance\": \"sda\",\n"
+            + "\"type\": \"disk_octets\",\n"
+            + "\"type_instance\": \"\"\n"
+            + "},{\n"
+            + "\"dstypes\": [\"counter\", \"counter\"],\n"
+            + "\"dsnames\": [\"read\", \"write\"],\n"
+            + "\"time\": "
+            + alignedStartTimeEpochSeconds
+            + ",\n"
+            + "\"interval\": 10,\n"
+            + "\"plugin\": \"disk\",\n"
+            + "\"plugin_instance\": \"sda\",\n"
+            + "\"type\": \"disk_octets\",\n"
+            + "\"type_instance\": \"\"\n"
+            + "}]\n";
+
+    // should be an array
+    assertEquals(400, gzippedHttpPost("http://localhost:" + port, "{}"));
+    assertEquals(200, gzippedHttpPost("http://localhost:" + port, payloadStr));
+    verify(mockPointHandler);
+  }
+
+  @Test
+  public void testRelayPortHandlerGzipped() throws Exception {
+    port = findAvailablePort(2888);
+    proxy.proxyConfig.pushRelayListenerPorts = String.valueOf(port);
+    proxy.proxyConfig.pushRelayHistogramAggregator = true;
+    proxy.proxyConfig.pushRelayHistogramAggregatorAccumulatorSize = 10L;
+    proxy.proxyConfig.pushRelayHistogramAggregatorFlushSecs = 1;
+    proxy.startRelayListener(
+        proxy.proxyConfig.getPushRelayListenerPorts(), mockHandlerFactory, null);
+    waitUntilListenerIsOnline(port);
+    reset(mockPointHandler, mockHistogramHandler, mockTraceHandler, mockTraceSpanLogsHandler);
+    String traceId = UUID.randomUUID().toString();
+    long timestamp1 = alignedStartTimeEpochSeconds * 1000000 + 12345;
+    long timestamp2 = alignedStartTimeEpochSeconds * 1000000 + 23456;
+    String spanData =
+        "testSpanName parent=parent1 source=testsource spanId=testspanid "
+            + "traceId=\""
+            + traceId
+            + "\" parent=parent2 "
+            + alignedStartTimeEpochSeconds
+            + " "
+            + (alignedStartTimeEpochSeconds + 1);
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(0.0d)
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test2")
+            .setTimestamp((alignedStartTimeEpochSeconds + 1) * 1000)
+            .setValue(1.0d)
+            .build());
+    expectLastCall();
+    mockPointHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric4.test")
+            .setHost("test3")
+            .setTimestamp((alignedStartTimeEpochSeconds + 2) * 1000)
+            .setValue(2.0d)
+            .build());
+    expectLastCall();
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId(traceId)
+            .setSpanId("testspanid")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp1)
+                        .setFields(ImmutableMap.of("key", "value", "key2", "value2"))
+                        .build(),
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp2)
+                        .setFields(ImmutableMap.of("key3", "value3", "key4", "value4"))
+                        .build()))
+            .build());
+    expectLastCall();
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId(traceId)
+            .setSpanId("testspanid")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp1)
+                        .setFields(ImmutableMap.of("key", "value", "key2", "value2"))
+                        .build(),
+                    SpanLog.newBuilder()
+                        .setTimestamp(timestamp2)
+                        .setFields(ImmutableMap.of("key3", "value3"))
+                        .build()))
+            .setSpan(spanData)
+            .build());
+    expectLastCall();
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(alignedStartTimeEpochSeconds * 1000)
+            .setDuration(1000)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("parent", "parent1"), new Annotation("parent", "parent2")))
+            .build());
+    expectLastCall();
+    mockPointHandler.reject(anyString(), anyString());
+    expectLastCall().times(2);
+
+    replay(mockPointHandler, mockHistogramHandler, mockTraceHandler, mockTraceSpanLogsHandler);
+
+    String payloadStr =
+        "metric4.test 0 "
+            + alignedStartTimeEpochSeconds
+            + " source=test1\n"
+            + "metric4.test 1 "
+            + (alignedStartTimeEpochSeconds + 1)
+            + " source=test2\n"
+            + "metric4.test 2 "
+            + (alignedStartTimeEpochSeconds + 2)
+            + " source=test3"; // note the lack of newline at the end!
+    String histoData =
+        "!M "
+            + alignedStartTimeEpochSeconds
+            + " #5 10.0 #10 100.0 metric.test.histo source=test1\n"
+            + "!M "
+            + (alignedStartTimeEpochSeconds + 60)
+            + " #5 20.0 #6 30.0 #7 40.0 metric.test.histo source=test2";
+    String spanLogData =
+        "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}]}\n";
+    String spanLogDataWithSpanField =
+        "{\"spanId\":\"testspanid\",\"traceId\":\""
+            + traceId
+            + "\",\"logs\":[{\"timestamp\":"
+            + timestamp1
+            + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":"
+            + timestamp2
+            + ",\"fields\":{\"key3\":\"value3\"}}],"
+            + "\"span\":\""
+            + escapeSpanData(spanData)
+            + "\"}\n";
+    String badData =
+        "@SourceTag action=save source=testSource newtag1 newtag2\n"
+            + "@Event "
+            + alignedStartTimeEpochSeconds
+            + " \"Event name for testing\" host=host1 host=host2 tag=tag1 "
+            + "severity=INFO multi=bar multi=baz\n"
+            + "!M "
+            + (alignedStartTimeEpochSeconds + 60)
+            + " #5 20.0 #6 30.0 #7 40.0 metric.test.histo source=test2";
+
+    assertEquals(
+        500,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/checkin",
+            "{}")); // apiContainer not available
+    assertEquals(
+        200,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/report?format=wavefront", payloadStr));
+    assertEquals(
+        200,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/report?format=histogram", histoData));
+    assertEquals(
+        200,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/report?format=trace", spanData));
+    assertEquals(
+        200,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/report?format=spanLogs", spanLogData));
+    assertEquals(
+        200,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/report?format=spanLogs",
+            spanLogDataWithSpanField));
+    proxy
+        .entityPropertiesFactoryMap
+        .get("central")
+        .get(ReportableEntityType.HISTOGRAM)
+        .setFeatureDisabled(true);
+    assertEquals(
+        403,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/report?format=histogram", histoData));
+    proxy
+        .entityPropertiesFactoryMap
+        .get("central")
+        .get(ReportableEntityType.TRACE)
+        .setFeatureDisabled(true);
+    assertEquals(
+        403,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/report?format=trace", spanData));
+    proxy
+        .entityPropertiesFactoryMap
+        .get("central")
+        .get(ReportableEntityType.TRACE_SPAN_LOGS)
+        .setFeatureDisabled(true);
+    assertEquals(
+        403,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/report?format=spanLogs", spanLogData));
+    assertEquals(
+        403,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/report?format=spanLogs",
+            spanLogDataWithSpanField));
+    assertEquals(
+        400,
+        gzippedHttpPost(
+            "http://localhost:" + port + "/api/v2/wfproxy/report?format=wavefront", badData));
+    verify(mockPointHandler, mockHistogramHandler, mockTraceHandler, mockTraceSpanLogsHandler);
+  }
+
+  @Test
+  public void testHealthCheckAdminPorts() throws Exception {
+    port = findAvailablePort(2888);
+    int port2 = findAvailablePort(3888);
+    int port3 = findAvailablePort(4888);
+    int port4 = findAvailablePort(5888);
+    int adminPort = findAvailablePort(6888);
+    proxy.proxyConfig.pushListenerPorts = port + "," + port2 + "," + port3 + "," + port4;
+    proxy.proxyConfig.adminApiListenerPort = adminPort;
+    proxy.proxyConfig.httpHealthCheckPath = "/health";
+    proxy.proxyConfig.httpHealthCheckAllPorts = true;
+    proxy.proxyConfig.httpHealthCheckFailStatusCode = 403;
+    proxy.healthCheckManager = new HealthCheckManagerImpl(proxy.proxyConfig);
+    SpanSampler sampler = new SpanSampler(new RateSampler(1.0D), () -> null);
+    proxy.startGraphiteListener(String.valueOf(port), mockHandlerFactory, null, sampler);
+    proxy.startGraphiteListener(String.valueOf(port2), mockHandlerFactory, null, sampler);
+    proxy.startGraphiteListener(String.valueOf(port3), mockHandlerFactory, null, sampler);
+    proxy.startGraphiteListener(String.valueOf(port4), mockHandlerFactory, null, sampler);
+    proxy.startAdminListener(adminPort);
+    waitUntilListenerIsOnline(adminPort);
+    assertEquals(404, httpGet("http://localhost:" + adminPort + "/"));
+    assertEquals(200, httpGet("http://localhost:" + port + "/health"));
+    assertEquals(200, httpGet("http://localhost:" + port2 + "/health"));
+    assertEquals(200, httpGet("http://localhost:" + port3 + "/health"));
+    assertEquals(200, httpGet("http://localhost:" + port4 + "/health"));
+    assertEquals(202, httpGet("http://localhost:" + port + "/health2"));
+    assertEquals(400, httpGet("http://localhost:" + adminPort + "/status"));
+    assertEquals(405, httpPost("http://localhost:" + adminPort + "/status", ""));
+    assertEquals(404, httpGet("http://localhost:" + adminPort + "/status/somethingelse"));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port2));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port3));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port4));
+    assertEquals(405, httpGet("http://localhost:" + adminPort + "/disable"));
+    assertEquals(405, httpGet("http://localhost:" + adminPort + "/enable"));
+    assertEquals(405, httpGet("http://localhost:" + adminPort + "/disable/" + port));
+    assertEquals(405, httpGet("http://localhost:" + adminPort + "/enable/" + port));
+
+    // disabling port and port3
+    assertEquals(200, httpPost("http://localhost:" + adminPort + "/disable/" + port, ""));
+    assertEquals(200, httpPost("http://localhost:" + adminPort + "/disable/" + port3, ""));
+    assertEquals(503, httpGet("http://localhost:" + adminPort + "/status/" + port));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port2));
+    assertEquals(503, httpGet("http://localhost:" + adminPort + "/status/" + port3));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port4));
+    assertEquals(403, httpGet("http://localhost:" + port + "/health"));
+    assertEquals(200, httpGet("http://localhost:" + port2 + "/health"));
+    assertEquals(403, httpGet("http://localhost:" + port3 + "/health"));
+    assertEquals(200, httpGet("http://localhost:" + port4 + "/health"));
+
+    // disable all
+    assertEquals(200, httpPost("http://localhost:" + adminPort + "/disable", ""));
+    assertEquals(503, httpGet("http://localhost:" + adminPort + "/status/" + port));
+    assertEquals(503, httpGet("http://localhost:" + adminPort + "/status/" + port2));
+    assertEquals(503, httpGet("http://localhost:" + adminPort + "/status/" + port3));
+    assertEquals(503, httpGet("http://localhost:" + adminPort + "/status/" + port4));
+    assertEquals(403, httpGet("http://localhost:" + port + "/health"));
+    assertEquals(403, httpGet("http://localhost:" + port2 + "/health"));
+    assertEquals(403, httpGet("http://localhost:" + port3 + "/health"));
+    assertEquals(403, httpGet("http://localhost:" + port4 + "/health"));
+
+    // enable port3 and port4
+    assertEquals(200, httpPost("http://localhost:" + adminPort + "/enable/" + port3, ""));
+    assertEquals(200, httpPost("http://localhost:" + adminPort + "/enable/" + port4, ""));
+    assertEquals(503, httpGet("http://localhost:" + adminPort + "/status/" + port));
+    assertEquals(503, httpGet("http://localhost:" + adminPort + "/status/" + port2));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port3));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port4));
+    assertEquals(403, httpGet("http://localhost:" + port + "/health"));
+    assertEquals(403, httpGet("http://localhost:" + port2 + "/health"));
+    assertEquals(200, httpGet("http://localhost:" + port3 + "/health"));
+    assertEquals(200, httpGet("http://localhost:" + port4 + "/health"));
+
+    // enable all
+    // enable port3 and port4
+    assertEquals(200, httpPost("http://localhost:" + adminPort + "/enable", ""));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port2));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port3));
+    assertEquals(200, httpGet("http://localhost:" + adminPort + "/status/" + port4));
+    assertEquals(200, httpGet("http://localhost:" + port + "/health"));
+    assertEquals(200, httpGet("http://localhost:" + port2 + "/health"));
+    assertEquals(200, httpGet("http://localhost:" + port3 + "/health"));
+    assertEquals(200, httpGet("http://localhost:" + port4 + "/health"));
+  }
+
+  @Test
+  public void testLargeHistogramDataOnWavefrontUnifiedPortHandler() throws Exception {
+    port = findAvailablePort(2988);
+    proxy.proxyConfig.pushListenerPorts = String.valueOf(port);
+    proxy.startGraphiteListener(
+        proxy.proxyConfig.getPushListenerPorts(),
+        mockHandlerFactory,
+        null,
+        new SpanSampler(new RateSampler(1.0D), () -> null));
+    waitUntilListenerIsOnline(port);
+    reset(mockHistogramHandler);
+    List bins = new ArrayList<>();
+    List counts = new ArrayList<>();
+    for (int i = 0; i < 50; i++) bins.add(10.0d);
+    for (int i = 0; i < 150; i++) bins.add(99.0d);
+    for (int i = 0; i < 200; i++) counts.add(1);
+    mockHistogramHandler.report(
+        ReportPoint.newBuilder()
+            .setTable("dummy")
+            .setMetric("metric.test.histo")
+            .setHost("test1")
+            .setTimestamp(alignedStartTimeEpochSeconds * 1000)
+            .setValue(
+                Histogram.newBuilder()
+                    .setType(HistogramType.TDIGEST)
+                    .setDuration(60000)
+                    .setBins(bins)
+                    .setCounts(counts)
+                    .build())
+            .build());
+    expectLastCall();
+    replay(mockHistogramHandler);
+
+    Socket socket = SocketFactory.getDefault().createSocket("localhost", port);
+    BufferedOutputStream stream = new BufferedOutputStream(socket.getOutputStream());
+    StringBuilder payloadStr = new StringBuilder("!M ");
+    payloadStr.append(alignedStartTimeEpochSeconds);
+    for (int i = 0; i < 50; i++) payloadStr.append(" #1 10.0");
+    for (int i = 0; i < 150; i++) payloadStr.append(" #1 99.0");
+    payloadStr.append(" metric.test.histo source=test1\n");
+    stream.write(payloadStr.toString().getBytes());
+    stream.flush();
+    socket.close();
+    verifyWithTimeout(500, mockHistogramHandler);
+  }
+
+  private String escapeSpanData(String spanData) {
+    return spanData.replace("\"", "\\\"").replace("\n", "\\n");
+  }
+
+  @Test
+  public void testIgnoreBackendSpanHeadSamplingPercent() {
+    proxy.proxyConfig.backendSpanHeadSamplingPercentIgnored = true;
+    proxy.proxyConfig.traceSamplingRate = 1.0;
+    AgentConfiguration agentConfiguration = new AgentConfiguration();
+    agentConfiguration.setSpanSamplingRate(0.5);
+    proxy.processConfiguration("cetnral", agentConfiguration);
+    assertEquals(
+        1.0,
+        proxy
+            .entityPropertiesFactoryMap
+            .get("central")
+            .getGlobalProperties()
+            .getTraceSamplingRate(),
+        1e-3);
+
+    proxy.proxyConfig.backendSpanHeadSamplingPercentIgnored = false;
+    proxy.processConfiguration("central", agentConfiguration);
+    assertEquals(
+        0.5,
+        proxy
+            .entityPropertiesFactoryMap
+            .get("central")
+            .getGlobalProperties()
+            .getTraceSamplingRate(),
+        1e-3);
   }
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/QueuedAgentServiceTest.java b/proxy/src/test/java/com/wavefront/agent/QueuedAgentServiceTest.java
deleted file mode 100644
index 6dbe75c15..000000000
--- a/proxy/src/test/java/com/wavefront/agent/QueuedAgentServiceTest.java
+++ /dev/null
@@ -1,722 +0,0 @@
-package com.wavefront.agent;
-
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.RecyclableRateLimiter;
-
-import com.squareup.tape.TaskInjector;
-import com.wavefront.agent.QueuedAgentService.PostPushDataResultTask;
-import com.wavefront.api.WavefrontAPI;
-import com.wavefront.api.agent.ShellOutputDTO;
-import com.wavefront.ingester.StringLineIngester;
-
-import net.jcip.annotations.NotThreadSafe;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.StringUtils;
-import org.easymock.EasyMock;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.Executors;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.ws.rs.core.Response;
-
-import io.netty.util.internal.StringUtil;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author Andrew Kao (andrew@wavefront.com)
- */
-@NotThreadSafe
-public class QueuedAgentServiceTest {
-
-  private QueuedAgentService queuedAgentService;
-  private WavefrontAPI mockAgentAPI;
-  private UUID newAgentId;
-  private AtomicInteger splitBatchSize = new AtomicInteger(50000);
-
-  @Before
-  public void testSetup() throws IOException {
-    mockAgentAPI = EasyMock.createMock(WavefrontAPI.class);
-    newAgentId = UUID.randomUUID();
-
-    int retryThreads = 1;
-    QueuedAgentService.setSplitBatchSize(splitBatchSize);
-
-    queuedAgentService = new QueuedAgentService(mockAgentAPI, "unitTestBuffer", retryThreads,
-        Executors.newScheduledThreadPool(retryThreads + 1, new ThreadFactory() {
-
-          private AtomicLong counter = new AtomicLong();
-
-          @Override
-          public Thread newThread(Runnable r) {
-            Thread toReturn = new Thread(r);
-            toReturn.setName("unit test submission worker: " + counter.getAndIncrement());
-            return toReturn;
-          }
-        }), true, newAgentId, false, (RecyclableRateLimiter) null, StringUtil.EMPTY_STRING);
-  }
-  // post sourcetag metadata
-
-  /**
-   * This test will try to delete a source tag and verify it works properly.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void postSourceTagDataPoint() throws Exception {
-    String id = "localhost";
-    String tagValue = "sourceTag1";
-    EasyMock.expect(mockAgentAPI.removeTag(id, StringUtils.EMPTY, tagValue)).andReturn(Response.ok().build()).once();
-    EasyMock.replay(mockAgentAPI);
-    Response response = queuedAgentService.removeTag(id, StringUtils.EMPTY, tagValue);
-    EasyMock.verify(mockAgentAPI);
-    assertEquals(Response.Status.OK.getStatusCode(), response.getStatus());
-  }
-
-  /**
-   * This test will try to delete a source tag, but makes sure it goes to the queue instead.
-   */
-  @Test
-  public void postSourceTagIntoQueue() {
-    String id = "localhost";
-    String tagValue = "sourceTag1";
-    Response response = queuedAgentService.removeTag(id, tagValue, true);
-    assertEquals(Response.Status.NOT_ACCEPTABLE.getStatusCode(), response.getStatus());
-    assertEquals(1, queuedAgentService.getQueuedSourceTagTasksCount());
-  }
-
-  /**
-   * This test will delete a description and verifies that the server side api was called properly
-   *
-   * @throws Exception
-   */
-  @Test
-  public void removeSourceDescription() throws Exception {
-    String id = "dummy";
-    EasyMock.expect(mockAgentAPI.removeDescription(id, StringUtils.EMPTY)).andReturn(Response.ok().build()).once();
-    EasyMock.replay(mockAgentAPI);
-    Response response = queuedAgentService.removeDescription(id, false);
-    EasyMock.verify(mockAgentAPI);
-    assertEquals("Response code was incorrect.", Response.Status.OK.getStatusCode(), response
-        .getStatus());
-  }
-
-  /**
-   * This test will add a description and make sure it goes into the queue instead of going to
-   * the server api directly.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void postSourceDescriptionIntoQueue() throws Exception {
-    String id = "localhost";
-    String desc = "A Description";
-    Response response = queuedAgentService.setDescription(id, desc, true);
-    assertEquals("Response code did not match", Response.Status.NOT_ACCEPTABLE.getStatusCode(),
-        response.getStatus());
-    assertEquals("No task found in the backlog queue", 1, queuedAgentService
-        .getQueuedSourceTagTasksCount());
-  }
-
-  /**
-   * This test will add 3 source tags and verify that the server api is called properly.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void postSourceTagsDataPoint() throws Exception {
-    String id = "dummy";
-    String tags[] = new String[]{"tag1", "tag2", "tag3"};
-    EasyMock.expect(mockAgentAPI.setTags(id, StringUtils.EMPTY,
-        Arrays.asList(tags))).andReturn(Response.ok().build()).once();
-    EasyMock.replay(mockAgentAPI);
-    Response response = queuedAgentService.setTags(id, Arrays.asList(tags), false);
-    EasyMock.verify(mockAgentAPI);
-    assertEquals(Response.Status.OK.getStatusCode(), response.getStatus());
-  }
-
-  /**
-   * This test is used to add a description to the source and verify that the server api is
-   * called accurately.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void postSourceDescriptionData() throws Exception {
-    String id = "dummy";
-    String desc = "A Description";
-    EasyMock.expect(mockAgentAPI.setDescription(id, StringUtils.EMPTY, desc)).andReturn(Response.ok()
-        .build()).once();
-    EasyMock.replay(mockAgentAPI);
-    Response response = queuedAgentService.setDescription(id, desc, false);
-    EasyMock.verify(mockAgentAPI);
-    assertEquals("Response code did not match.", Response.Status.OK.getStatusCode(),
-        response.getStatus());
-  }
-
-  /**
-   * This test will try to delete a source tag and mock a 406 response from the server. The delete
-   * request should get queued and tried again.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void postSourceTagAndHandle406Response() throws Exception{
-    // set up the mocks
-    String id = "localhost";
-    String tagValue = "sourceTag1";
-    EasyMock.expect(mockAgentAPI.removeTag(id, StringUtils.EMPTY, tagValue)).andReturn(Response.status(Response
-        .Status.NOT_ACCEPTABLE).build())
-        .once();
-    EasyMock.expect(mockAgentAPI.removeTag(id, StringUtils.EMPTY, tagValue)).andReturn(Response.status(Response
-        .Status.OK).build()).once();
-    EasyMock.replay(mockAgentAPI);
-    // call the api
-    Response response = queuedAgentService.removeTag(id, tagValue, false);
-    // verify
-    assertEquals(Response.Status.NOT_ACCEPTABLE.getStatusCode(), response.getStatus());
-    assertEquals(1, queuedAgentService.getQueuedSourceTagTasksCount());
-    // wait for a few seconds for the task to be picked up from the queue
-    TimeUnit.SECONDS.sleep(5);
-    EasyMock.verify(mockAgentAPI);
-    assertEquals(0, queuedAgentService.getQueuedSourceTagTasksCount());
-  }
-
-  /**
-   * This test will add source tags and mock a 406 response from the server. The add requests
-   * should get queued and tried out again.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void postSourceTagsAndHandle406Response() throws Exception {
-    String id = "dummy";
-    String tags[] = new String[]{"tag1", "tag2", "tag3"};
-    EasyMock.expect(mockAgentAPI.setTags(id, StringUtils.EMPTY, Arrays.asList(tags))).andReturn(
-        Response.status(Response.Status.NOT_ACCEPTABLE).build()).once();
-    EasyMock.expect(mockAgentAPI.setTags(id, StringUtils.EMPTY, Arrays.asList(tags))).andReturn(
-        Response.status(Response.Status.OK).build()).once();
-    EasyMock.replay(mockAgentAPI);
-    Response response = queuedAgentService.setTags(id, Arrays.asList(tags), false);
-    // verify
-    assertEquals(Response.Status.NOT_ACCEPTABLE.getStatusCode(), response.getStatus());
-    assertEquals(1, queuedAgentService.getQueuedSourceTagTasksCount());
-    // wait for a few seconds for the task to be picked up from the queue
-    TimeUnit.SECONDS.sleep(5);
-    EasyMock.verify(mockAgentAPI);
-    assertEquals(0, queuedAgentService.getQueuedSourceTagTasksCount());
-  }
-
-  /**
-   * This test will add source description and mock a 406 response from the server. The add
-   * requests should get queued and tried again.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void postSourceDescriptionAndHandle406Response() throws Exception {
-    String id = "dummy";
-    String description = "A Description";
-    EasyMock.expect(mockAgentAPI.setDescription(id, StringUtils.EMPTY, description)).andReturn(Response
-        .status
-        (Response.Status.NOT_ACCEPTABLE).build()).once();
-    EasyMock.expect(mockAgentAPI.setDescription(id, StringUtils.EMPTY, description)).andReturn(Response
-        .status
-        (Response.Status.OK).build()).once();
-    EasyMock.replay(mockAgentAPI);
-    Response response = queuedAgentService.setDescription(id, description, false);
-    // verify
-    assertEquals(Response.Status.NOT_ACCEPTABLE.getStatusCode(), response.getStatus());
-    assertEquals(1, queuedAgentService.getQueuedSourceTagTasksCount());
-    // wait for a few seconds for the task to be picked up from the queue
-    TimeUnit.SECONDS.sleep(5);
-    EasyMock.verify(mockAgentAPI);
-    assertEquals(0, queuedAgentService.getQueuedSourceTagTasksCount());
-  }
-
-  // postPushData
-  @Test
-  public void postPushDataCallsApproriateServiceMethodAndReturnsOK() {
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    List pretendPushDataList = new ArrayList();
-    pretendPushDataList.add("string line 1");
-    pretendPushDataList.add("string line 2");
-
-    String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-    EasyMock.expect(mockAgentAPI.postPushData(agentId, workUnitId, now, format, pretendPushData)).
-        andReturn(Response.ok().build()).once();
-    EasyMock.replay(mockAgentAPI);
-
-    Response response = queuedAgentService.postPushData(agentId, workUnitId, now, format, pretendPushData);
-
-    EasyMock.verify(mockAgentAPI);
-    assertEquals(Response.Status.OK.getStatusCode(), response.getStatus());
-  }
-
-  @Test
-  public void postPushDataServiceReturns406RequeuesAndReturnsNotAcceptable() {
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    List pretendPushDataList = new ArrayList();
-    pretendPushDataList.add("string line 1");
-    pretendPushDataList.add("string line 2");
-
-    String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-    EasyMock.expect(mockAgentAPI.postPushData(agentId, workUnitId, now, format, pretendPushData)).
-        andReturn(Response.status(Response.Status.NOT_ACCEPTABLE).build()).once();
-    EasyMock.replay(mockAgentAPI);
-
-    Response response = queuedAgentService.postPushData(agentId, workUnitId, now, format, pretendPushData);
-
-    EasyMock.verify(mockAgentAPI);
-    assertEquals(Response.Status.NOT_ACCEPTABLE.getStatusCode(), response.getStatus());
-    assertEquals(1, queuedAgentService.getQueuedTasksCount());
-  }
-
-  @Test
-  public void postPushDataServiceReturns413RequeuesAndReturnsNotAcceptableAndSplitsDataAndSuccessfullySendsIt() {
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    List pretendPushDataList = new ArrayList();
-
-    String str1 = "string line 1";
-    String str2 = "string line 2";
-
-    pretendPushDataList.add(str1);
-    pretendPushDataList.add(str2);
-
-    String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-    EasyMock.expect(mockAgentAPI.postPushData(agentId, workUnitId, now, format, pretendPushData)).
-        andReturn(Response.status(Response.Status.REQUEST_ENTITY_TOO_LARGE).build()).once();
-    EasyMock.expect(mockAgentAPI.postPushData(agentId, workUnitId, now, format, str1)).
-        andReturn(Response.ok().build()).once();
-    EasyMock.expect(mockAgentAPI.postPushData(agentId, workUnitId, now, format, str2)).
-        andReturn(Response.ok().build()).once();
-
-    EasyMock.replay(mockAgentAPI);
-
-    Response response = queuedAgentService.postPushData(agentId, workUnitId, now, format, pretendPushData);
-
-    EasyMock.verify(mockAgentAPI);
-    assertEquals(Response.Status.NOT_ACCEPTABLE.getStatusCode(), response.getStatus());
-    assertEquals(0, queuedAgentService.getQueuedTasksCount());
-  }
-
-  @Test
-  public void postPushDataServiceReturns413RequeuesAndReturnsNotAcceptableAndSplitsDataAndQueuesIfFailsAgain() {
-    // this is the same as the test above, but w/ the split still being too big
-    // -- and instead of spinning on resends, this should actually add it to the Q
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    List pretendPushDataList = new ArrayList();
-
-    String str1 = "string line 1";
-    String str2 = "string line 2";
-
-    pretendPushDataList.add(str1);
-    pretendPushDataList.add(str2);
-
-    String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-    EasyMock.expect(mockAgentAPI.postPushData(agentId, workUnitId, now, format, pretendPushData)).andReturn(Response
-        .status(Response.Status.REQUEST_ENTITY_TOO_LARGE).build()).once();
-    EasyMock.expect(mockAgentAPI.postPushData(agentId, workUnitId, now, format, str1)).andReturn(Response.status
-        (Response.Status.REQUEST_ENTITY_TOO_LARGE).build()).once();
-    EasyMock.expect(mockAgentAPI.postPushData(agentId, workUnitId, now, format, str2)).andReturn(Response.status
-        (Response.Status.REQUEST_ENTITY_TOO_LARGE).build()).once();
-
-    EasyMock.replay(mockAgentAPI);
-
-    Response response = queuedAgentService.postPushData(agentId, workUnitId, now, format, pretendPushData);
-
-    EasyMock.verify(mockAgentAPI);
-    assertEquals(Response.Status.NOT_ACCEPTABLE.getStatusCode(), response.getStatus());
-    assertEquals(2, queuedAgentService.getQueuedTasksCount());
-  }
-
-  private void injectServiceToResubmissionTask(ResubmissionTask task) {
-    new TaskInjector() {
-      @Override
-      public void injectMembers(ResubmissionTask task) {
-        task.service = mockAgentAPI;
-        task.currentAgentId = newAgentId;
-      }
-    }.injectMembers(task);
-  }
-
-  // *******
-  // ** PostPushDataResultTask
-  // *******
-  @Test
-  public void postPushDataResultTaskExecuteCallsAppropriateService() {
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    List pretendPushDataList = new ArrayList();
-    pretendPushDataList.add("string line 1");
-    pretendPushDataList.add("string line 2");
-
-    String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-    QueuedAgentService.PostPushDataResultTask task = new QueuedAgentService.PostPushDataResultTask(
-        agentId,
-        workUnitId,
-        now,
-        format,
-        pretendPushData
-    );
-
-    injectServiceToResubmissionTask(task);
-
-    EasyMock.expect(mockAgentAPI.postPushData(newAgentId, workUnitId, now, format, pretendPushData))
-        .andReturn(Response.ok().build()).once();
-
-    EasyMock.replay(mockAgentAPI);
-
-    task.execute(null);
-
-    EasyMock.verify(mockAgentAPI);
-  }
-
-  @Test
-  public void postPushDataResultTaskExecuteServiceReturns406ThrowsException() {
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    List pretendPushDataList = new ArrayList();
-    pretendPushDataList.add("string line 1");
-    pretendPushDataList.add("string line 2");
-
-    String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-    QueuedAgentService.PostPushDataResultTask task = new QueuedAgentService.PostPushDataResultTask(
-        agentId,
-        workUnitId,
-        now,
-        format,
-        pretendPushData
-    );
-
-    injectServiceToResubmissionTask(task);
-
-    EasyMock.expect(mockAgentAPI.postPushData(newAgentId, workUnitId, now, format, pretendPushData))
-        .andReturn(Response.status(Response.Status.NOT_ACCEPTABLE).build()).once();
-
-    EasyMock.replay(mockAgentAPI);
-
-    boolean exceptionThrown = false;
-
-    try {
-      task.execute(null);
-    } catch (RejectedExecutionException e) {
-      exceptionThrown = true;
-    }
-
-    assertTrue(exceptionThrown);
-    EasyMock.verify(mockAgentAPI);
-  }
-
-  @Test
-  public void postPushDataResultTaskExecuteServiceReturns413ThrowsException() {
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    List pretendPushDataList = new ArrayList();
-    pretendPushDataList.add("string line 1");
-    pretendPushDataList.add("string line 2");
-
-    String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-    QueuedAgentService.PostPushDataResultTask task = new QueuedAgentService.PostPushDataResultTask(
-        agentId,
-        workUnitId,
-        now,
-        format,
-        pretendPushData
-    );
-
-    injectServiceToResubmissionTask(task);
-
-    EasyMock.expect(mockAgentAPI.postPushData(newAgentId, workUnitId, now, format, pretendPushData))
-        .andReturn(Response.status(Response.Status.REQUEST_ENTITY_TOO_LARGE).build()).once();
-
-    EasyMock.replay(mockAgentAPI);
-
-    boolean exceptionThrown = false;
-
-    try {
-      task.execute(null);
-    } catch (QueuedPushTooLargeException e) {
-      exceptionThrown = true;
-    }
-
-    assertTrue(exceptionThrown);
-    EasyMock.verify(mockAgentAPI);
-  }
-
-  @Test
-  public void postPushDataResultTaskSplitReturnsListOfOneIfOnlyOne() {
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    String str1 = "string line 1";
-
-    List pretendPushDataList = new ArrayList();
-    pretendPushDataList.add(str1);
-
-    String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-    QueuedAgentService.PostPushDataResultTask task = new QueuedAgentService.PostPushDataResultTask(
-        agentId,
-        workUnitId,
-        now,
-        format,
-        pretendPushData
-    );
-
-    List splitTasks = task.splitTask();
-    assertEquals(1, splitTasks.size());
-
-    String firstSplitDataString = splitTasks.get(0).getPushData();
-    List firstSplitData = StringLineIngester.unjoinPushData(firstSplitDataString);
-
-    assertEquals(1, firstSplitData.size());
-  }
-
-  @Test
-  public void postPushDataResultTaskSplitTaskSplitsEvenly() {
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    String str1 = "string line 1";
-    String str2 = "string line 2";
-    String str3 = "string line 3";
-    String str4 = "string line 4";
-
-    List pretendPushDataList = new ArrayList();
-    pretendPushDataList.add(str1);
-    pretendPushDataList.add(str2);
-    pretendPushDataList.add(str3);
-    pretendPushDataList.add(str4);
-
-    String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-    PostPushDataResultTask task = new PostPushDataResultTask(
-        agentId,
-        workUnitId,
-        now,
-        format,
-        pretendPushData
-    );
-
-    List splitTasks = task.splitTask();
-    assertEquals(2, splitTasks.size());
-
-    String firstSplitDataString = splitTasks.get(0).getPushData();
-    List firstSplitData = StringLineIngester.unjoinPushData(firstSplitDataString);
-    assertEquals(2, firstSplitData.size());
-
-    String secondSplitDataString = splitTasks.get(1).getPushData();
-    List secondSplitData = StringLineIngester.unjoinPushData(secondSplitDataString);
-    assertEquals(2, secondSplitData.size());
-
-    // and all the data is the same...
-    for (ResubmissionTask taskUnderTest : splitTasks) {
-      PostPushDataResultTask taskUnderTestCasted = (PostPushDataResultTask) taskUnderTest;
-      assertEquals(agentId, taskUnderTestCasted.getAgentId());
-      assertEquals(workUnitId, taskUnderTestCasted.getWorkUnitId());
-      assertEquals(now, (long) taskUnderTestCasted.getCurrentMillis());
-      assertEquals(format, taskUnderTestCasted.getFormat());
-    }
-
-    // first list should have the first 2 strings
-    assertEquals(StringLineIngester.joinPushData(Arrays.asList(str1, str2)), firstSplitDataString);
-    // second list should have the last 2
-    assertEquals(StringLineIngester.joinPushData(Arrays.asList(str3, str4)), secondSplitDataString);
-
-  }
-
-  @Test
-  public void postPushDataResultTaskSplitsRoundsUpToLastElement() {
-    /* its probably sufficient to test that all the points get included in the split
-    as opposed to ensuring how they get split */
-
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    String str1 = "string line 1";
-    String str2 = "string line 2";
-    String str3 = "string line 3";
-    String str4 = "string line 4";
-    String str5 = "string line 5";
-
-    List pretendPushDataList = new ArrayList();
-    pretendPushDataList.add(str1);
-    pretendPushDataList.add(str2);
-    pretendPushDataList.add(str3);
-    pretendPushDataList.add(str4);
-    pretendPushDataList.add(str5);
-
-    String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-    PostPushDataResultTask task = new PostPushDataResultTask(
-        agentId,
-        workUnitId,
-        now,
-        format,
-        pretendPushData
-    );
-
-    List splitTasks = task.splitTask();
-    assertEquals(2, splitTasks.size());
-
-    String firstSplitDataString = splitTasks.get(0).getPushData();
-    List firstSplitData = StringLineIngester.unjoinPushData(firstSplitDataString);
-
-    assertEquals(3, firstSplitData.size());
-
-    String secondSplitDataString = splitTasks.get(1).getPushData();
-    List secondSplitData = StringLineIngester.unjoinPushData(secondSplitDataString);
-
-    assertEquals(2, secondSplitData.size());
-  }
-
-  @Test
-  public void postPushDataResultTaskSplitsIntoManyTask() {
-    for (int targetBatchSize = 1; targetBatchSize <= 10; targetBatchSize++) {
-      splitBatchSize.set(targetBatchSize);
-
-      UUID agentId = UUID.randomUUID();
-      UUID workUnitId = UUID.randomUUID();
-
-      long now = System.currentTimeMillis();
-
-      String format = "unitTestFormat";
-
-      for (int numTestStrings = 1; numTestStrings <= 51; numTestStrings += 1) {
-        List pretendPushDataList = new ArrayList();
-        for (int i = 0; i < numTestStrings; i++) {
-          pretendPushDataList.add(RandomStringUtils.randomAlphabetic(6));
-        }
-
-        String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-        PostPushDataResultTask task = new PostPushDataResultTask(
-            agentId,
-            workUnitId,
-            now,
-            format,
-            pretendPushData
-        );
-
-        List splitTasks = task.splitTask();
-        Set splitData = Sets.newHashSet();
-        for (PostPushDataResultTask taskN : splitTasks) {
-          List dataStrings = StringLineIngester.unjoinPushData(taskN.getPushData());
-          splitData.addAll(dataStrings);
-          assertTrue(dataStrings.size() <= targetBatchSize + 1);
-        }
-        assertEquals(Sets.newHashSet(pretendPushDataList), splitData);
-      }
-    }
-  }
-
-  @Test
-  public void splitIntoTwoTest() {
-    splitBatchSize.set(10000000);
-
-    UUID agentId = UUID.randomUUID();
-    UUID workUnitId = UUID.randomUUID();
-
-    long now = System.currentTimeMillis();
-
-    String format = "unitTestFormat";
-
-    for (int numTestStrings = 2; numTestStrings <= 51; numTestStrings += 1) {
-      List pretendPushDataList = new ArrayList();
-      for (int i = 0; i < numTestStrings; i++) {
-        pretendPushDataList.add(RandomStringUtils.randomAlphabetic(6));
-      }
-
-      String pretendPushData = StringLineIngester.joinPushData(pretendPushDataList);
-
-      PostPushDataResultTask task = new PostPushDataResultTask(
-          agentId,
-          workUnitId,
-          now,
-          format,
-          pretendPushData
-      );
-
-      List splitTasks = task.splitTask();
-      assertEquals(2, splitTasks.size());
-      Set splitData = Sets.newHashSet();
-      for (PostPushDataResultTask taskN : splitTasks) {
-        List dataStrings = StringLineIngester.unjoinPushData(taskN.getPushData());
-        splitData.addAll(dataStrings);
-        assertTrue(dataStrings.size() <= numTestStrings / 2 + 1);
-      }
-      assertEquals(Sets.newHashSet(pretendPushDataList), splitData);
-    }
-  }
-}
diff --git a/proxy/src/test/java/com/wavefront/agent/TenantInfoTest.java b/proxy/src/test/java/com/wavefront/agent/TenantInfoTest.java
new file mode 100644
index 000000000..eefa318ea
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/TenantInfoTest.java
@@ -0,0 +1,287 @@
+package com.wavefront.agent;
+
+import static com.wavefront.agent.api.APIContainer.CENTRAL_TENANT_NAME;
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.agent.api.CSPAPI;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.UUID;
+import javax.ws.rs.core.Response;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Unit tests for {@link TokenWorkerCSP}.
+ *
+ * @author Norayr Chaparyan(nchaparyan@vmware.com).
+ */
+public class TenantInfoTest {
+  private static final String wfServer = "wfServer";
+  private final CSPAPI cspApi = EasyMock.createMock(CSPAPI.class);
+  private final Response mockResponse = EasyMock.createMock(Response.class);
+  private final Response.StatusType statusTypeMock = EasyMock.createMock(Response.StatusType.class);
+  private final TokenExchangeResponseDTO mockTokenExchangeResponseDTO =
+      EasyMock.createMock(TokenExchangeResponseDTO.class);
+
+  @After
+  public void cleanup() {
+    TokenManager.reset();
+    reset(mockResponse, statusTypeMock, mockTokenExchangeResponseDTO, cspApi);
+  }
+
+  private void replayAll() {
+    replay(mockResponse, statusTypeMock, mockTokenExchangeResponseDTO, cspApi);
+  }
+
+  private void verifyAll() {
+    verify(mockResponse, statusTypeMock, mockTokenExchangeResponseDTO, cspApi);
+  }
+
+  @Test
+  public void testRun_SuccessfulResponseWavefrontAPIToken() {
+    final String wfToken = "wavefront-api-token";
+    // Replay mocks
+    replayAll();
+
+    TenantInfo tokenWorkerCSP = new TokenWorkerWF(wfToken, wfServer);
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, tokenWorkerCSP);
+    TokenManager.start(new APIContainer(null, null, null, null, cspApi));
+
+    // Verify the results
+    assertEquals(wfToken, tokenWorkerCSP.getBearerToken());
+    assertEquals(wfServer, tokenWorkerCSP.getWFServer());
+    verifyAll();
+  }
+
+  @Test
+  public void testRun_SuccessfulResponseUsingOAuthApp() {
+    TokenWorkerCSP tokenWorkerCSP = new TokenWorkerCSP("appId", "appSecret", "orgId", wfServer);
+
+    // Set up expectations
+    expect(mockResponse.getStatusInfo()).andReturn(statusTypeMock).times(2);
+    expect(statusTypeMock.getStatusCode()).andReturn(200);
+    expect(statusTypeMock.getFamily()).andReturn(Response.Status.Family.SUCCESSFUL);
+    expect(mockResponse.readEntity(TokenExchangeResponseDTO.class))
+        .andReturn(mockTokenExchangeResponseDTO);
+    expect(mockTokenExchangeResponseDTO.getAccessToken()).andReturn("newAccessToken");
+    expect(mockTokenExchangeResponseDTO.getExpiresIn()).andReturn(600);
+    expect(
+            cspApi.getTokenByClientCredentialsWithOrgId(
+                "Basic YXBwSWQ6YXBwU2VjcmV0", "client_credentials", "orgId"))
+        .andReturn(mockResponse)
+        .times(1);
+    replayAll();
+
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, tokenWorkerCSP);
+    TokenManager.start(new APIContainer(null, null, null, null, cspApi));
+
+    // Verify the results
+    assertEquals("newAccessToken", tokenWorkerCSP.getBearerToken());
+    assertEquals(wfServer, tokenWorkerCSP.getWFServer());
+    verifyAll();
+  }
+
+  @Test
+  public void testRun_SuccessfulResponseUsingOAuthAppWithBlankOrgId() {
+    TokenWorkerCSP tokenWorkerCSP = new TokenWorkerCSP("appId", "appSecret", "", wfServer);
+
+    // Set up expectations
+    expect(mockResponse.getStatusInfo()).andReturn(statusTypeMock).times(2);
+    expect(statusTypeMock.getStatusCode()).andReturn(200);
+    expect(statusTypeMock.getFamily()).andReturn(Response.Status.Family.SUCCESSFUL);
+    expect(mockResponse.readEntity(TokenExchangeResponseDTO.class))
+        .andReturn(mockTokenExchangeResponseDTO);
+    expect(mockTokenExchangeResponseDTO.getAccessToken()).andReturn("newAccessToken");
+    expect(mockTokenExchangeResponseDTO.getExpiresIn()).andReturn(600);
+    expect(cspApi.getTokenByClientCredentials("Basic YXBwSWQ6YXBwU2VjcmV0", "client_credentials"))
+        .andReturn(mockResponse)
+        .times(1);
+    replayAll();
+
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, tokenWorkerCSP);
+    TokenManager.start(new APIContainer(null, null, null, null, cspApi));
+
+    // Verify the results
+    assertEquals("newAccessToken", tokenWorkerCSP.getBearerToken());
+    assertEquals(wfServer, tokenWorkerCSP.getWFServer());
+    verifyAll();
+  }
+
+  @Test
+  public void testRun_SuccessfulResponseUsingOAuthAppWithNullOrgId() {
+    TokenWorkerCSP tokenWorkerCSP = new TokenWorkerCSP("appId", "appSecret", null, wfServer);
+
+    // Set up expectations
+    expect(mockResponse.getStatusInfo()).andReturn(statusTypeMock).times(2);
+    expect(statusTypeMock.getStatusCode()).andReturn(200);
+    expect(statusTypeMock.getFamily()).andReturn(Response.Status.Family.SUCCESSFUL);
+    expect(mockResponse.readEntity(TokenExchangeResponseDTO.class))
+        .andReturn(mockTokenExchangeResponseDTO);
+    expect(mockTokenExchangeResponseDTO.getAccessToken()).andReturn("newAccessToken");
+    expect(mockTokenExchangeResponseDTO.getExpiresIn()).andReturn(600);
+    expect(cspApi.getTokenByClientCredentials("Basic YXBwSWQ6YXBwU2VjcmV0", "client_credentials"))
+        .andReturn(mockResponse)
+        .times(1);
+    replayAll();
+
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, tokenWorkerCSP);
+    TokenManager.start(new APIContainer(null, null, null, null, cspApi));
+
+    // Verify the results
+    assertEquals("newAccessToken", tokenWorkerCSP.getBearerToken());
+    assertEquals(wfServer, tokenWorkerCSP.getWFServer());
+    verifyAll();
+  }
+
+  @Ignore
+  @Test
+  public void testRun_UnsuccessfulResponseUsingOAuthApp() {
+    TokenWorkerCSP tokenWorkerCSP = new TokenWorkerCSP("appId", "appSecret", "orgId", wfServer);
+
+    // Set up expectations
+    expect(mockResponse.getStatusInfo()).andReturn(statusTypeMock).times(3);
+    expect(mockResponse.readEntity(String.class)).andReturn("");
+    expect(statusTypeMock.getStatusCode()).andReturn(400).times(2);
+    expect(statusTypeMock.getFamily()).andReturn(Response.Status.Family.SERVER_ERROR);
+    expect(
+            cspApi.getTokenByClientCredentialsWithOrgId(
+                "Basic YXBwSWQ6YXBwU2VjcmV0", "client_credentials", "orgId"))
+        .andReturn(mockResponse)
+        .times(1);
+    replayAll();
+
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, tokenWorkerCSP);
+    TokenManager.start(new APIContainer(null, null, null, null, cspApi));
+
+    // Verify the results
+    assertNull(tokenWorkerCSP.getBearerToken());
+    assertEquals(wfServer, tokenWorkerCSP.getWFServer());
+    verifyAll();
+  }
+
+  @Test
+  public void testRun_SuccessfulResponseUsingCSPAPIToken() {
+    TokenWorkerCSP tokenWorkerCSP = new TokenWorkerCSP("csp-api-token", wfServer);
+
+    // Set up expectations
+    expect(mockResponse.getStatusInfo()).andReturn(statusTypeMock).times(2);
+    expect(statusTypeMock.getStatusCode()).andReturn(200);
+    expect(statusTypeMock.getFamily()).andReturn(Response.Status.Family.SUCCESSFUL);
+    expect(mockResponse.readEntity(TokenExchangeResponseDTO.class))
+        .andReturn(mockTokenExchangeResponseDTO);
+    expect(mockTokenExchangeResponseDTO.getAccessToken()).andReturn("newAccessToken");
+    expect(mockTokenExchangeResponseDTO.getExpiresIn()).andReturn(600);
+    expect(cspApi.getTokenByAPIToken("api_token", "csp-api-token"))
+        .andReturn(mockResponse)
+        .times(1);
+    replayAll();
+
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, tokenWorkerCSP);
+    TokenManager.start(new APIContainer(null, null, null, null, cspApi));
+
+    // Verify the results
+    assertEquals("newAccessToken", tokenWorkerCSP.getBearerToken());
+    assertEquals(wfServer, tokenWorkerCSP.getWFServer());
+    verifyAll();
+  }
+
+  @Test
+  public void testRun_UnsuccessfulResponseUsingCSPAPIToken() {
+    TokenWorkerCSP tokenWorkerCSP = new TokenWorkerCSP("csp-api-token", wfServer);
+
+    // Set up expectations
+    expect(mockResponse.readEntity(String.class)).andReturn("");
+    expect(mockResponse.getStatusInfo()).andReturn(statusTypeMock).times(3);
+    expect(statusTypeMock.getStatusCode()).andReturn(400).times(2);
+    expect(statusTypeMock.getFamily()).andReturn(Response.Status.Family.SERVER_ERROR);
+    expect(cspApi.getTokenByAPIToken("api_token", "csp-api-token"))
+        .andReturn(mockResponse)
+        .times(1);
+    replayAll();
+
+    TokenManager.addTenant(CENTRAL_TENANT_NAME, tokenWorkerCSP);
+    TokenManager.start(new APIContainer(null, null, null, null, cspApi));
+
+    // Verify the results
+    assertNull(tokenWorkerCSP.getBearerToken());
+    assertEquals(wfServer, tokenWorkerCSP.getWFServer());
+    verifyAll();
+  }
+
+  @Ignore
+  @Test
+  public void full_test() throws IOException {
+    File cfgFile = File.createTempFile("proxy", ".cfg");
+    cfgFile.deleteOnExit();
+
+    Properties props = new Properties();
+    props.setProperty("pushListenerPorts", "1234");
+
+    props.setProperty("multicastingTenants", "2");
+
+    props.setProperty("multicastingTenantName_1", "name1");
+    props.setProperty("multicastingServer_1", "server1");
+    props.setProperty("multicastingCSPAPIToken_1", "csp-api-token1");
+
+    props.setProperty("multicastingTenantName_2", "name2");
+    props.setProperty("multicastingServer_2", "server2");
+    props.setProperty("multicastingCSPAppId_2", "app-id2");
+    props.setProperty("multicastingCSPAppSecret_2", "app-secret2");
+    props.setProperty("multicastingCSPOrgId_2", "org-id2");
+
+    FileOutputStream out = new FileOutputStream(cfgFile);
+    props.store(out, "");
+    out.close();
+
+    String token = UUID.randomUUID().toString();
+    String[] args =
+        new String[] {
+          "-f",
+          cfgFile.getAbsolutePath(),
+          "--pushListenerPorts",
+          "4321",
+          "--proxyname",
+          "proxyname",
+          "--token",
+          token,
+        };
+
+    ProxyConfig proxyConfig = new ProxyConfig();
+    proxyConfig.parseArguments(args, "test");
+
+    CSPAPI cpsApi = EasyMock.createMock(CSPAPI.class);
+    Response mockResponse = EasyMock.createMock(Response.class);
+    Response.StatusType statusTypeMock = EasyMock.createMock(Response.StatusType.class);
+    TokenExchangeResponseDTO mockTokenExchangeResponseDTO =
+        EasyMock.createMock(TokenExchangeResponseDTO.class);
+
+    expect(mockResponse.getStatusInfo()).andReturn(statusTypeMock).times(4);
+    expect(statusTypeMock.getStatusCode()).andReturn(200).times(2);
+    expect(statusTypeMock.getFamily()).andReturn(Response.Status.Family.SUCCESSFUL).times(2);
+    expect(mockResponse.readEntity(TokenExchangeResponseDTO.class))
+        .andReturn(mockTokenExchangeResponseDTO)
+        .times(2);
+    expect(mockTokenExchangeResponseDTO.getAccessToken()).andReturn("newAccessToken").times(2);
+    expect(mockTokenExchangeResponseDTO.getExpiresIn()).andReturn(600).times(2);
+    expect(
+            cpsApi.getTokenByClientCredentialsWithOrgId(
+                "Basic YXBwLWlkMjphcHAtc2VjcmV0Mg==", "client_credentials", "org-id2"))
+        .andReturn(mockResponse)
+        .times(1);
+    expect(cpsApi.getTokenByAPIToken("api_token", "csp-api-token1"))
+        .andReturn(mockResponse)
+        .times(1);
+
+    replay(mockResponse, statusTypeMock, mockTokenExchangeResponseDTO, cpsApi);
+    TokenManager.start(new APIContainer(null, null, null, null, cpsApi));
+    verify(mockResponse, statusTypeMock, mockTokenExchangeResponseDTO, cpsApi);
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/TestUtils.java b/proxy/src/test/java/com/wavefront/agent/TestUtils.java
index ae602436a..3dcbea4d6 100644
--- a/proxy/src/test/java/com/wavefront/agent/TestUtils.java
+++ b/proxy/src/test/java/com/wavefront/agent/TestUtils.java
@@ -1,42 +1,66 @@
 package com.wavefront.agent;
 
+import com.google.common.collect.Lists;
+import com.google.common.io.Resources;
+import com.wavefront.ingester.SpanDecoder;
+import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.net.HttpURLConnection;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.GZIPOutputStream;
+import javax.net.SocketFactory;
+import org.apache.commons.io.FileUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.StatusLine;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.message.BasicHeader;
-import org.easymock.Capture;
 import org.easymock.EasyMock;
 import org.easymock.IArgumentMatcher;
+import wavefront.report.Span;
 
-import java.io.ByteArrayInputStream;
-
+/** @author vasily@wavefront.com */
 public class TestUtils {
+  private static final Logger logger = Logger.getLogger(TestUtils.class.getCanonicalName());
+
   public static  T httpEq(HttpRequestBase request) {
-    EasyMock.reportMatcher(new IArgumentMatcher() {
-      @Override
-      public boolean matches(Object o) {
-        return o instanceof HttpRequestBase &&
-            o.getClass().getCanonicalName().equals(request.getClass().getCanonicalName()) &&
-            ((HttpRequestBase) o).getMethod().equals(request.getMethod()) &&
-            ((HttpRequestBase) o).getProtocolVersion().equals(request.getProtocolVersion()) &&
-            ((HttpRequestBase) o).getURI().equals(request.getURI());
-      }
+    EasyMock.reportMatcher(
+        new IArgumentMatcher() {
+          @Override
+          public boolean matches(Object o) {
+            return o instanceof HttpRequestBase
+                && o.getClass().getCanonicalName().equals(request.getClass().getCanonicalName())
+                && ((HttpRequestBase) o).getMethod().equals(request.getMethod())
+                && ((HttpRequestBase) o).getProtocolVersion().equals(request.getProtocolVersion())
+                && ((HttpRequestBase) o).getURI().equals(request.getURI());
+          }
 
-      @Override
-      public void appendTo(StringBuffer stringBuffer) {
-        stringBuffer.append("httpEq(");
-        stringBuffer.append(request.toString());
-        stringBuffer.append(")");
-      }
-    });
+          @Override
+          public void appendTo(StringBuffer stringBuffer) {
+            stringBuffer.append("httpEq(");
+            stringBuffer.append(request.toString());
+            stringBuffer.append(")");
+          }
+        });
     return null;
   }
 
-  public static void expectHttpResponse(HttpClient httpClient, HttpRequestBase req,
-                                        byte[] content, int httpStatus) throws Exception {
+  public static void expectHttpResponse(
+      HttpClient httpClient, HttpRequestBase req, byte[] content, int httpStatus) throws Exception {
     HttpResponse response = EasyMock.createMock(HttpResponse.class);
     HttpEntity entity = EasyMock.createMock(HttpEntity.class);
     StatusLine line = EasyMock.createMock(StatusLine.class);
@@ -47,10 +71,178 @@ public static void expectHttpResponse(HttpClient httpClient, HttpRequestBase req
     EasyMock.expect(line.getReasonPhrase()).andReturn("OK").anyTimes();
     EasyMock.expect(entity.getContent()).andReturn(new ByteArrayInputStream(content)).anyTimes();
     EasyMock.expect(entity.getContentLength()).andReturn((long) content.length).atLeastOnce();
-    EasyMock.expect(entity.getContentType()).andReturn(new BasicHeader("Content-Type", "application/json")).anyTimes();
+    EasyMock.expect(entity.getContentType())
+        .andReturn(new BasicHeader("Content-Type", "application/json"))
+        .anyTimes();
 
     EasyMock.expect(httpClient.execute(httpEq(req))).andReturn(response).once();
 
     EasyMock.replay(httpClient, response, entity, line);
   }
+
+  public static int findAvailablePort(int startingPortNumber) {
+    int portNum = startingPortNumber;
+    ServerSocket socket;
+    while (portNum < startingPortNumber + 1000) {
+      try {
+        socket = new ServerSocket(portNum);
+        socket.close();
+        logger.log(Level.INFO, "Found available port: " + portNum);
+        return portNum;
+      } catch (IOException exc) {
+        logger.log(Level.WARNING, "Port " + portNum + " is not available:" + exc.getMessage());
+      }
+      portNum++;
+    }
+    throw new RuntimeException(
+        "Unable to find an available port in the ["
+            + startingPortNumber
+            + ";"
+            + (startingPortNumber + 1000)
+            + ") range");
+  }
+
+  public static void waitUntilListenerIsOnline(int port) throws Exception {
+    final int maxTries = 100;
+    for (int i = 0; i < maxTries; i++) {
+      try {
+        Socket socket = SocketFactory.getDefault().createSocket("localhost", port);
+        socket.close();
+        return;
+      } catch (IOException exc) {
+        TimeUnit.MILLISECONDS.sleep(50);
+      }
+    }
+    throw new RuntimeException(
+        "Giving up connecting to port " + port + " after " + maxTries + " tries.");
+  }
+
+  public static int gzippedHttpPost(String postUrl, String payload) throws Exception {
+    URL url = new URL(postUrl);
+    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+    connection.setRequestMethod("POST");
+    connection.setDoOutput(true);
+    connection.setDoInput(true);
+    connection.setRequestProperty("Content-Encoding", "gzip");
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    GZIPOutputStream gzip = new GZIPOutputStream(baos);
+    gzip.write(payload.getBytes(StandardCharsets.UTF_8));
+    gzip.close();
+    connection.getOutputStream().write(baos.toByteArray());
+    connection.getOutputStream().flush();
+    int response = connection.getResponseCode();
+    logger.info("HTTP response code (gzipped content): " + response);
+    return response;
+  }
+
+  public static int httpGet(String urlGet) throws Exception {
+    URL url = new URL(urlGet);
+    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+    connection.setRequestMethod("GET");
+    connection.setDoOutput(false);
+    connection.setDoInput(true);
+    int response = connection.getResponseCode();
+    logger.info("HTTP GET response code: " + response);
+    return response;
+  }
+
+  public static int httpPost(String urlGet, String payload) throws Exception {
+    URL url = new URL(urlGet);
+    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+    connection.setRequestMethod("POST");
+    connection.setDoOutput(true);
+    connection.setDoInput(true);
+    BufferedWriter writer =
+        new BufferedWriter(new OutputStreamWriter(connection.getOutputStream(), "UTF-8"));
+    writer.write(payload);
+    writer.flush();
+    writer.close();
+    int response = connection.getResponseCode();
+    logger.info("HTTP POST response code (plaintext content): " + response);
+    return response;
+  }
+
+  public static int httpPost(String urlGet, byte[] payload, String mediaType) throws Exception {
+    URL url = new URL(urlGet);
+    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+    connection.setRequestMethod("POST");
+    connection.setRequestProperty("content-type", mediaType);
+    connection.setDoOutput(true);
+    connection.setDoInput(true);
+
+    DataOutputStream writer = new DataOutputStream(connection.getOutputStream());
+    writer.write(payload);
+    writer.flush();
+    writer.close();
+    int response = connection.getResponseCode();
+    logger.info("HTTP POST response code (plaintext content): " + response);
+    return response;
+  }
+
+  public static String getResource(String resourceName) throws Exception {
+    URL url = Resources.getResource("com.wavefront.agent/" + resourceName);
+    File myFile = new File(url.toURI());
+    return FileUtils.readFileToString(myFile, "UTF-8");
+  }
+
+  /**
+   * Verify mocks with retries until specified timeout expires. A healthier alternative to putting
+   * Thread.sleep() before verify().
+   *
+   * @param timeout Desired timeout in milliseconds
+   * @param mocks Mock objects to verify (sequentially).
+   */
+  public static void verifyWithTimeout(int timeout, Object... mocks) {
+    int sleepIntervalMillis = 10;
+    for (Object mock : mocks) {
+      int millisLeft = timeout;
+      while (true) {
+        try {
+          EasyMock.verify(mock);
+          break;
+        } catch (AssertionError e) {
+          if (millisLeft <= 0) {
+            logger.warning("verify() failed after : " + (timeout - millisLeft) + "ms");
+            throw e;
+          }
+          try {
+            TimeUnit.MILLISECONDS.sleep(sleepIntervalMillis);
+          } catch (InterruptedException x) {
+            //
+          }
+          millisLeft -= sleepIntervalMillis;
+        }
+      }
+      long waitTime = timeout - millisLeft;
+      if (waitTime > 0) {
+        logger.info("verify() wait time: " + waitTime + "ms");
+      }
+    }
+  }
+
+  public static void assertTrueWithTimeout(int timeout, Supplier assertion) {
+    int sleepIntervalMillis = 10;
+    int millisLeft = timeout;
+    while (true) {
+      if (assertion.get()) break;
+      if (millisLeft <= 0)
+        throw new AssertionError("Assertion timed out (" + (timeout - millisLeft) + "ms)");
+      try {
+        TimeUnit.MILLISECONDS.sleep(sleepIntervalMillis);
+      } catch (InterruptedException x) {
+        //
+      }
+      millisLeft -= sleepIntervalMillis;
+    }
+    long waitTime = timeout - millisLeft;
+    if (waitTime > 0) {
+      logger.info("assertTrueWithTimeout() wait time: " + waitTime + "ms");
+    }
+  }
+
+  public static Span parseSpan(String line) {
+    List out = Lists.newArrayListWithExpectedSize(1);
+    new SpanDecoder("unknown").decode(line, out, "dummy");
+    return out.get(0);
+  }
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/api/APIContainerTest.java b/proxy/src/test/java/com/wavefront/agent/api/APIContainerTest.java
new file mode 100644
index 000000000..e065b2654
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/api/APIContainerTest.java
@@ -0,0 +1,87 @@
+package com.wavefront.agent.api;
+
+import static org.junit.Assert.*;
+
+import com.wavefront.agent.ProxyConfig;
+import com.wavefront.agent.TokenManager;
+import com.wavefront.agent.TokenWorkerCSP;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class APIContainerTest {
+  private final int NUM_TENANTS = 5;
+  private ProxyConfig proxyConfig;
+
+  @Before
+  public void setup() {
+    this.proxyConfig = new ProxyConfig();
+    TokenWorkerCSP tokenWorkerCSP = new TokenWorkerCSP("fake-token", "fake-url");
+    TokenManager.addTenant(APIContainer.CENTRAL_TENANT_NAME, tokenWorkerCSP);
+    for (int i = 0; i < NUM_TENANTS; i++) {
+      TokenWorkerCSP tokenWorkerCSP1 = new TokenWorkerCSP("fake-token" + i, "fake-url" + i);
+      TokenManager.addTenant("tenant-" + i, tokenWorkerCSP1);
+    }
+  }
+
+  @Test
+  public void testAPIContainerInitiationWithDiscardData() {
+    APIContainer apiContainer = new APIContainer(this.proxyConfig, true);
+    assertEquals(apiContainer.getTenantNameList().size(), 1);
+    assertTrue(apiContainer.getProxyV2APIForTenant("central") instanceof NoopProxyV2API);
+    assertTrue(apiContainer.getSourceTagAPIForTenant("central") instanceof NoopSourceTagAPI);
+    assertTrue(apiContainer.getEventAPIForTenant("central") instanceof NoopEventAPI);
+  }
+
+  @Test(expected = IllegalStateException.class)
+  public void testUpdateServerEndpointURLWithNullProxyConfig() {
+    APIContainer apiContainer = new APIContainer(null, null, null, null, null);
+    apiContainer.updateServerEndpointURL("central", "fake-url");
+  }
+
+  @Ignore
+  @Test
+  public void testUpdateServerEndpointURLWithValidProxyConfig() {
+    APIContainer apiContainer = new APIContainer(this.proxyConfig, false);
+    assertEquals(apiContainer.getTenantNameList().size(), NUM_TENANTS + 1);
+    apiContainer.updateServerEndpointURL("central", "another-fake-url");
+    assertEquals(apiContainer.getTenantNameList().size(), NUM_TENANTS + 1);
+    assertNotNull(apiContainer.getProxyV2APIForTenant("central"));
+
+    apiContainer = new APIContainer(this.proxyConfig, true);
+    assertEquals(apiContainer.getTenantNameList().size(), 1);
+    apiContainer.updateServerEndpointURL("central", "another-fake-url");
+    assertEquals(apiContainer.getTenantNameList().size(), 1);
+    assertNotNull(apiContainer.getProxyV2APIForTenant("central"));
+    assertTrue(apiContainer.getProxyV2APIForTenant("central") instanceof NoopProxyV2API);
+    assertTrue(apiContainer.getSourceTagAPIForTenant("central") instanceof NoopSourceTagAPI);
+    assertTrue(apiContainer.getEventAPIForTenant("central") instanceof NoopEventAPI);
+  }
+
+  @Test
+  public void testUpdateLogServerEndpointURLandToken() {
+    APIContainer apiContainer = new APIContainer(this.proxyConfig, false);
+
+    apiContainer.updateLogServerEndpointURLandToken(null, null);
+    assertEquals("NOT_SET", apiContainer.getLogServerToken());
+    assertEquals("NOT_SET", apiContainer.getLogServerEndpointUrl());
+
+    apiContainer.updateLogServerEndpointURLandToken("", "");
+    assertEquals("NOT_SET", apiContainer.getLogServerToken());
+    assertEquals("NOT_SET", apiContainer.getLogServerEndpointUrl());
+
+    apiContainer.updateLogServerEndpointURLandToken("testURL", "");
+    assertEquals("NOT_SET", apiContainer.getLogServerToken());
+    assertEquals("NOT_SET", apiContainer.getLogServerEndpointUrl());
+
+    apiContainer.updateLogServerEndpointURLandToken("testURL", "testToken");
+    assertEquals("testToken", apiContainer.getLogServerToken());
+    assertEquals("testURL", apiContainer.getLogServerEndpointUrl());
+
+    apiContainer.updateLogServerEndpointURLandToken(
+        "https://data.lint-be.symphony-dev.com/le-mans/v1/streams/ingestion-pipeline-stream",
+        "testToken");
+    assertEquals("testToken", apiContainer.getLogServerToken());
+    assertEquals("https://data.lint-be.symphony-dev.com/", apiContainer.getLogServerEndpointUrl());
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/auth/DummyAuthenticatorTest.java b/proxy/src/test/java/com/wavefront/agent/auth/DummyAuthenticatorTest.java
index 89ccee4a8..ac80e6ba9 100644
--- a/proxy/src/test/java/com/wavefront/agent/auth/DummyAuthenticatorTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/auth/DummyAuthenticatorTest.java
@@ -1,10 +1,10 @@
 package com.wavefront.agent.auth;
 
+import static org.junit.Assert.assertTrue;
+
 import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.Test;
 
-import static org.junit.Assert.assertTrue;
-
 public class DummyAuthenticatorTest {
 
   @Test
diff --git a/proxy/src/test/java/com/wavefront/agent/auth/HttpGetTokenIntrospectionAuthenticatorTest.java b/proxy/src/test/java/com/wavefront/agent/auth/HttpGetTokenIntrospectionAuthenticatorTest.java
index f40b41216..8a057693b 100644
--- a/proxy/src/test/java/com/wavefront/agent/auth/HttpGetTokenIntrospectionAuthenticatorTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/auth/HttpGetTokenIntrospectionAuthenticatorTest.java
@@ -1,5 +1,13 @@
 package com.wavefront.agent.auth;
 
+import static com.wavefront.agent.TestUtils.assertTrueWithTimeout;
+import static com.wavefront.agent.TestUtils.httpEq;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.http.HttpVersion;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
@@ -7,53 +15,44 @@
 import org.easymock.EasyMock;
 import org.junit.Test;
 
-import java.io.IOException;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static com.wavefront.agent.TestUtils.httpEq;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
 public class HttpGetTokenIntrospectionAuthenticatorTest {
 
   @Test
   public void testIntrospectionUrlInvocation() throws Exception {
     HttpClient client = EasyMock.createMock(HttpClient.class);
     AtomicLong fakeClock = new AtomicLong(1_000_000);
-    TokenAuthenticator authenticator = new HttpGetTokenIntrospectionAuthenticator(client,
-        "http://acme.corp/{{token}}/something", null, 300, 600, fakeClock::get);
+    TokenAuthenticator authenticator =
+        new HttpGetTokenIntrospectionAuthenticator(
+            client,
+            "http://acme.corp/{{token}}/something",
+            "Bearer: abcde12345",
+            300,
+            600,
+            fakeClock::get);
+    assertTrue(authenticator.authRequired());
+    assertFalse(authenticator.authorize(null));
 
     String uuid = UUID.randomUUID().toString();
-    EasyMock.expect(client.execute(httpEq(new HttpGet("http://acme.corp/" + uuid + "/something")))).
-        andReturn(new BasicHttpResponse(HttpVersion.HTTP_1_1, 401, "Auth failed")).once().
-        andReturn(new BasicHttpResponse(HttpVersion.HTTP_1_1, 204, "")).once().
-        andReturn(new BasicHttpResponse(HttpVersion.HTTP_1_1, 401, "Auth failed")).once();
+    EasyMock.expect(client.execute(httpEq(new HttpGet("http://acme.corp/" + uuid + "/something"))))
+        .andReturn(new BasicHttpResponse(HttpVersion.HTTP_1_1, 401, "Auth failed"))
+        .once()
+        .andReturn(new BasicHttpResponse(HttpVersion.HTTP_1_1, 204, ""))
+        .once()
+        .andReturn(new BasicHttpResponse(HttpVersion.HTTP_1_1, 401, "Auth failed"))
+        .once();
     EasyMock.replay(client);
-
     assertFalse(authenticator.authorize(uuid)); // should call http
-    Thread.sleep(100);
-
     fakeClock.getAndAdd(60_000);
-
     assertFalse(authenticator.authorize(uuid)); // should be cached
-
     fakeClock.getAndAdd(300_000);
-
     assertFalse(authenticator.authorize(uuid)); // cache expired - should trigger a refresh
-    Thread.sleep(100);
-    assertTrue(authenticator.authorize(uuid)); // should call http and get an updated token
-
+    // should call http and get an updated token
+    assertTrueWithTimeout(100, () -> authenticator.authorize(uuid));
     fakeClock.getAndAdd(180_000);
-
     assertTrue(authenticator.authorize(uuid)); // should be cached
-
     fakeClock.getAndAdd(180_000);
-
     assertTrue(authenticator.authorize(uuid)); // cache expired - should trigger a refresh
-    Thread.sleep(100);
-    assertFalse(authenticator.authorize(uuid)); // should call http
-
+    assertTrueWithTimeout(100, () -> !authenticator.authorize(uuid)); // should call http
     EasyMock.verify(client);
   }
 
@@ -61,28 +60,27 @@ public void testIntrospectionUrlInvocation() throws Exception {
   public void testIntrospectionUrlCachedLastResultExpires() throws Exception {
     HttpClient client = EasyMock.createMock(HttpClient.class);
     AtomicLong fakeClock = new AtomicLong(1_000_000);
-    TokenAuthenticator authenticator = new HttpGetTokenIntrospectionAuthenticator(client,
-        "http://acme.corp/{{token}}/something", null, 300, 600, fakeClock::get);
+    TokenAuthenticator authenticator =
+        new HttpGetTokenIntrospectionAuthenticator(
+            client, "http://acme.corp/{{token}}/something", null, 300, 600, fakeClock::get);
 
     String uuid = UUID.randomUUID().toString();
-    EasyMock.expect(client.execute(httpEq(new HttpGet("http://acme.corp/" + uuid + "/something")))).
-        andReturn(new BasicHttpResponse(HttpVersion.HTTP_1_1, 204, "")).once().
-        andThrow(new IOException("Timeout!")).times(3);
+    EasyMock.expect(client.execute(httpEq(new HttpGet("http://acme.corp/" + uuid + "/something"))))
+        .andReturn(new BasicHttpResponse(HttpVersion.HTTP_1_1, 204, ""))
+        .once()
+        .andThrow(new IOException("Timeout!"))
+        .times(3);
     EasyMock.replay(client);
-
     assertTrue(authenticator.authorize(uuid)); // should call http
-    Thread.sleep(100);
-
     fakeClock.getAndAdd(630_000);
-
-    assertTrue(authenticator.authorize(uuid)); // should call http, fail, but still return last valid result
-    Thread.sleep(100);
-
-    assertFalse(authenticator.authorize(uuid)); // TTL expired - should fail
-
-    Thread.sleep(100);
-    assertFalse(authenticator.authorize(uuid)); // Should call http again - TTL expired
-
+    assertTrue(
+        authenticator.authorize(
+            uuid)); // should call http, fail, but still return last valid result
+    // Thread.sleep(100);
+    assertTrueWithTimeout(100, () -> !authenticator.authorize(uuid)); // TTL expired - should fail
+    // Thread.sleep(100);
+    // Should call http again - TTL expired
+    assertTrueWithTimeout(100, () -> !authenticator.authorize(uuid));
     EasyMock.verify(client);
   }
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/auth/Oauth2TokenIntrospectionAuthenticatorTest.java b/proxy/src/test/java/com/wavefront/agent/auth/Oauth2TokenIntrospectionAuthenticatorTest.java
index 246d6eda8..2d85b8679 100644
--- a/proxy/src/test/java/com/wavefront/agent/auth/Oauth2TokenIntrospectionAuthenticatorTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/auth/Oauth2TokenIntrospectionAuthenticatorTest.java
@@ -1,24 +1,28 @@
 package com.wavefront.agent.auth;
 
-import com.google.common.collect.ImmutableList;
+import static com.wavefront.agent.TestUtils.assertTrueWithTimeout;
+import static com.wavefront.agent.TestUtils.httpEq;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
+import com.google.common.collect.ImmutableList;
 import com.wavefront.agent.TestUtils;
-
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.MetricName;
+import java.io.IOException;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+import javax.annotation.concurrent.NotThreadSafe;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.message.BasicNameValuePair;
 import org.easymock.EasyMock;
+import org.junit.Ignore;
 import org.junit.Test;
 
-import java.io.IOException;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static com.wavefront.agent.TestUtils.httpEq;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
+@NotThreadSafe
 public class Oauth2TokenIntrospectionAuthenticatorTest {
 
   @Test
@@ -26,66 +30,60 @@ public void testIntrospectionUrlInvocation() throws Exception {
     HttpClient client = EasyMock.createMock(HttpClient.class);
 
     AtomicLong fakeClock = new AtomicLong(1_000_000);
-    TokenAuthenticator authenticator = new Oauth2TokenIntrospectionAuthenticator(client,
-        "http://acme.corp/oauth", null, 300, 600, fakeClock::get);
+    TokenAuthenticator authenticator =
+        new Oauth2TokenIntrospectionAuthenticator(
+            client, "http://acme.corp/oauth", null, 300, 600, fakeClock::get);
+    assertTrue(authenticator.authRequired());
+    assertFalse(authenticator.authorize(null));
 
     String uuid = UUID.randomUUID().toString();
 
     HttpPost request = new HttpPost("http://acme.corp/oauth");
     request.setHeader("Content-Type", "application/x-www-form-urlencoded");
     request.setHeader("Accept", "application/json");
-    request.setEntity(new UrlEncodedFormEntity(ImmutableList.of(new BasicNameValuePair("token", uuid))));
+    request.setEntity(
+        new UrlEncodedFormEntity(ImmutableList.of(new BasicNameValuePair("token", uuid))));
 
     TestUtils.expectHttpResponse(client, request, "{\"active\": false}".getBytes(), 200);
 
     assertFalse(authenticator.authorize(uuid)); // should call http
-    Thread.sleep(100);
-
     fakeClock.getAndAdd(60_000);
-
     assertFalse(authenticator.authorize(uuid)); // should be cached
-
     fakeClock.getAndAdd(300_000);
-
     EasyMock.verify(client);
     EasyMock.reset(client);
     TestUtils.expectHttpResponse(client, request, "{\"active\": true}".getBytes(), 200);
-
     assertFalse(authenticator.authorize(uuid)); // cache expired - should trigger a refresh
-    Thread.sleep(100);
-
-    assertTrue(authenticator.authorize(uuid)); // should call http and get an updated token
-
+    // should call http and get an updated token
+    assertTrueWithTimeout(100, () -> authenticator.authorize(uuid));
     fakeClock.getAndAdd(180_000);
-
     assertTrue(authenticator.authorize(uuid)); // should be cached
-
     fakeClock.getAndAdd(180_000);
-
     EasyMock.verify(client);
     EasyMock.reset(client);
     TestUtils.expectHttpResponse(client, request, "{\"active\": false}".getBytes(), 200);
-
     assertTrue(authenticator.authorize(uuid)); // cache expired - should trigger a refresh
-    Thread.sleep(100);
-    assertFalse(authenticator.authorize(uuid)); // should call http
-
+    // Thread.sleep(100);
+    assertTrueWithTimeout(100, () -> !authenticator.authorize(uuid)); // should call http
     EasyMock.verify(client);
   }
 
+  @Ignore
   @Test
   public void testIntrospectionUrlCachedLastResultExpires() throws Exception {
     HttpClient client = EasyMock.createMock(HttpClient.class);
     AtomicLong fakeClock = new AtomicLong(1_000_000);
-    TokenAuthenticator authenticator = new Oauth2TokenIntrospectionAuthenticator(client,
-        "http://acme.corp/oauth", null, 300, 600, fakeClock::get);
+    TokenAuthenticator authenticator =
+        new Oauth2TokenIntrospectionAuthenticator(
+            client, "http://acme.corp/oauth", "Bearer: abcde12345", 300, 600, fakeClock::get);
 
     String uuid = UUID.randomUUID().toString();
 
     HttpPost request = new HttpPost("http://acme.corp/oauth");
     request.setHeader("Content-Type", "application/x-www-form-urlencoded");
     request.setHeader("Accept", "application/json");
-    request.setEntity(new UrlEncodedFormEntity(ImmutableList.of(new BasicNameValuePair("token", uuid))));
+    request.setEntity(
+        new UrlEncodedFormEntity(ImmutableList.of(new BasicNameValuePair("token", uuid))));
 
     TestUtils.expectHttpResponse(client, request, "{\"active\": true}".getBytes(), 204);
 
@@ -97,15 +95,41 @@ public void testIntrospectionUrlCachedLastResultExpires() throws Exception {
     EasyMock.verify(client);
     EasyMock.reset(client);
 
-    EasyMock.expect(client.execute(httpEq(new HttpPost("http://acme.corp/oauth")))).
-        andThrow(new IOException("Timeout!")).times(3);
+    EasyMock.expect(client.execute(httpEq(new HttpPost("http://acme.corp/oauth"))))
+        .andThrow(new IOException("Timeout!"))
+        .times(3);
     EasyMock.replay(client);
 
-    assertTrue(authenticator.authorize(uuid)); // should call http, fail, but still return last valid result
+    assertTrue(
+        authenticator.authorize(
+            uuid)); // should call http, fail, but still return last valid result
     Thread.sleep(100);
     assertFalse(authenticator.authorize(uuid)); // TTL expired - should fail
     assertFalse(authenticator.authorize(uuid)); // Should call http again - TTL expired
 
     EasyMock.verify(client);
   }
+
+  @Test
+  public void testIntrospectionUrlInvalidResponseThrows() throws Exception {
+    HttpClient client = EasyMock.createMock(HttpClient.class);
+    AtomicLong fakeClock = new AtomicLong(1_000_000);
+    TokenAuthenticator authenticator =
+        new Oauth2TokenIntrospectionAuthenticator(
+            client, "http://acme.corp/oauth", "Bearer: abcde12345", 300, 600, fakeClock::get);
+
+    String uuid = UUID.randomUUID().toString();
+
+    HttpPost request = new HttpPost("http://acme.corp/oauth");
+    request.setHeader("Content-Type", "application/x-www-form-urlencoded");
+    request.setHeader("Accept", "application/json");
+    request.setEntity(
+        new UrlEncodedFormEntity(ImmutableList.of(new BasicNameValuePair("token", uuid))));
+
+    TestUtils.expectHttpResponse(client, request, "{\"inActive\": true}".getBytes(), 204);
+
+    long count = Metrics.newCounter(new MetricName("auth", "", "api-errors")).count();
+    authenticator.authorize(uuid); // should call http
+    assertEquals(1, Metrics.newCounter(new MetricName("auth", "", "api-errors")).count() - count);
+  }
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/auth/StaticTokenAuthenticatorTest.java b/proxy/src/test/java/com/wavefront/agent/auth/StaticTokenAuthenticatorTest.java
index 2c9ca26a2..21e368654 100644
--- a/proxy/src/test/java/com/wavefront/agent/auth/StaticTokenAuthenticatorTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/auth/StaticTokenAuthenticatorTest.java
@@ -1,16 +1,17 @@
 package com.wavefront.agent.auth;
 
-import org.apache.commons.lang3.RandomStringUtils;
-import org.junit.Test;
-
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.Test;
+
 public class StaticTokenAuthenticatorTest {
 
   @Test
   public void testValidTokenWorks() {
     TokenAuthenticator authenticator = new StaticTokenAuthenticator("staticToken");
+    assertTrue(authenticator.authRequired());
 
     // null should fail
     assertFalse(authenticator.authorize(null));
diff --git a/proxy/src/test/java/com/wavefront/agent/auth/TokenAuthenticatorBuilderTest.java b/proxy/src/test/java/com/wavefront/agent/auth/TokenAuthenticatorBuilderTest.java
index 6a90f492a..fe9726ae4 100644
--- a/proxy/src/test/java/com/wavefront/agent/auth/TokenAuthenticatorBuilderTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/auth/TokenAuthenticatorBuilderTest.java
@@ -1,63 +1,102 @@
 package com.wavefront.agent.auth;
 
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
 import org.apache.http.client.HttpClient;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.junit.Test;
 
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
 public class TokenAuthenticatorBuilderTest {
 
   @Test
   public void testBuilderOutput() {
     assertTrue(TokenAuthenticatorBuilder.create().build() instanceof DummyAuthenticator);
 
-    assertTrue(TokenAuthenticatorBuilder.create().
-        setTokenValidationMethod(TokenValidationMethod.NONE).build() instanceof DummyAuthenticator);
+    assertTrue(
+        TokenAuthenticatorBuilder.create()
+                .setTokenValidationMethod(TokenValidationMethod.NONE)
+                .build()
+            instanceof DummyAuthenticator);
 
-    assertTrue(TokenAuthenticatorBuilder.create().setTokenValidationMethod(TokenValidationMethod.STATIC_TOKEN).
-        setStaticToken("statictoken").build() instanceof StaticTokenAuthenticator);
+    assertTrue(
+        TokenAuthenticatorBuilder.create()
+                .setTokenValidationMethod(TokenValidationMethod.STATIC_TOKEN)
+                .setStaticToken("statictoken")
+                .build()
+            instanceof StaticTokenAuthenticator);
 
     HttpClient httpClient = HttpClientBuilder.create().useSystemProperties().build();
 
-    assertTrue(TokenAuthenticatorBuilder.create().setTokenValidationMethod(TokenValidationMethod.HTTP_GET).
-        setHttpClient(httpClient).setTokenIntrospectionServiceUrl("https://acme.corp/url").
-        build() instanceof HttpGetTokenIntrospectionAuthenticator);
+    assertTrue(
+        TokenAuthenticatorBuilder.create()
+                .setTokenValidationMethod(TokenValidationMethod.HTTP_GET)
+                .setHttpClient(httpClient)
+                .setTokenIntrospectionServiceUrl("https://acme.corp/url")
+                .build()
+            instanceof HttpGetTokenIntrospectionAuthenticator);
 
-    assertTrue(TokenAuthenticatorBuilder.create().setTokenValidationMethod(TokenValidationMethod.OAUTH2).
-        setHttpClient(httpClient).setTokenIntrospectionServiceUrl("https://acme.corp/url").
-        build() instanceof Oauth2TokenIntrospectionAuthenticator);
+    assertTrue(
+        TokenAuthenticatorBuilder.create()
+                .setTokenValidationMethod(TokenValidationMethod.HTTP_GET)
+                .setHttpClient(httpClient)
+                .setTokenIntrospectionServiceUrl("https://acme.corp/url")
+                .setAuthResponseMaxTtl(10)
+                .setAuthResponseRefreshInterval(60)
+                .setTokenIntrospectionAuthorizationHeader("Bearer: 12345secret")
+                .build()
+            instanceof HttpGetTokenIntrospectionAuthenticator);
+
+    assertTrue(
+        TokenAuthenticatorBuilder.create()
+                .setTokenValidationMethod(TokenValidationMethod.OAUTH2)
+                .setHttpClient(httpClient)
+                .setTokenIntrospectionServiceUrl("https://acme.corp/url")
+                .build()
+            instanceof Oauth2TokenIntrospectionAuthenticator);
+
+    assertNull(TokenValidationMethod.fromString("random"));
   }
 
   @Test(expected = RuntimeException.class)
   public void testBuilderStaticTokenIncompleteArgumentsThrows() {
-    TokenAuthenticatorBuilder.create().setTokenValidationMethod(TokenValidationMethod.STATIC_TOKEN).build();
+    TokenAuthenticatorBuilder.create()
+        .setTokenValidationMethod(TokenValidationMethod.STATIC_TOKEN)
+        .build();
   }
 
   @Test(expected = RuntimeException.class)
   public void testBuilderHttpGetIncompleteArgumentsThrows() {
-    TokenAuthenticatorBuilder.create().setTokenValidationMethod(TokenValidationMethod.HTTP_GET).build();
+    TokenAuthenticatorBuilder.create()
+        .setTokenValidationMethod(TokenValidationMethod.HTTP_GET)
+        .build();
   }
 
   @Test(expected = RuntimeException.class)
   public void testBuilderHttpGetIncompleteArguments2Throws() {
-    TokenAuthenticatorBuilder.create().setTokenValidationMethod(TokenValidationMethod.HTTP_GET).
-        setTokenIntrospectionServiceUrl("http://acme.corp").build();
+    TokenAuthenticatorBuilder.create()
+        .setTokenValidationMethod(TokenValidationMethod.HTTP_GET)
+        .setTokenIntrospectionServiceUrl("http://acme.corp")
+        .build();
   }
 
   @Test(expected = RuntimeException.class)
   public void testBuilderOauth2IncompleteArgumentsThrows() {
-    TokenAuthenticatorBuilder.create().setTokenValidationMethod(TokenValidationMethod.OAUTH2).build();
+    TokenAuthenticatorBuilder.create()
+        .setTokenValidationMethod(TokenValidationMethod.OAUTH2)
+        .build();
   }
 
   @Test(expected = RuntimeException.class)
   public void testBuilderOauth2IncompleteArguments2Throws() {
-    TokenAuthenticatorBuilder.create().setTokenValidationMethod(TokenValidationMethod.OAUTH2).
-        setTokenIntrospectionServiceUrl("http://acme.corp").build();
+    TokenAuthenticatorBuilder.create()
+        .setTokenValidationMethod(TokenValidationMethod.OAUTH2)
+        .setTokenIntrospectionServiceUrl("http://acme.corp")
+        .build();
+  }
+
+  @Test(expected = RuntimeException.class)
+  public void testBuilderInvalidMethodThrows() {
+    TokenAuthenticatorBuilder.create().setTokenValidationMethod(null).build();
   }
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/channel/SharedGraphiteHostAnnotatorTest.java b/proxy/src/test/java/com/wavefront/agent/channel/SharedGraphiteHostAnnotatorTest.java
new file mode 100644
index 000000000..443817241
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/channel/SharedGraphiteHostAnnotatorTest.java
@@ -0,0 +1,52 @@
+package com.wavefront.agent.channel;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import org.junit.Test;
+
+/** @author vasily@wavefront.com */
+public class SharedGraphiteHostAnnotatorTest {
+
+  @Test
+  public void testHostAnnotator() throws Exception {
+    ChannelHandlerContext ctx = createMock(ChannelHandlerContext.class);
+    Channel channel = createMock(Channel.class);
+    InetSocketAddress remote = new InetSocketAddress(InetAddress.getLocalHost(), 2878);
+    expect(ctx.channel()).andReturn(channel).anyTimes();
+    expect(channel.remoteAddress()).andReturn(remote).anyTimes();
+    replay(channel, ctx);
+    SharedGraphiteHostAnnotator annotator =
+        new SharedGraphiteHostAnnotator(ImmutableList.of("tag1", "tag2", "tag3"), x -> "default");
+
+    String point;
+    point = "request.count 1 source=test.wavefront.com";
+    assertEquals(point, annotator.apply(ctx, point));
+    point = "\"request.count\" 1 \"source\"=\"test.wavefront.com\"";
+    assertEquals(point, annotator.apply(ctx, point));
+    point = "request.count 1 host=test.wavefront.com";
+    assertEquals(point, annotator.apply(ctx, point));
+    point = "request.count 1 \"host\"=test.wavefront.com";
+    assertEquals(point, annotator.apply(ctx, point));
+    point = "request.count 1 tag1=test.wavefront.com";
+    assertEquals(point, annotator.apply(ctx, point));
+    point = "request.count 1 tag2=\"test.wavefront.com\"";
+    assertEquals(point, annotator.apply(ctx, point));
+    point = "request.count 1 tag3=test.wavefront.com";
+    assertEquals(point, annotator.apply(ctx, point));
+    point = "request.count 1 tag4=test.wavefront.com";
+    assertEquals(
+        "request.count 1 tag4=test.wavefront.com source=\"default\"", annotator.apply(ctx, point));
+    String log = "{\"tag4\":\"test.wavefront.com\"}";
+    assertEquals(
+        "{\"source\":\"default\", \"tag4\":\"test.wavefront.com\"}",
+        annotator.apply(ctx, log, true));
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/common/HostMetricTagsPairTest.java b/proxy/src/test/java/com/wavefront/agent/common/HostMetricTagsPairTest.java
new file mode 100644
index 000000000..9192eaa32
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/common/HostMetricTagsPairTest.java
@@ -0,0 +1,92 @@
+package com.wavefront.agent.common;
+
+import com.wavefront.common.HostMetricTagsPair;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/** @author Jia Deng (djia@vmware.com) */
+public class HostMetricTagsPairTest {
+
+  @Rule public ExpectedException thrown = ExpectedException.none();
+
+  @Test
+  public void testEmptyHost() {
+    thrown.expect(NullPointerException.class);
+    Map tags = new HashMap<>();
+    tags.put("key", "value");
+    new HostMetricTagsPair(null, "metric", tags);
+  }
+
+  @Test
+  public void testEmptyMetric() {
+    Map tags = new HashMap<>();
+    tags.put("key", "value");
+    thrown.expect(NullPointerException.class);
+    new HostMetricTagsPair("host", null, tags);
+  }
+
+  @Test
+  public void testGetMetric() {
+    HostMetricTagsPair hostMetricTagsPair = new HostMetricTagsPair("host", "metric", null);
+    Assert.assertEquals(hostMetricTagsPair.getMetric(), "metric");
+  }
+
+  @Test
+  public void testGetHost() {
+    HostMetricTagsPair hostMetricTagsPair = new HostMetricTagsPair("host", "metric", null);
+    Assert.assertEquals(hostMetricTagsPair.getHost(), "host");
+  }
+
+  @Test
+  public void testGetTags() {
+    Map tags = new HashMap<>();
+    tags.put("key", "value");
+    HostMetricTagsPair hostMetricTagsPair = new HostMetricTagsPair("host", "metric", tags);
+    Assert.assertEquals(hostMetricTagsPair.getTags(), tags);
+  }
+
+  @Test
+  public void testEquals() throws Exception {
+    Map tags1 = new HashMap<>();
+    tags1.put("key1", "value1");
+    HostMetricTagsPair hostMetricTagsPair1 = new HostMetricTagsPair("host1", "metric1", tags1);
+
+    // equals itself
+    Assert.assertTrue(hostMetricTagsPair1.equals(hostMetricTagsPair1));
+
+    // same hostMetricTagsPair
+    HostMetricTagsPair hostMetricTagsPair2 = new HostMetricTagsPair("host1", "metric1", tags1);
+    Assert.assertTrue(hostMetricTagsPair1.equals(hostMetricTagsPair2));
+
+    // compare different host with hostMetricTagsPair1
+    HostMetricTagsPair hostMetricTagsPair3 = new HostMetricTagsPair("host2", "metric1", tags1);
+    Assert.assertFalse(hostMetricTagsPair1.equals(hostMetricTagsPair3));
+
+    // compare different metric with hostMetricTagsPair1
+    HostMetricTagsPair hostMetricTagsPair4 = new HostMetricTagsPair("host1", "metric2", tags1);
+    Assert.assertFalse(hostMetricTagsPair1.equals(hostMetricTagsPair4));
+
+    // compare different tags with hostMetricTagsPair1
+    Map tags2 = new HashMap<>();
+    tags2.put("key2", "value2");
+    HostMetricTagsPair hostMetricTagsPair5 = new HostMetricTagsPair("host1", "metric1", tags2);
+    Assert.assertFalse(hostMetricTagsPair1.equals(hostMetricTagsPair5));
+
+    // compare empty tags with hostMetricTagsPair1
+    HostMetricTagsPair hostMetricTagsPair6 = new HostMetricTagsPair("host1", "metric1", null);
+    Assert.assertFalse(hostMetricTagsPair1.equals(hostMetricTagsPair6));
+  }
+
+  @Test
+  public void testToString() throws Exception {
+    Map tags = new HashMap<>();
+    tags.put("key", "value");
+    HostMetricTagsPair hostMetricTagsPair = new HostMetricTagsPair("host", "metric", tags);
+    Assert.assertEquals(
+        hostMetricTagsPair.toString(), "[host: host, metric: metric, tags: " + "{key=value}]");
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/data/DefaultEntityPropertiesFactoryForTesting.java b/proxy/src/test/java/com/wavefront/agent/data/DefaultEntityPropertiesFactoryForTesting.java
new file mode 100644
index 000000000..dffa75430
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/data/DefaultEntityPropertiesFactoryForTesting.java
@@ -0,0 +1,19 @@
+package com.wavefront.agent.data;
+
+import com.wavefront.data.ReportableEntityType;
+
+/** @author vasily@wavefront.com */
+public class DefaultEntityPropertiesFactoryForTesting implements EntityPropertiesFactory {
+  private final EntityProperties props = new DefaultEntityPropertiesForTesting();
+  private final GlobalProperties globalProps = new DefaultGlobalPropertiesForTesting();
+
+  @Override
+  public EntityProperties get(ReportableEntityType entityType) {
+    return props;
+  }
+
+  @Override
+  public GlobalProperties getGlobalProperties() {
+    return globalProps;
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/data/DefaultEntityPropertiesForTesting.java b/proxy/src/test/java/com/wavefront/agent/data/DefaultEntityPropertiesForTesting.java
new file mode 100644
index 000000000..45a4d914c
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/data/DefaultEntityPropertiesForTesting.java
@@ -0,0 +1,91 @@
+package com.wavefront.agent.data;
+
+import com.google.common.util.concurrent.RecyclableRateLimiter;
+import com.google.common.util.concurrent.RecyclableRateLimiterImpl;
+import javax.annotation.Nullable;
+
+/** @author vasily@wavefront.com */
+public class DefaultEntityPropertiesForTesting implements EntityProperties {
+
+  @Override
+  public int getDataPerBatchOriginal() {
+    return DEFAULT_BATCH_SIZE;
+  }
+
+  @Override
+  public boolean isSplitPushWhenRateLimited() {
+    return DEFAULT_SPLIT_PUSH_WHEN_RATE_LIMITED;
+  }
+
+  @Override
+  public double getRateLimit() {
+    return NO_RATE_LIMIT;
+  }
+
+  @Override
+  public int getRateLimitMaxBurstSeconds() {
+    return DEFAULT_MAX_BURST_SECONDS;
+  }
+
+  @Override
+  public RecyclableRateLimiter getRateLimiter() {
+    return RecyclableRateLimiterImpl.create(NO_RATE_LIMIT, getRateLimitMaxBurstSeconds());
+  }
+
+  @Override
+  public int getFlushThreads() {
+    return 2;
+  }
+
+  @Override
+  public int getPushFlushInterval() {
+    return 100000;
+  }
+
+  @Override
+  public int getDataPerBatch() {
+    return DEFAULT_BATCH_SIZE;
+  }
+
+  @Override
+  public void setDataPerBatch(@Nullable Integer dataPerBatch) {}
+
+  @Override
+  public int getMinBatchSplitSize() {
+    return DEFAULT_MIN_SPLIT_BATCH_SIZE;
+  }
+
+  @Override
+  public int getMemoryBufferLimit() {
+    return DEFAULT_MIN_SPLIT_BATCH_SIZE;
+  }
+
+  @Override
+  public TaskQueueLevel getTaskQueueLevel() {
+    return TaskQueueLevel.ANY_ERROR;
+  }
+
+  @Override
+  public boolean isFeatureDisabled() {
+    return false;
+  }
+
+  @Override
+  public void setFeatureDisabled(boolean featureDisabled) {}
+
+  @Override
+  public int getTotalBacklogSize() {
+    return 0;
+  }
+
+  @Override
+  public void reportBacklogSize(String handle, int backlogSize) {}
+
+  @Override
+  public long getTotalReceivedRate() {
+    return 0;
+  }
+
+  @Override
+  public void reportReceivedRate(String handle, long receivedRate) {}
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/data/DefaultGlobalPropertiesForTesting.java b/proxy/src/test/java/com/wavefront/agent/data/DefaultGlobalPropertiesForTesting.java
new file mode 100644
index 000000000..db7e314f1
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/data/DefaultGlobalPropertiesForTesting.java
@@ -0,0 +1,52 @@
+package com.wavefront.agent.data;
+
+import static com.wavefront.agent.data.EntityProperties.DEFAULT_RETRY_BACKOFF_BASE_SECONDS;
+
+import com.wavefront.api.agent.SpanSamplingPolicy;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/** @author vasily@wavefront.com */
+public class DefaultGlobalPropertiesForTesting implements GlobalProperties {
+
+  @Override
+  public double getRetryBackoffBaseSeconds() {
+    return DEFAULT_RETRY_BACKOFF_BASE_SECONDS;
+  }
+
+  @Override
+  public void setRetryBackoffBaseSeconds(@Nullable Double retryBackoffBaseSeconds) {}
+
+  @Override
+  public short getHistogramStorageAccuracy() {
+    return 32;
+  }
+
+  @Override
+  public void setHistogramStorageAccuracy(short histogramStorageAccuracy) {}
+
+  @Override
+  public double getTraceSamplingRate() {
+    return 1.0d;
+  }
+
+  @Override
+  public void setTraceSamplingRate(Double traceSamplingRate) {}
+
+  @Override
+  public Integer getDropSpansDelayedMinutes() {
+    return null;
+  }
+
+  @Override
+  public void setDropSpansDelayedMinutes(Integer dropSpansDelayedMinutes) {}
+
+  @Override
+  public List getActiveSpanSamplingPolicies() {
+    return null;
+  }
+
+  @Override
+  public void setActiveSpanSamplingPolicies(
+      @Nullable List activeSpanSamplingPolicies) {}
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/data/LineDelimitedDataSubmissionTaskTest.java b/proxy/src/test/java/com/wavefront/agent/data/LineDelimitedDataSubmissionTaskTest.java
new file mode 100644
index 000000000..468df48e5
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/data/LineDelimitedDataSubmissionTaskTest.java
@@ -0,0 +1,77 @@
+package com.wavefront.agent.data;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+import com.google.common.collect.ImmutableList;
+import com.wavefront.data.ReportableEntityType;
+import java.util.List;
+import org.junit.Test;
+
+/** @author vasily@wavefront.com */
+public class LineDelimitedDataSubmissionTaskTest {
+  @Test
+  public void testSplitTask() {
+    LineDelimitedDataSubmissionTask task =
+        new LineDelimitedDataSubmissionTask(
+            null,
+            null,
+            null,
+            null,
+            "graphite_v2",
+            ReportableEntityType.POINT,
+            "2878",
+            ImmutableList.of("A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K"),
+            null);
+
+    List split;
+
+    // don't split if task is smaller than min split size
+    split = task.splitTask(11, 4);
+    assertEquals(1, split.size());
+    assertSame(split.get(0), task);
+
+    // split in 2
+    split = task.splitTask(10, 11);
+    assertEquals(2, split.size());
+    assertArrayEquals(new String[] {"A", "B", "C", "D", "E", "F"}, split.get(0).payload.toArray());
+    assertArrayEquals(new String[] {"G", "H", "I", "J", "K"}, split.get(1).payload.toArray());
+
+    split = task.splitTask(10, 6);
+    assertEquals(2, split.size());
+    assertArrayEquals(new String[] {"A", "B", "C", "D", "E", "F"}, split.get(0).payload.toArray());
+    assertArrayEquals(new String[] {"G", "H", "I", "J", "K"}, split.get(1).payload.toArray());
+
+    // split in 3
+    split = task.splitTask(10, 5);
+    assertEquals(3, split.size());
+    assertArrayEquals(new String[] {"A", "B", "C", "D", "E"}, split.get(0).payload.toArray());
+    assertArrayEquals(new String[] {"F", "G", "H", "I", "J"}, split.get(1).payload.toArray());
+    assertArrayEquals(new String[] {"K"}, split.get(2).payload.toArray());
+
+    split = task.splitTask(7, 4);
+    assertEquals(3, split.size());
+    assertArrayEquals(new String[] {"A", "B", "C", "D"}, split.get(0).payload.toArray());
+    assertArrayEquals(new String[] {"E", "F", "G", "H"}, split.get(1).payload.toArray());
+    assertArrayEquals(new String[] {"I", "J", "K"}, split.get(2).payload.toArray());
+
+    // split in 4
+    split = task.splitTask(7, 3);
+    assertEquals(4, split.size());
+    assertArrayEquals(new String[] {"A", "B", "C"}, split.get(0).payload.toArray());
+    assertArrayEquals(new String[] {"D", "E", "F"}, split.get(1).payload.toArray());
+    assertArrayEquals(new String[] {"G", "H", "I"}, split.get(2).payload.toArray());
+    assertArrayEquals(new String[] {"J", "K"}, split.get(3).payload.toArray());
+
+    // split in 6
+    split = task.splitTask(7, 2);
+    assertEquals(6, split.size());
+    assertArrayEquals(new String[] {"A", "B"}, split.get(0).payload.toArray());
+    assertArrayEquals(new String[] {"C", "D"}, split.get(1).payload.toArray());
+    assertArrayEquals(new String[] {"E", "F"}, split.get(2).payload.toArray());
+    assertArrayEquals(new String[] {"G", "H"}, split.get(3).payload.toArray());
+    assertArrayEquals(new String[] {"I", "J"}, split.get(4).payload.toArray());
+    assertArrayEquals(new String[] {"K"}, split.get(5).payload.toArray());
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/data/LogDataSubmissionTaskTest.java b/proxy/src/test/java/com/wavefront/agent/data/LogDataSubmissionTaskTest.java
new file mode 100644
index 000000000..d03b94a78
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/data/LogDataSubmissionTaskTest.java
@@ -0,0 +1,46 @@
+package com.wavefront.agent.data;
+
+import static com.wavefront.agent.data.LogDataSubmissionTask.AGENT_PREFIX;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.api.LogAPI;
+import com.wavefront.dto.Log;
+import java.io.IOException;
+import java.util.UUID;
+import javax.ws.rs.core.Response;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import wavefront.report.ReportLog;
+
+public class LogDataSubmissionTaskTest {
+
+  private final LogAPI logAPI = EasyMock.createMock(LogAPI.class);
+  private final EntityProperties props = new DefaultEntityPropertiesForTesting();
+
+  @Test
+  public void test429() throws IOException {
+    TaskQueue queue = createMock(TaskQueue.class);
+    reset(logAPI, queue);
+    ReportLog testLog = new ReportLog(0L, "msg", "host", ImmutableList.of());
+    Log log = new Log(testLog);
+    UUID uuid = UUID.randomUUID();
+    LogDataSubmissionTask task =
+        new LogDataSubmissionTask(
+            logAPI, uuid, props, queue, "2878", ImmutableList.of(log), System::currentTimeMillis);
+    expect(logAPI.proxyLogs(AGENT_PREFIX + uuid, ImmutableList.of(log)))
+        .andReturn(Response.status(429).build())
+        .once();
+    expectLastCall();
+    replay(logAPI, queue);
+    assertEquals(TaskResult.REMOVED, task.execute());
+    verify(logAPI, queue);
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/data/SourceTagSubmissionTaskTest.java b/proxy/src/test/java/com/wavefront/agent/data/SourceTagSubmissionTaskTest.java
new file mode 100644
index 000000000..6a4148742
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/data/SourceTagSubmissionTaskTest.java
@@ -0,0 +1,184 @@
+package com.wavefront.agent.data;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.*;
+
+import com.google.common.collect.ImmutableList;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.api.SourceTagAPI;
+import com.wavefront.dto.SourceTag;
+import javax.ws.rs.core.Response;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import wavefront.report.ReportSourceTag;
+import wavefront.report.SourceOperationType;
+import wavefront.report.SourceTagAction;
+
+/** @author vasily@wavefront.com */
+public class SourceTagSubmissionTaskTest {
+
+  private SourceTagAPI sourceTagAPI = EasyMock.createMock(SourceTagAPI.class);
+  private final EntityProperties props = new DefaultEntityPropertiesForTesting();
+
+  @Test
+  public void test200() {
+    TaskQueue queue = createMock(TaskQueue.class);
+    reset(sourceTagAPI, queue);
+    ReportSourceTag sourceDescDelete =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_DESCRIPTION,
+            SourceTagAction.DELETE,
+            "dummy",
+            ImmutableList.of());
+    ReportSourceTag sourceTagDelete =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG, SourceTagAction.DELETE, "src", ImmutableList.of("tag"));
+    ReportSourceTag sourceTagAdd =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG, SourceTagAction.ADD, "src", ImmutableList.of("tag"));
+    SourceTagSubmissionTask task =
+        new SourceTagSubmissionTask(
+            sourceTagAPI,
+            props,
+            queue,
+            "2878",
+            new SourceTag(sourceDescDelete),
+            System::currentTimeMillis);
+    SourceTagSubmissionTask task2 =
+        new SourceTagSubmissionTask(
+            sourceTagAPI,
+            props,
+            queue,
+            "2878",
+            new SourceTag(sourceTagDelete),
+            System::currentTimeMillis);
+    SourceTagSubmissionTask task3 =
+        new SourceTagSubmissionTask(
+            sourceTagAPI,
+            props,
+            queue,
+            "2878",
+            new SourceTag(sourceTagAdd),
+            System::currentTimeMillis);
+    expect(sourceTagAPI.removeDescription("dummy")).andReturn(Response.status(200).build()).once();
+    expect(sourceTagAPI.removeTag("src", "tag")).andReturn(Response.status(200).build()).once();
+    expect(sourceTagAPI.appendTag("src", "tag")).andReturn(Response.status(200).build()).once();
+    replay(sourceTagAPI, queue);
+    assertEquals(TaskResult.DELIVERED, task.execute());
+    assertEquals(TaskResult.DELIVERED, task2.execute());
+    assertEquals(TaskResult.DELIVERED, task3.execute());
+    verify(sourceTagAPI, queue);
+  }
+
+  @Test
+  public void test404() throws Exception {
+    TaskQueue queue = createMock(TaskQueue.class);
+    reset(sourceTagAPI, queue);
+    ReportSourceTag sourceDescDelete =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_DESCRIPTION,
+            SourceTagAction.DELETE,
+            "dummy",
+            ImmutableList.of());
+    ReportSourceTag sourceTagDelete =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG, SourceTagAction.DELETE, "src", ImmutableList.of("tag"));
+    ReportSourceTag sourceTagAdd =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG, SourceTagAction.ADD, "src", ImmutableList.of("tag"));
+    SourceTagSubmissionTask task =
+        new SourceTagSubmissionTask(
+            sourceTagAPI,
+            props,
+            queue,
+            "2878",
+            new SourceTag(sourceDescDelete),
+            System::currentTimeMillis);
+    SourceTagSubmissionTask task2 =
+        new SourceTagSubmissionTask(
+            sourceTagAPI,
+            props,
+            queue,
+            "2878",
+            new SourceTag(sourceTagDelete),
+            System::currentTimeMillis);
+    SourceTagSubmissionTask task3 =
+        new SourceTagSubmissionTask(
+            sourceTagAPI,
+            props,
+            queue,
+            "2878",
+            new SourceTag(sourceTagAdd),
+            System::currentTimeMillis);
+    expect(sourceTagAPI.removeDescription("dummy")).andReturn(Response.status(404).build()).once();
+    expect(sourceTagAPI.removeTag("src", "tag")).andReturn(Response.status(404).build()).once();
+    expect(sourceTagAPI.appendTag("src", "tag")).andReturn(Response.status(404).build()).once();
+    queue.add(task3);
+    expectLastCall();
+    replay(sourceTagAPI, queue);
+
+    assertEquals(TaskResult.DELIVERED, task.execute());
+    assertEquals(TaskResult.DELIVERED, task2.execute());
+    assertEquals(TaskResult.PERSISTED, task3.execute());
+    verify(sourceTagAPI, queue);
+  }
+
+  @Test
+  public void test500() throws Exception {
+    TaskQueue queue = createMock(TaskQueue.class);
+    reset(sourceTagAPI, queue);
+    ReportSourceTag sourceDescDelete =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_DESCRIPTION,
+            SourceTagAction.DELETE,
+            "dummy",
+            ImmutableList.of());
+    ReportSourceTag sourceTagDelete =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG, SourceTagAction.DELETE, "src", ImmutableList.of("tag"));
+    ReportSourceTag sourceTagAdd =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG, SourceTagAction.ADD, "src", ImmutableList.of("tag"));
+    SourceTagSubmissionTask task =
+        new SourceTagSubmissionTask(
+            sourceTagAPI,
+            props,
+            queue,
+            "2878",
+            new SourceTag(sourceDescDelete),
+            System::currentTimeMillis);
+    SourceTagSubmissionTask task2 =
+        new SourceTagSubmissionTask(
+            sourceTagAPI,
+            props,
+            queue,
+            "2878",
+            new SourceTag(sourceTagDelete),
+            System::currentTimeMillis);
+    SourceTagSubmissionTask task3 =
+        new SourceTagSubmissionTask(
+            sourceTagAPI,
+            props,
+            queue,
+            "2878",
+            new SourceTag(sourceTagAdd),
+            System::currentTimeMillis);
+    expect(sourceTagAPI.removeDescription("dummy")).andReturn(Response.status(500).build()).once();
+    expect(sourceTagAPI.removeTag("src", "tag")).andReturn(Response.status(500).build()).once();
+    expect(sourceTagAPI.appendTag("src", "tag")).andReturn(Response.status(500).build()).once();
+    queue.add(task);
+    queue.add(task2);
+    queue.add(task3);
+    expectLastCall();
+    replay(sourceTagAPI, queue);
+    assertEquals(TaskResult.PERSISTED, task.execute());
+    assertEquals(TaskResult.PERSISTED, task2.execute());
+    assertEquals(TaskResult.PERSISTED, task3.execute());
+    verify(sourceTagAPI, queue);
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/formatter/GraphiteFormatterTest.java b/proxy/src/test/java/com/wavefront/agent/formatter/GraphiteFormatterTest.java
index a4b1480e4..f9da23128 100644
--- a/proxy/src/test/java/com/wavefront/agent/formatter/GraphiteFormatterTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/formatter/GraphiteFormatterTest.java
@@ -1,20 +1,24 @@
 package com.wavefront.agent.formatter;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/**
- * @author Andrew Kao (andrew@wavefront.com)
- */
+/** @author Andrew Kao (andrew@wavefront.com) */
 public class GraphiteFormatterTest {
 
   private static final Logger logger = LoggerFactory.getLogger(GraphiteFormatterTest.class);
 
   @Test
   public void testCollectdGraphiteParsing() {
-    String format = "4,3,2"; // Extract the 4th, 3rd, and 2nd segments of the metric as the hostname, in that order
+    String format =
+        "4,3,2"; // Extract the 4th, 3rd, and 2nd segments of the metric as the hostname, in
+    // that
+    // order
     String format2 = "2";
     String delimiter = "_";
 
@@ -23,12 +27,14 @@ public void testCollectdGraphiteParsing() {
     String testString2 = "collectd.com.bigcorp.www02_web.cpu.loadavg.1m 40 1415233342";
     String testString3 = "collectd.almost.too.short 40 1415233342";
     String testString4 = "collectd.too.short 40 1415233342";
-    String testString5 = "collectd.www02_web_bigcorp_com.cpu.loadavg.1m;context=abc;hostname=www02.web.bigcorp.com 40 1415233342";
+    String testString5 =
+        "collectd.www02_web_bigcorp_com.cpu.loadavg.1m;context=abc;hostname=www02.web.bigcorp.com 40 1415233342";
 
     // Test output
     String expected1 = "collectd.cpu.loadavg.1m 40 source=www02.web.bigcorp.com";
     String expected2 = "collectd.cpu.loadavg.1m 40 1415233342 source=www02.web.bigcorp.com";
-    String expected5 = "collectd.cpu.loadavg.1m 40 1415233342 source=www02.web.bigcorp.com context=abc hostname=www02.web.bigcorp.com";
+    String expected5 =
+        "collectd.cpu.loadavg.1m 40 1415233342 source=www02.web.bigcorp.com context=abc hostname=www02.web.bigcorp.com";
 
     // Test basic functionality with correct input
     GraphiteFormatter formatter = new GraphiteFormatter(format, delimiter, "");
@@ -67,11 +73,13 @@ public void testCollectdGraphiteParsing() {
     long end = System.nanoTime();
 
     // Report/validate performance
-    logger.error(" Time to parse 1M strings: " + (end - start) + " ns for " + formatter.getOps() + " runs");
+    logger.error(
+        " Time to parse 1M strings: " + (end - start) + " ns for " + formatter.getOps() + " runs");
     long nsPerOps = (end - start) / formatter.getOps();
     logger.error(" ns per op: " + nsPerOps + " and ops/sec " + (1000 * 1000 * 1000 / nsPerOps));
-    assertTrue(formatter.getOps() >= 1000 * 1000);  // make sure we actually ran it 1M times
-    assertTrue(nsPerOps < 10 * 1000); // make sure it was less than 10 Îŧs per run; it's around 1 Îŧs on my machine
+    assertTrue(formatter.getOps() >= 1000 * 1000); // make sure we actually ran it 1M times
+    assertTrue(nsPerOps < 10 * 1000); // make sure it was less than 10 Îŧs per run; it's around 1
+    // Îŧs on my machine
 
     // new addition to test the point tags inside the metric names
     formatter = new GraphiteFormatter(format2, delimiter, "");
@@ -83,7 +91,7 @@ public void testCollectdGraphiteParsing() {
   public void testFieldsToRemove() {
     String format = "2"; // Extract the 2nd field for host name
     String delimiter = "_";
-    String remove = "1,3";  // remove the 1st and 3rd fields from metric name
+    String remove = "1,3"; // remove the 1st and 3rd fields from metric name
 
     // Test input
     String testString1 = "hosts.host1.collectd.cpu.loadavg.1m 40";
@@ -125,4 +133,4 @@ public void testFieldsToRemoveInvalidFormats() {
       // expected
     }
   }
-}
\ No newline at end of file
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/handlers/MockReportableEntityHandlerFactory.java b/proxy/src/test/java/com/wavefront/agent/handlers/MockReportableEntityHandlerFactory.java
index 775e3906a..ad6425047 100644
--- a/proxy/src/test/java/com/wavefront/agent/handlers/MockReportableEntityHandlerFactory.java
+++ b/proxy/src/test/java/com/wavefront/agent/handlers/MockReportableEntityHandlerFactory.java
@@ -1,10 +1,14 @@
 package com.wavefront.agent.handlers;
 
+import com.wavefront.dto.Event;
+import com.wavefront.dto.SourceTag;
+import javax.annotation.Nonnull;
 import org.easymock.EasyMock;
-
+import wavefront.report.ReportEvent;
 import wavefront.report.ReportPoint;
 import wavefront.report.ReportSourceTag;
 import wavefront.report.Span;
+import wavefront.report.SpanLogs;
 
 /**
  * Mock factory for testing
@@ -29,25 +33,45 @@ public static SpanHandlerImpl getMockTraceHandler() {
     return EasyMock.createMock(SpanHandlerImpl.class);
   }
 
+  public static SpanLogsHandlerImpl getMockTraceSpanLogsHandler() {
+    return EasyMock.createMock(SpanLogsHandlerImpl.class);
+  }
+
+  public static EventHandlerImpl getMockEventHandlerImpl() {
+    return EasyMock.createMock(EventHandlerImpl.class);
+  }
+
   public static ReportableEntityHandlerFactory createMockHandlerFactory(
-      ReportableEntityHandler mockReportPointHandler,
-      ReportableEntityHandler mockSourceTagHandler,
-      ReportableEntityHandler mockHistogramHandler,
-      ReportableEntityHandler mockTraceHandler) {
-    return handlerKey -> {
-      switch (handlerKey.getEntityType()) {
-        case POINT:
-          return mockReportPointHandler;
-        case SOURCE_TAG:
-          return mockSourceTagHandler;
-        case HISTOGRAM:
-          return mockHistogramHandler;
-        case TRACE:
-          return mockTraceHandler;
-        default:
-          throw new IllegalArgumentException("Unknown entity type");
+      ReportableEntityHandler mockReportPointHandler,
+      ReportableEntityHandler mockSourceTagHandler,
+      ReportableEntityHandler mockHistogramHandler,
+      ReportableEntityHandler mockTraceHandler,
+      ReportableEntityHandler mockTraceSpanLogsHandler,
+      ReportableEntityHandler mockEventHandler) {
+    return new ReportableEntityHandlerFactory() {
+      @SuppressWarnings("unchecked")
+      @Override
+      public  ReportableEntityHandler getHandler(HandlerKey handlerKey) {
+        switch (handlerKey.getEntityType()) {
+          case POINT:
+            return (ReportableEntityHandler) mockReportPointHandler;
+          case SOURCE_TAG:
+            return (ReportableEntityHandler) mockSourceTagHandler;
+          case HISTOGRAM:
+            return (ReportableEntityHandler) mockHistogramHandler;
+          case TRACE:
+            return (ReportableEntityHandler) mockTraceHandler;
+          case TRACE_SPAN_LOGS:
+            return (ReportableEntityHandler) mockTraceSpanLogsHandler;
+          case EVENT:
+            return (ReportableEntityHandler) mockEventHandler;
+          default:
+            throw new IllegalArgumentException("Unknown entity type");
+        }
       }
+
+      @Override
+      public void shutdown(@Nonnull String handle) {}
     };
   }
-
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/handlers/ReportSourceTagHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/handlers/ReportSourceTagHandlerTest.java
index feb380302..d00d4a06d 100644
--- a/proxy/src/test/java/com/wavefront/agent/handlers/ReportSourceTagHandlerTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/handlers/ReportSourceTagHandlerTest.java
@@ -1,25 +1,31 @@
 package com.wavefront.agent.handlers;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.util.concurrent.RecyclableRateLimiter;
-
-import com.wavefront.agent.api.ForceQueueEnabledAgentAPI;
+import com.google.common.collect.ImmutableMap;
+import com.wavefront.agent.api.APIContainer;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.data.DefaultEntityPropertiesFactoryForTesting;
+import com.wavefront.agent.queueing.TaskQueue;
+import com.wavefront.agent.queueing.TaskQueueFactory;
+import com.wavefront.api.SourceTagAPI;
 import com.wavefront.data.ReportableEntityType;
-
-import org.easymock.EasyMock;
-import org.junit.Before;
-import org.junit.Test;
-
+import com.wavefront.dto.SourceTag;
+import edu.emory.mathcs.backport.java.util.Collections;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
+import java.util.logging.Logger;
+import javax.annotation.Nonnull;
 import javax.ws.rs.core.Response;
-
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
 import wavefront.report.ReportSourceTag;
+import wavefront.report.SourceOperationType;
+import wavefront.report.SourceTagAction;
 
 /**
  * This class tests the ReportSourceTagHandler.
@@ -30,69 +36,178 @@ public class ReportSourceTagHandlerTest {
 
   private ReportSourceTagHandlerImpl sourceTagHandler;
   private SenderTaskFactory senderTaskFactory;
-  private ForceQueueEnabledAgentAPI mockAgentAPI;
+  private SourceTagAPI mockAgentAPI;
+  private TaskQueueFactory taskQueueFactory;
   private UUID newAgentId;
+  private HandlerKey handlerKey;
+  private Logger blockedLogger = Logger.getLogger("RawBlockedPoints");
 
   @Before
   public void setup() {
-    mockAgentAPI = EasyMock.createMock(ForceQueueEnabledAgentAPI.class);
+    mockAgentAPI = EasyMock.createMock(SourceTagAPI.class);
+    taskQueueFactory =
+        new TaskQueueFactory() {
+          @Override
+          public > TaskQueue getTaskQueue(
+              @Nonnull HandlerKey handlerKey, int threadNum) {
+            return null;
+          }
+        };
     newAgentId = UUID.randomUUID();
-    senderTaskFactory = new SenderTaskFactoryImpl(mockAgentAPI, newAgentId, null, new AtomicInteger(100),
-        new AtomicInteger(10), new AtomicInteger(1000));
-    sourceTagHandler = new ReportSourceTagHandlerImpl("4878", 10, senderTaskFactory.createSenderTasks(
-        HandlerKey.of(ReportableEntityType.SOURCE_TAG, "4878"), 2));
+    senderTaskFactory =
+        new SenderTaskFactoryImpl(
+            new APIContainer(null, mockAgentAPI, null, null, null),
+            newAgentId,
+            taskQueueFactory,
+            null,
+            Collections.singletonMap(
+                APIContainer.CENTRAL_TENANT_NAME, new DefaultEntityPropertiesFactoryForTesting()));
+
+    handlerKey = HandlerKey.of(ReportableEntityType.SOURCE_TAG, "4878");
+    sourceTagHandler =
+        new ReportSourceTagHandlerImpl(
+            handlerKey, 10, senderTaskFactory.createSenderTasks(handlerKey), null, blockedLogger);
   }
 
-  /**
-   * This test will add 3 source tags and verify that the server side api is called properly.
-   *
-   */
+  /** This test will add 3 source tags and verify that the server side api is called properly. */
   @Test
-  public void testSourceTagsSetting() throws Exception {
-    String[] annotations = new String[]{"tag1", "tag2", "tag3"};
-    ReportSourceTag sourceTag = new ReportSourceTag("SourceTag", "save", "dummy", "desc", Arrays.asList
-        (annotations));
-    EasyMock.expect(mockAgentAPI.setTags("dummy", Arrays.asList(annotations), false)).andReturn(
-        Response.ok().build()).once();
+  public void testSourceTagsSetting() {
+    String[] annotations = new String[] {"tag1", "tag2", "tag3"};
+    ReportSourceTag sourceTag =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG,
+            SourceTagAction.SAVE,
+            "dummy",
+            Arrays.asList(annotations));
+    EasyMock.expect(mockAgentAPI.setTags("dummy", Arrays.asList(annotations)))
+        .andReturn(Response.ok().build())
+        .once();
+    EasyMock.replay(mockAgentAPI);
+    sourceTagHandler.report(sourceTag);
+    ((SenderTaskFactoryImpl) senderTaskFactory).flushNow(handlerKey);
+    EasyMock.verify(mockAgentAPI);
+  }
 
+  @Test
+  public void testSourceTagAppend() {
+    ReportSourceTag sourceTag =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG, SourceTagAction.ADD, "dummy", ImmutableList.of("tag1"));
+    EasyMock.expect(mockAgentAPI.appendTag("dummy", "tag1"))
+        .andReturn(Response.ok().build())
+        .once();
     EasyMock.replay(mockAgentAPI);
+    sourceTagHandler.report(sourceTag);
+    ((SenderTaskFactoryImpl) senderTaskFactory).flushNow(handlerKey);
+    EasyMock.verify(mockAgentAPI);
+  }
+
+  @Test
+  public void testSourceTagDelete() {
+    ReportSourceTag sourceTag =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG,
+            SourceTagAction.DELETE,
+            "dummy",
+            ImmutableList.of("tag1"));
+    EasyMock.expect(mockAgentAPI.removeTag("dummy", "tag1"))
+        .andReturn(Response.ok().build())
+        .once();
+    EasyMock.replay(mockAgentAPI);
+    sourceTagHandler.report(sourceTag);
+    ((SenderTaskFactoryImpl) senderTaskFactory).flushNow(handlerKey);
+    EasyMock.verify(mockAgentAPI);
+  }
 
+  @Test
+  public void testSourceAddDescription() {
+    ReportSourceTag sourceTag =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_DESCRIPTION,
+            SourceTagAction.SAVE,
+            "dummy",
+            ImmutableList.of("description"));
+    EasyMock.expect(mockAgentAPI.setDescription("dummy", "description"))
+        .andReturn(Response.ok().build())
+        .once();
+    EasyMock.replay(mockAgentAPI);
+    sourceTagHandler.report(sourceTag);
+    ((SenderTaskFactoryImpl) senderTaskFactory).flushNow(handlerKey);
+    EasyMock.verify(mockAgentAPI);
+  }
+
+  @Test
+  public void testSourceDeleteDescription() {
+    ReportSourceTag sourceTag =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_DESCRIPTION,
+            SourceTagAction.DELETE,
+            "dummy",
+            ImmutableList.of());
+    EasyMock.expect(mockAgentAPI.removeDescription("dummy"))
+        .andReturn(Response.ok().build())
+        .once();
+    EasyMock.replay(mockAgentAPI);
     sourceTagHandler.report(sourceTag);
-    TimeUnit.SECONDS.sleep(1);
+    ((SenderTaskFactoryImpl) senderTaskFactory).flushNow(handlerKey);
     EasyMock.verify(mockAgentAPI);
   }
 
   @Test
   public void testSourceTagsTaskAffinity() {
-    ReportSourceTag sourceTag1 = new ReportSourceTag("SourceTag", "save", "dummy", "desc",
-        ImmutableList.of("tag1", "tag2"));
-    ReportSourceTag sourceTag2 = new ReportSourceTag("SourceTag", "save", "dummy", "desc 2",
-        ImmutableList.of("tag2", "tag3"));
-    ReportSourceTag sourceTag3 = new ReportSourceTag("SourceTag", "save", "dummy-2", "desc 3",
-        ImmutableList.of("tag3"));
-    ReportSourceTag sourceTag4 = new ReportSourceTag("SourceTag", "save", "dummy", "desc 4",
-        ImmutableList.of("tag1", "tag4", "tag5"));
-    List tasks = new ArrayList<>();
-    ReportSourceTagSenderTask task1 = EasyMock.createMock(ReportSourceTagSenderTask.class);
-    ReportSourceTagSenderTask task2 = EasyMock.createMock(ReportSourceTagSenderTask.class);
+    ReportSourceTag sourceTag1 =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG,
+            SourceTagAction.SAVE,
+            "dummy",
+            ImmutableList.of("tag1", "tag2"));
+    ReportSourceTag sourceTag2 =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG,
+            SourceTagAction.SAVE,
+            "dummy",
+            ImmutableList.of("tag2", "tag3"));
+    ReportSourceTag sourceTag3 =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG,
+            SourceTagAction.SAVE,
+            "dummy-2",
+            ImmutableList.of("tag3"));
+    ReportSourceTag sourceTag4 =
+        new ReportSourceTag(
+            SourceOperationType.SOURCE_TAG,
+            SourceTagAction.SAVE,
+            "dummy",
+            ImmutableList.of("tag1", "tag4", "tag5"));
+    List> tasks = new ArrayList<>();
+    SourceTagSenderTask task1 = EasyMock.createMock(SourceTagSenderTask.class);
+    SourceTagSenderTask task2 = EasyMock.createMock(SourceTagSenderTask.class);
     tasks.add(task1);
     tasks.add(task2);
-    ReportSourceTagHandlerImpl sourceTagHandler = new ReportSourceTagHandlerImpl("4878", 10, tasks);
-    task1.add(sourceTag1);
+    Map>> taskMap =
+        ImmutableMap.of(APIContainer.CENTRAL_TENANT_NAME, tasks);
+    ReportSourceTagHandlerImpl sourceTagHandler =
+        new ReportSourceTagHandlerImpl(
+            HandlerKey.of(ReportableEntityType.SOURCE_TAG, "4878"),
+            10,
+            taskMap,
+            null,
+            blockedLogger);
+    task1.add(new SourceTag(sourceTag1));
     EasyMock.expectLastCall();
-    task1.add(sourceTag2);
+    task1.add(new SourceTag(sourceTag2));
     EasyMock.expectLastCall();
-    task2.add(sourceTag3);
+    task2.add(new SourceTag(sourceTag3));
     EasyMock.expectLastCall();
-    task1.add(sourceTag4);
+    task1.add(new SourceTag(sourceTag4));
     EasyMock.expectLastCall();
-    task1.add(sourceTag4);
+    task1.add(new SourceTag(sourceTag4));
     EasyMock.expectLastCall();
-    task2.add(sourceTag3);
+    task2.add(new SourceTag(sourceTag3));
     EasyMock.expectLastCall();
-    task1.add(sourceTag2);
+    task1.add(new SourceTag(sourceTag2));
     EasyMock.expectLastCall();
-    task1.add(sourceTag1);
+    task1.add(new SourceTag(sourceTag1));
     EasyMock.expectLastCall();
 
     EasyMock.replay(task1);
diff --git a/proxy/src/test/java/com/wavefront/agent/histogram/HistogramRecompressorTest.java b/proxy/src/test/java/com/wavefront/agent/histogram/HistogramRecompressorTest.java
new file mode 100644
index 000000000..616b4b202
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/histogram/HistogramRecompressorTest.java
@@ -0,0 +1,61 @@
+package com.wavefront.agent.histogram;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Test;
+import wavefront.report.Histogram;
+import wavefront.report.HistogramType;
+
+/** @author vasily@wavefront.com */
+public class HistogramRecompressorTest {
+
+  @Test
+  public void testHistogramRecompressor() {
+    HistogramRecompressor recompressor = new HistogramRecompressor(() -> (short) 32);
+    Histogram testHistogram =
+        Histogram.newBuilder()
+            .setType(HistogramType.TDIGEST)
+            .setDuration(60000)
+            .setBins(ImmutableList.of(1.0, 2.0, 3.0))
+            .setCounts(ImmutableList.of(3, 2, 1))
+            .build();
+    Histogram outputHistoram = recompressor.apply(testHistogram);
+    // nothing to compress
+    assertEquals(outputHistoram, testHistogram);
+
+    testHistogram.setBins(ImmutableList.of(1.0, 1.0, 1.0, 2.0, 3.0, 3.0, 3.0));
+    testHistogram.setCounts(ImmutableList.of(3, 1, 2, 2, 1, 2, 3));
+    outputHistoram = recompressor.apply(testHistogram);
+    // compacted histogram
+    assertEquals(ImmutableList.of(1.0, 2.0, 3.0), outputHistoram.getBins());
+    assertEquals(ImmutableList.of(6, 2, 6), outputHistoram.getCounts());
+
+    List bins = new ArrayList<>();
+    List counts = new ArrayList<>();
+    for (int i = 0; i < 1000; i++) {
+      bins.add((double) i);
+      counts.add(1);
+    }
+    testHistogram.setBins(bins);
+    testHistogram.setCounts(counts);
+    outputHistoram = recompressor.apply(testHistogram);
+    assertTrue(outputHistoram.getBins().size() < 48);
+    assertEquals(1000, outputHistoram.getCounts().stream().mapToInt(x -> x).sum());
+
+    testHistogram.setBins(bins.subList(0, 65));
+    testHistogram.setCounts(counts.subList(0, 65));
+    outputHistoram = recompressor.apply(testHistogram);
+    assertTrue(outputHistoram.getBins().size() < 48);
+    assertEquals(65, outputHistoram.getCounts().stream().mapToInt(x -> x).sum());
+
+    testHistogram.setBins(bins.subList(0, 64));
+    testHistogram.setCounts(counts.subList(0, 64));
+    outputHistoram = recompressor.apply(testHistogram);
+    assertEquals(64, outputHistoram.getBins().size());
+    assertEquals(64, outputHistoram.getCounts().stream().mapToInt(x -> x).sum());
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/histogram/MapLoaderTest.java b/proxy/src/test/java/com/wavefront/agent/histogram/MapLoaderTest.java
index 53cde64c2..994556ef5 100644
--- a/proxy/src/test/java/com/wavefront/agent/histogram/MapLoaderTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/histogram/MapLoaderTest.java
@@ -1,57 +1,57 @@
 package com.wavefront.agent.histogram;
 
+import static com.google.common.truth.Truth.assertThat;
+import static com.wavefront.agent.histogram.TestUtils.makeKey;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 import com.tdunning.math.stats.AgentDigest;
 import com.tdunning.math.stats.AgentDigest.AgentDigestMarshaller;
-import com.wavefront.agent.histogram.Utils.HistogramKey;
-import com.wavefront.agent.histogram.Utils.HistogramKeyMarshaller;
-
+import com.wavefront.agent.histogram.HistogramUtils.HistogramKeyMarshaller;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.concurrent.ConcurrentMap;
+import net.openhft.chronicle.hash.ChronicleHashRecoveryFailedException;
 import net.openhft.chronicle.map.ChronicleMap;
 import net.openhft.chronicle.map.VanillaChronicleMap;
-
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.concurrent.ConcurrentMap;
-
-import static com.google.common.truth.Truth.assertThat;
-import static com.wavefront.agent.histogram.TestUtils.makeKey;
-
 /**
  * Unit tests around {@link MapLoader}.
  *
  * @author Tim Schmidt (tim@wavefront.com).
  */
 public class MapLoaderTest {
-  private final static short COMPRESSION = 100;
+  private static final short COMPRESSION = 100;
 
   private HistogramKey key = makeKey("mapLoaderTest");
   private AgentDigest digest = new AgentDigest(COMPRESSION, 100L);
   private File file;
-  private MapLoader loader;
+  private MapLoader
+      loader;
 
   @Before
   public void setup() {
     try {
-      file = File.createTempFile("test-file", ".tmp");
-      file.deleteOnExit();
+      file = new File(File.createTempFile("test-file-chronicle", null).getPath() + ".map");
+
     } catch (IOException e) {
       e.printStackTrace();
     }
 
-    loader = new MapLoader<>(
-        HistogramKey.class,
-        AgentDigest.class,
-        100,
-        200,
-        1000,
-        HistogramKeyMarshaller.get(),
-        AgentDigestMarshaller.get(),
-        true);
+    loader =
+        new MapLoader<>(
+            HistogramKey.class,
+            AgentDigest.class,
+            100,
+            200,
+            1000,
+            HistogramKeyMarshaller.get(),
+            AgentDigestMarshaller.get(),
+            true);
   }
 
   @After
@@ -68,62 +68,81 @@ private void testPutRemove(ConcurrentMap map) {
   }
 
   @Test
-  public void testPersistence() {
-    ConcurrentMap map = loader.get(file);
+  public void testReconfigureMap() throws Exception {
+    ChronicleMap map = loader.get(file);
     map.put(key, digest);
-
-    loader = new MapLoader<>(
-        HistogramKey.class,
-        AgentDigest.class,
-        100,
-        200,
-        1000,
-        HistogramKeyMarshaller.get(),
-        AgentDigestMarshaller.get(),
-        true);
-
+    map.close();
+    loader =
+        new MapLoader<>(
+            HistogramKey.class,
+            AgentDigest.class,
+            50,
+            100,
+            500,
+            HistogramKeyMarshaller.get(),
+            AgentDigestMarshaller.get(),
+            true);
     map = loader.get(file);
+    assertThat(map).containsKey(key);
+  }
 
+  @Test
+  public void testPersistence() throws Exception {
+    ChronicleMap map = loader.get(file);
+    map.put(key, digest);
+    map.close();
+    loader =
+        new MapLoader<>(
+            HistogramKey.class,
+            AgentDigest.class,
+            100,
+            200,
+            1000,
+            HistogramKeyMarshaller.get(),
+            AgentDigestMarshaller.get(),
+            true);
+    map = loader.get(file);
     assertThat(map).containsKey(key);
   }
 
   @Test
-  public void testFileDoesNotExist() throws IOException {
+  public void testFileDoesNotExist() throws Exception {
     file.delete();
     ConcurrentMap map = loader.get(file);
-    assertThat(((VanillaChronicleMap)map).file()).isNotNull();
+    assertThat(((VanillaChronicleMap) map).file()).isNotNull();
     testPutRemove(map);
   }
 
   @Test
-  public void testDoNotPersist() throws IOException {
-    loader = new MapLoader<>(
-        HistogramKey.class,
-        AgentDigest.class,
-        100,
-        200,
-        1000,
-        HistogramKeyMarshaller.get(),
-        AgentDigestMarshaller.get(),
-        false);
+  public void testDoNotPersist() throws Exception {
+    loader =
+        new MapLoader<>(
+            HistogramKey.class,
+            AgentDigest.class,
+            100,
+            200,
+            1000,
+            HistogramKeyMarshaller.get(),
+            AgentDigestMarshaller.get(),
+            false);
 
     ConcurrentMap map = loader.get(file);
-    assertThat(((VanillaChronicleMap)map).file()).isNull();
+    assertThat(((VanillaChronicleMap) map).file()).isNull();
     testPutRemove(map);
   }
 
-
-  // NOTE: Chronicle's repair attempt takes >1min for whatever reason.
-  @Ignore
   @Test
-  public void testCorruptedFileFallsBackToInMemory() throws IOException {
-    FileOutputStream fos = new FileOutputStream(file);
-    fos.write("Nonsense".getBytes());
-    fos.flush();
-
-    ConcurrentMap map = loader.get(file);
-    assertThat(((VanillaChronicleMap)map).file()).isNull();
-
-    testPutRemove(map);
+  public void testCorruptedFileThrows() throws Exception {
+    try (FileOutputStream fos = new FileOutputStream(file)) {
+      // fake broken ChronicleMap header so it doesn't wait 1 minute for file sync
+      fos.write(new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 127, 127, 127, 127});
+      fos.flush();
+    }
+    try {
+      loader.get(file);
+      fail();
+    } catch (RuntimeException e) {
+      assertTrue(e.getCause().getCause() instanceof ChronicleHashRecoveryFailedException);
+    }
   }
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/histogram/PointHandlerDispatcherTest.java b/proxy/src/test/java/com/wavefront/agent/histogram/PointHandlerDispatcherTest.java
index 7123c5b57..e5195e56d 100644
--- a/proxy/src/test/java/com/wavefront/agent/histogram/PointHandlerDispatcherTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/histogram/PointHandlerDispatcherTest.java
@@ -1,82 +1,103 @@
 package com.wavefront.agent.histogram;
 
+import static com.google.common.truth.Truth.assertThat;
+
 import com.tdunning.math.stats.AgentDigest;
-import com.wavefront.agent.PointHandler;
+import com.wavefront.agent.formatter.DataFormat;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
 import com.wavefront.agent.histogram.accumulator.AccumulationCache;
-
-import org.junit.Before;
-import org.junit.Test;
-
+import com.wavefront.agent.histogram.accumulator.AgentDigestFactory;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicLong;
-
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.junit.Before;
+import org.junit.Test;
 import wavefront.report.ReportPoint;
 
-import static com.google.common.truth.Truth.assertThat;
-
-/**
- * @author Tim Schmidt (tim@wavefront.com).
- */
+/** @author Tim Schmidt (tim@wavefront.com). */
 public class PointHandlerDispatcherTest {
-  private final static short COMPRESSION = 100;
+  private static final short COMPRESSION = 100;
 
   private AccumulationCache in;
-  private ConcurrentMap backingStore;
+  private ConcurrentMap backingStore;
   private List pointOut;
   private List debugLineOut;
-  private List blockedOut;
+  private List blockedOut;
   private AtomicLong timeMillis;
   private PointHandlerDispatcher subject;
 
-  private Utils.HistogramKey keyA = TestUtils.makeKey("keyA");
-  private Utils.HistogramKey keyB = TestUtils.makeKey("keyB");
+  private HistogramKey keyA = TestUtils.makeKey("keyA");
+  private HistogramKey keyB = TestUtils.makeKey("keyB");
   private AgentDigest digestA;
   private AgentDigest digestB;
 
-
   @Before
   public void setup() {
     timeMillis = new AtomicLong(0L);
     backingStore = new ConcurrentHashMap<>();
-    in = new AccumulationCache(backingStore, 0, timeMillis::get);
+    AgentDigestFactory agentDigestFactory =
+        new AgentDigestFactory(() -> COMPRESSION, 100L, timeMillis::get);
+    in = new AccumulationCache(backingStore, agentDigestFactory, 0, "", timeMillis::get);
     pointOut = new LinkedList<>();
     debugLineOut = new LinkedList<>();
     blockedOut = new LinkedList<>();
     digestA = new AgentDigest(COMPRESSION, 100L);
     digestB = new AgentDigest(COMPRESSION, 1000L);
-    subject = new PointHandlerDispatcher(in, new PointHandler() {
-
-      @Override
-      public void reportPoint(ReportPoint point, String debugLine) {
-        pointOut.add(point);
-        debugLineOut.add(debugLine);
-      }
-
-      @Override
-      public void reportPoints(List points) {
-        pointOut.addAll(points);
-      }
-
-      @Override
-      public void handleBlockedPoint(String pointLine) {
-        blockedOut.add(pointLine);
-      }
-    }, timeMillis::get, null, null);
+    subject =
+        new PointHandlerDispatcher(
+            in,
+            new ReportableEntityHandler() {
+
+              @Override
+              public void report(ReportPoint reportPoint) {
+                pointOut.add(reportPoint);
+              }
+
+              @Override
+              public void block(ReportPoint reportPoint) {
+                blockedOut.add(reportPoint);
+              }
+
+              @Override
+              public void block(@Nullable ReportPoint reportPoint, @Nullable String message) {
+                blockedOut.add(reportPoint);
+              }
+
+              @Override
+              public void reject(@Nullable ReportPoint reportPoint, @Nullable String message) {
+                blockedOut.add(reportPoint);
+              }
+
+              @Override
+              public void reject(@Nonnull String t, @Nullable String message) {}
+
+              @Override
+              public void setLogFormat(DataFormat format) {
+                throw new UnsupportedOperationException();
+              }
+
+              @Override
+              public void shutdown() {}
+            },
+            timeMillis::get,
+            () -> false,
+            null,
+            null);
   }
 
   @Test
   public void testBasicDispatch() {
     in.put(keyA, digestA);
-    in.getResolveTask().run();
+    in.flush();
 
     timeMillis.set(101L);
     subject.run();
 
     assertThat(pointOut).hasSize(1);
-    assertThat(debugLineOut).hasSize(1);
     assertThat(blockedOut).hasSize(0);
     assertThat(backingStore).isEmpty();
 
@@ -89,14 +110,13 @@ public void testBasicDispatch() {
   public void testOnlyRipeEntriesAreDispatched() {
     in.put(keyA, digestA);
     in.put(keyB, digestB);
-    in.getResolveTask().run();
+    in.flush();
 
     timeMillis.set(101L);
     subject.run();
-    in.getResolveTask().run();
+    in.flush();
 
     assertThat(pointOut).hasSize(1);
-    assertThat(debugLineOut).hasSize(1);
     assertThat(blockedOut).hasSize(0);
     assertThat(backingStore).containsEntry(keyB, digestB);
 
diff --git a/proxy/src/test/java/com/wavefront/agent/histogram/TapeDispatcherTest.java b/proxy/src/test/java/com/wavefront/agent/histogram/TapeDispatcherTest.java
deleted file mode 100644
index 097011cba..000000000
--- a/proxy/src/test/java/com/wavefront/agent/histogram/TapeDispatcherTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-package com.wavefront.agent.histogram;
-
-import com.squareup.tape.InMemoryObjectQueue;
-import com.squareup.tape.ObjectQueue;
-import com.tdunning.math.stats.AgentDigest;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import wavefront.report.ReportPoint;
-
-import static com.google.common.truth.Truth.assertThat;
-
-/**
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class TapeDispatcherTest {
-  private final static short COMPRESSION = 100;
-
-  private ConcurrentMap in;
-  private ObjectQueue out;
-  private AtomicLong timeMillis;
-  private TapeDispatcher subject;
-
-  private Utils.HistogramKey keyA = TestUtils.makeKey("keyA");
-  private Utils.HistogramKey keyB = TestUtils.makeKey("keyB");
-  private AgentDigest digestA;
-  private AgentDigest digestB;
-
-
-  @Before
-  public void setup() {
-    in = new ConcurrentHashMap<>();
-    out = new InMemoryObjectQueue<>();
-    digestA = new AgentDigest(COMPRESSION, 100L);
-    digestB = new AgentDigest(COMPRESSION, 1000L);
-    timeMillis = new AtomicLong(0L);
-    subject = new TapeDispatcher(in, out, timeMillis::get);
-  }
-
-  @Test
-  public void testBasicDispatch() {
-    in.put(keyA, digestA);
-
-    timeMillis.set(TimeUnit.MILLISECONDS.toNanos(101L));
-    subject.run();
-
-    assertThat(out.size()).isEqualTo(1);
-    assertThat(in).isEmpty();
-
-    ReportPoint point = out.peek();
-
-    TestUtils.testKeyPointMatch(keyA, point);
-  }
-
-  @Test
-  public void testOnlyRipeEntriesAreDispatched() {
-    in.put(keyA, digestA);
-    in.put(keyB, digestB);
-
-    timeMillis.set(101L);
-    subject.run();
-
-    assertThat(out.size()).isEqualTo(1);
-    assertThat(in).containsEntry(keyB, digestB);
-
-    ReportPoint point = out.peek();
-
-    TestUtils.testKeyPointMatch(keyA, point);
-  }
-}
diff --git a/proxy/src/test/java/com/wavefront/agent/histogram/TestUtils.java b/proxy/src/test/java/com/wavefront/agent/histogram/TestUtils.java
index e81781876..83e923ebd 100644
--- a/proxy/src/test/java/com/wavefront/agent/histogram/TestUtils.java
+++ b/proxy/src/test/java/com/wavefront/agent/histogram/TestUtils.java
@@ -1,13 +1,12 @@
 package com.wavefront.agent.histogram;
 
-import java.util.concurrent.TimeUnit;
+import static com.google.common.truth.Truth.assertThat;
 
+import com.google.common.collect.ImmutableMap;
+import java.util.concurrent.TimeUnit;
 import wavefront.report.Histogram;
 import wavefront.report.ReportPoint;
 
-import static com.google.common.truth.Truth.assertThat;
-import static com.wavefront.agent.histogram.Utils.*;
-
 /**
  * Shared test helpers around histograms
  *
@@ -18,28 +17,33 @@ private TestUtils() {
     // final abstract...
   }
 
-
   public static long DEFAULT_TIME_MILLIS =
       TimeUnit.MINUTES.toMillis(TimeUnit.MILLISECONDS.toMinutes(System.currentTimeMillis()));
   public static double DEFAULT_VALUE = 1D;
 
   /**
-   * Creates a histogram accumulation key for given metric at minute granularity and DEFAULT_TIME_MILLIS
+   * Creates a histogram accumulation key for given metric at minute granularity and
+   * DEFAULT_TIME_MILLIS
    */
   public static HistogramKey makeKey(String metric) {
     return makeKey(metric, Granularity.MINUTE);
   }
 
   /**
-   * Creates a histogram accumulation key for a given metric and granularity around DEFAULT_TIME_MILLIS
+   * Creates a histogram accumulation key for a given metric and granularity around
+   * DEFAULT_TIME_MILLIS
    */
   public static HistogramKey makeKey(String metric, Granularity granularity) {
-    return Utils.makeKey(
-        ReportPoint.newBuilder().setMetric(metric).setTimestamp(DEFAULT_TIME_MILLIS).setValue(DEFAULT_VALUE).build(),
+    return HistogramUtils.makeKey(
+        ReportPoint.newBuilder()
+            .setMetric(metric)
+            .setAnnotations(ImmutableMap.of("tagk", "tagv"))
+            .setTimestamp(DEFAULT_TIME_MILLIS)
+            .setValue(DEFAULT_VALUE)
+            .build(),
         granularity);
   }
 
-
   static void testKeyPointMatch(HistogramKey key, ReportPoint point) {
     assertThat(key).isNotNull();
     assertThat(point).isNotNull();
@@ -50,6 +54,7 @@ static void testKeyPointMatch(HistogramKey key, ReportPoint point) {
     assertThat(key.getSource()).isEqualTo(point.getHost());
     assertThat(key.getTagsAsMap()).isEqualTo(point.getAnnotations());
     assertThat(key.getBinTimeMillis()).isEqualTo(point.getTimestamp());
-    assertThat(key.getBinDurationInMillis()).isEqualTo(((Histogram) point.getValue()).getDuration());
+    assertThat(key.getBinDurationInMillis())
+        .isEqualTo(((Histogram) point.getValue()).getDuration());
   }
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/histogram/accumulator/AccumulationCacheTest.java b/proxy/src/test/java/com/wavefront/agent/histogram/accumulator/AccumulationCacheTest.java
index 268a1d2a1..cd0695f5e 100644
--- a/proxy/src/test/java/com/wavefront/agent/histogram/accumulator/AccumulationCacheTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/histogram/accumulator/AccumulationCacheTest.java
@@ -1,24 +1,21 @@
 package com.wavefront.agent.histogram.accumulator;
 
+import static com.google.common.truth.Truth.assertThat;
+
 import com.github.benmanes.caffeine.cache.Cache;
 import com.tdunning.math.stats.AgentDigest;
+import com.wavefront.agent.histogram.HistogramKey;
+import com.wavefront.agent.histogram.HistogramUtils.HistogramKeyMarshaller;
 import com.wavefront.agent.histogram.TestUtils;
-import com.wavefront.agent.histogram.Utils;
-import com.wavefront.agent.histogram.Utils.HistogramKey;
-
-import net.openhft.chronicle.map.ChronicleMap;
-
-import org.junit.Before;
-import org.junit.Test;
-
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Logger;
-
-import static com.google.common.truth.Truth.assertThat;
+import net.openhft.chronicle.map.ChronicleMap;
+import org.junit.Before;
+import org.junit.Test;
 
 /**
  * Unit tests around {@link AccumulationCache}
@@ -26,15 +23,15 @@
  * @author Tim Schmidt (tim@wavefront.com).
  */
 public class AccumulationCacheTest {
-  private final static Logger logger = Logger.getLogger(AccumulationCacheTest.class.getCanonicalName());
-
-  private final static long CAPACITY = 2L;
-  private final static short COMPRESSION = 100;
+  private static final Logger logger =
+      Logger.getLogger(AccumulationCacheTest.class.getCanonicalName());
 
+  private static final long CAPACITY = 2L;
+  private static final short COMPRESSION = 100;
 
   private ConcurrentMap backingStore;
   private Cache cache;
-  private Runnable resolveTask;
+  private AccumulationCache ac;
 
   private HistogramKey keyA = TestUtils.makeKey("keyA");
   private HistogramKey keyB = TestUtils.makeKey("keyB");
@@ -48,8 +45,9 @@ public class AccumulationCacheTest {
   public void setup() {
     backingStore = new ConcurrentHashMap<>();
     tickerTime = new AtomicLong(0L);
-    AccumulationCache ac = new AccumulationCache(backingStore, CAPACITY, tickerTime::get);
-    resolveTask = ac.getResolveTask();
+    AgentDigestFactory agentDigestFactory =
+        new AgentDigestFactory(() -> COMPRESSION, 100, tickerTime::get);
+    ac = new AccumulationCache(backingStore, agentDigestFactory, CAPACITY, "", tickerTime::get);
     cache = ac.getCache();
 
     digestA = new AgentDigest(COMPRESSION, 100L);
@@ -67,7 +65,7 @@ public void testAddCache() throws ExecutionException {
   @Test
   public void testResolveOnNewKey() throws ExecutionException {
     cache.put(keyA, digestA);
-    resolveTask.run();
+    ac.flush();
     assertThat(cache.getIfPresent(keyA)).isNull();
     assertThat(backingStore.get(keyA)).isEqualTo(digestA);
   }
@@ -78,7 +76,7 @@ public void testResolveOnExistingKey() throws ExecutionException {
     digestB.add(15D, 1);
     backingStore.put(keyA, digestB);
     cache.put(keyA, digestA);
-    resolveTask.run();
+    ac.flush();
     assertThat(cache.getIfPresent(keyA)).isNull();
     assertThat(backingStore.get(keyA).size()).isEqualTo(2L);
   }
@@ -98,20 +96,28 @@ public void testEvictsOnCapacityExceeded() throws ExecutionException {
 
   @Test
   public void testChronicleMapOverflow() {
-    ConcurrentMap chronicleMap = ChronicleMap.of(HistogramKey.class, AgentDigest.class).
-        keyMarshaller(Utils.HistogramKeyMarshaller.get()).
-        valueMarshaller(AgentDigest.AgentDigestMarshaller.get()).
-        entries(10)
-        .averageKeySize(20)
-        .averageValueSize(20)
-        .maxBloatFactor(10)
-        .create();
+    ConcurrentMap chronicleMap =
+        ChronicleMap.of(HistogramKey.class, AgentDigest.class)
+            .keyMarshaller(HistogramKeyMarshaller.get())
+            .valueMarshaller(AgentDigest.AgentDigestMarshaller.get())
+            .entries(10)
+            .averageKeySize(20)
+            .averageValueSize(20)
+            .maxBloatFactor(10)
+            .create();
     AtomicBoolean hasFailed = new AtomicBoolean(false);
-    AccumulationCache ac = new AccumulationCache(chronicleMap, 10, tickerTime::get, () -> hasFailed.set(true));
+    AccumulationCache ac =
+        new AccumulationCache(
+            chronicleMap,
+            new AgentDigestFactory(() -> COMPRESSION, 100L, tickerTime::get),
+            10,
+            "",
+            tickerTime::get,
+            () -> hasFailed.set(true));
 
     for (int i = 0; i < 1000; i++) {
       ac.put(TestUtils.makeKey("key-" + i), digestA);
-      ac.getResolveTask().run();
+      ac.flush();
       if (hasFailed.get()) {
         logger.info("Chronicle map overflow detected when adding object #" + i);
         break;
diff --git a/proxy/src/test/java/com/wavefront/agent/histogram/accumulator/AccumulationTaskTest.java b/proxy/src/test/java/com/wavefront/agent/histogram/accumulator/AccumulationTaskTest.java
deleted file mode 100644
index 86e75b4af..000000000
--- a/proxy/src/test/java/com/wavefront/agent/histogram/accumulator/AccumulationTaskTest.java
+++ /dev/null
@@ -1,265 +0,0 @@
-package com.wavefront.agent.histogram.accumulator;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
-import com.squareup.tape.InMemoryObjectQueue;
-import com.squareup.tape.ObjectQueue;
-import com.tdunning.math.stats.AgentDigest;
-import com.wavefront.agent.PointHandler;
-import com.wavefront.data.Validation;
-import com.wavefront.agent.histogram.Utils;
-import com.wavefront.agent.histogram.Utils.HistogramKey;
-import com.wavefront.ingester.GraphiteDecoder;
-import com.wavefront.ingester.HistogramDecoder;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import wavefront.report.ReportPoint;
-
-import static com.google.common.truth.Truth.assertThat;
-import static com.wavefront.agent.histogram.TestUtils.DEFAULT_TIME_MILLIS;
-import static com.wavefront.agent.histogram.TestUtils.DEFAULT_VALUE;
-import static com.wavefront.agent.histogram.TestUtils.makeKey;
-import static com.wavefront.agent.histogram.Utils.Granularity.*;
-
-/**
- * Unit tests around {@link AccumulationTask}
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class AccumulationTaskTest {
-  private ObjectQueue> in;
-  private ConcurrentMap out;
-  private List badPointsOut;
-  private AccumulationTask eventSubject, histoSubject;
-  private AccumulationCache cache;
-  private final static long TTL = 30L;
-  private final static short COMPRESSION = 100;
-
-  private String lineA = "minKeyA " + DEFAULT_VALUE + " " + DEFAULT_TIME_MILLIS;
-  private String lineB = "minKeyB " + DEFAULT_VALUE + " " + DEFAULT_TIME_MILLIS;
-  private String lineC = "minKeyC " + DEFAULT_VALUE + " " + DEFAULT_TIME_MILLIS;
-
-  private String histoMinLineA = "!M " + DEFAULT_TIME_MILLIS + " #2 20 #5 70 #6 123 minKeyA";
-  private String histoMinLineB = "!M " + DEFAULT_TIME_MILLIS + " #1 10 #2 1000 #3 1000 minKeyB";
-  private String histoHourLineA = "!H " + DEFAULT_TIME_MILLIS + " #11 10 #9 1000 #17 1000 hourKeyA";
-  private String histoDayLineA = "!D " + DEFAULT_TIME_MILLIS + " #5 10 #5 1000 #5 1000 dayKeyA";
-
-
-  private HistogramKey minKeyA = makeKey("minKeyA");
-  private HistogramKey minKeyB = makeKey("minKeyB");
-  private HistogramKey minKeyC = makeKey("minKeyC");
-  private HistogramKey hourKeyA = makeKey("hourKeyA", HOUR);
-  private HistogramKey dayKeyA = makeKey("dayKeyA", DAY);
-
-  @Before
-  public void setUp() throws Exception {
-    AtomicInteger timeMillis = new AtomicInteger(0);
-    in = new InMemoryObjectQueue<>();
-    out = new ConcurrentHashMap<>();
-    cache = new AccumulationCache(out, 0, timeMillis::get);
-    badPointsOut = Lists.newArrayList();
-
-    PointHandler pointHandler = new PointHandler() {
-      @Override
-      public void reportPoint(ReportPoint point, String debugLine) {
-        throw new UnsupportedOperationException();
-      }
-
-      @Override
-      public void reportPoints(List points) {
-        throw new UnsupportedOperationException();
-      }
-
-      @Override
-      public void handleBlockedPoint(String pointLine) {
-        badPointsOut.add(pointLine);
-      }
-    };
-
-    eventSubject = new AccumulationTask(
-        in,
-        cache,
-        new GraphiteDecoder("unknown", ImmutableList.of()),
-        pointHandler,
-        Validation.Level.NUMERIC_ONLY,
-        TTL,
-        MINUTE,
-        COMPRESSION);
-
-    histoSubject = new AccumulationTask(
-        in,
-        cache,
-        new HistogramDecoder(),
-        pointHandler,
-        Validation.Level.NUMERIC_ONLY,
-        TTL,
-        MINUTE,
-        COMPRESSION);
-  }
-
-  @Test
-  public void testSingleLine() {
-    in.add(ImmutableList.of(lineA));
-
-    eventSubject.run();
-    cache.getResolveTask().run();
-
-    assertThat(badPointsOut).hasSize(0);
-    assertThat(out.get(minKeyA)).isNotNull();
-    assertThat(out.get(minKeyA).size()).isEqualTo(1);
-  }
-
-  @Test
-  public void testMultipleLines() {
-    in.add(ImmutableList.of(lineA, lineB, lineC));
-
-    eventSubject.run();
-    cache.getResolveTask().run();
-
-    assertThat(badPointsOut).hasSize(0);
-    assertThat(out.get(minKeyA)).isNotNull();
-    assertThat(out.get(minKeyA).size()).isEqualTo(1);
-    assertThat(out.get(minKeyB)).isNotNull();
-    assertThat(out.get(minKeyB).size()).isEqualTo(1);
-    assertThat(out.get(minKeyC)).isNotNull();
-    assertThat(out.get(minKeyC).size()).isEqualTo(1);
-  }
-
-  @Test
-  public void testMultipleTimes() {
-    in.add(ImmutableList.of(
-        "min0 1 " + (DEFAULT_TIME_MILLIS - 60000),
-        "min0 1 " + (DEFAULT_TIME_MILLIS - 60000),
-        "min1 1 " + DEFAULT_TIME_MILLIS));
-
-    eventSubject.run();
-    cache.getResolveTask().run();
-
-    HistogramKey min0 = Utils.makeKey(ReportPoint.newBuilder()
-        .setMetric("min0")
-        .setTimestamp(DEFAULT_TIME_MILLIS - 60000)
-        .setValue(1).build(), MINUTE);
-
-    HistogramKey min1 = Utils.makeKey(ReportPoint.newBuilder()
-        .setMetric("min1")
-        .setTimestamp(DEFAULT_TIME_MILLIS)
-        .setValue(1).build(), MINUTE);
-
-    assertThat(badPointsOut).hasSize(0);
-    assertThat(out.get(min0)).isNotNull();
-    assertThat(out.get(min0).size()).isEqualTo(2);
-    assertThat(out.get(min1)).isNotNull();
-    assertThat(out.get(min1).size()).isEqualTo(1);
-  }
-
-  @Test
-  public void testAccumulation() {
-    in.add(ImmutableList.of(lineA, lineA, lineA));
-
-    eventSubject.run();
-    cache.getResolveTask().run();
-
-    assertThat(out.get(minKeyA)).isNotNull();
-    assertThat(out.get(minKeyA).size()).isEqualTo(3);
-  }
-
-  @Test
-  public void testAccumulationNoTime() {
-    in.add(ImmutableList.of("noTimeKey 100"));
-
-    eventSubject.run();
-    cache.getResolveTask().run();
-
-    assertThat(out).hasSize(1);
-  }
-
-  @Test
-  public void testAccumulateWithBadLine() {
-    in.add(ImmutableList.of("This is not really a valid sample", lineA, lineA, lineA));
-
-    eventSubject.run();
-    cache.getResolveTask().run();
-
-    assertThat(badPointsOut).hasSize(1);
-    assertThat(badPointsOut.get(0)).contains("This is not really a valid sample");
-
-    assertThat(out.get(minKeyA)).isNotNull();
-    assertThat(out.get(minKeyA).size()).isEqualTo(3);
-  }
-
-  @Test
-  public void testHistogramLine() {
-    in.add(ImmutableList.of(histoMinLineA));
-
-    histoSubject.run();
-    cache.getResolveTask().run();
-
-    assertThat(badPointsOut).hasSize(0);
-    assertThat(out.get(minKeyA)).isNotNull();
-    assertThat(out.get(minKeyA).size()).isEqualTo(13);
-  }
-
-  @Test
-  public void testHistogramAccumulation() {
-    in.add(ImmutableList.of(histoMinLineA, histoMinLineA, histoMinLineA));
-
-    histoSubject.run();
-    cache.getResolveTask().run();
-
-    assertThat(badPointsOut).hasSize(0);
-    assertThat(out.get(minKeyA)).isNotNull();
-    assertThat(out.get(minKeyA).size()).isEqualTo(39);
-  }
-
-  @Test
-  public void testHistogramAccumulationMultipleGranularities() {
-    in.add(ImmutableList.of(histoMinLineA, histoHourLineA, histoDayLineA));
-
-    histoSubject.run();
-    cache.getResolveTask().run();
-
-    assertThat(badPointsOut).hasSize(0);
-    assertThat(out.get(minKeyA)).isNotNull();
-    assertThat(out.get(minKeyA).size()).isEqualTo(13);
-    assertThat(out.get(hourKeyA)).isNotNull();
-    assertThat(out.get(hourKeyA).size()).isEqualTo(37);
-    assertThat(out.get(dayKeyA)).isNotNull();
-    assertThat(out.get(dayKeyA).size()).isEqualTo(15);
-  }
-
-  @Test
-  public void testHistogramMultiBinAccumulation() {
-    in.add(ImmutableList.of(histoMinLineA, histoMinLineB, histoMinLineA, histoMinLineB));
-
-    histoSubject.run();
-    cache.getResolveTask().run();
-
-    assertThat(badPointsOut).hasSize(0);
-    assertThat(out.get(minKeyA)).isNotNull();
-    assertThat(out.get(minKeyA).size()).isEqualTo(26);
-    assertThat(out.get(minKeyB)).isNotNull();
-    assertThat(out.get(minKeyB).size()).isEqualTo(12);
-  }
-
-  @Test
-  public void testHistogramAccumulationWithBadLine() {
-    in.add(ImmutableList.of(histoMinLineA, "not really valid...", histoMinLineA));
-
-    histoSubject.run();
-    cache.getResolveTask().run();
-
-    assertThat(badPointsOut).hasSize(1);
-    assertThat(badPointsOut.get(0)).contains("not really valid...");
-
-    assertThat(out.get(minKeyA)).isNotNull();
-    assertThat(out.get(minKeyA).size()).isEqualTo(26);
-  }
-}
diff --git a/proxy/src/test/java/com/wavefront/agent/histogram/accumulator/LayeringTest.java b/proxy/src/test/java/com/wavefront/agent/histogram/accumulator/LayeringTest.java
deleted file mode 100644
index 1e376cdd8..000000000
--- a/proxy/src/test/java/com/wavefront/agent/histogram/accumulator/LayeringTest.java
+++ /dev/null
@@ -1,110 +0,0 @@
-package com.wavefront.agent.histogram.accumulator;
-
-
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-import com.tdunning.math.stats.AgentDigest;
-import com.wavefront.agent.histogram.TestUtils;
-import com.wavefront.agent.histogram.Utils.HistogramKey;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static com.google.common.truth.Truth.assertThat;
-
-/**
- * Unit tests around Layering
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class LayeringTest {
-  private final static short COMPRESSION = 100;
-
-  private ConcurrentMap backingStore;
-  private LoadingCache cache;
-  private Runnable writeBackTask;
-  private HistogramKey keyA = TestUtils.makeKey("keyA");
-  private AgentDigest digestA = new AgentDigest(COMPRESSION, 100L);
-  private AgentDigest digestB = new AgentDigest(COMPRESSION, 1000L);
-  private AgentDigest digestC = new AgentDigest(COMPRESSION, 10000L);
-  private AtomicLong tickerTime;
-
-  @Before
-  public void setup() {
-    backingStore = new ConcurrentHashMap<>();
-    Layering layering = new Layering(backingStore);
-    writeBackTask = layering.getWriteBackTask();
-    tickerTime = new AtomicLong(0L);
-    cache = Caffeine.newBuilder()
-        .expireAfterAccess(1, TimeUnit.SECONDS)
-        .writer(layering)
-        .ticker(() -> tickerTime.get()).build(layering);
-  }
-
-  @Test
-  public void testNoWriteThrough() throws ExecutionException {
-    cache.put(keyA, digestA);
-    assertThat(cache.get(keyA)).isEqualTo(digestA);
-    assertThat(backingStore.get(keyA)).isNull();
-  }
-
-  @Test
-  public void testWriteBack() throws ExecutionException {
-    cache.put(keyA, digestA);
-    writeBackTask.run();
-    assertThat(cache.get(keyA)).isEqualTo(digestA);
-    assertThat(backingStore.get(keyA)).isEqualTo(digestA);
-  }
-
-  @Test
-  public void testOnlyWriteBackLastChange() throws ExecutionException {
-    cache.put(keyA, digestA);
-    writeBackTask.run();
-    assertThat(cache.get(keyA)).isEqualTo(digestA);
-    assertThat(backingStore.get(keyA)).isEqualTo(digestA);
-
-    cache.put(keyA, digestB);
-    cache.put(keyA, digestC);
-
-    assertThat(backingStore.get(keyA)).isEqualTo(digestA);
-
-    writeBackTask.run();
-    assertThat(backingStore.get(keyA)).isEqualTo(digestC);
-  }
-
-  @Test
-  public void testBasicRead() throws ExecutionException {
-    backingStore.put(keyA, digestA);
-    assertThat(cache.get(keyA)).isEqualTo(digestA);
-  }
-
-  @Test
-  public void testReadDirtyKeyAfterEviction() throws ExecutionException {
-    // Write a change
-    cache.put(keyA, digestA);
-    assertThat(cache.get(keyA)).isEqualTo(digestA);
-    // Let the entry expire
-    tickerTime.addAndGet(TimeUnit.SECONDS.toNanos(2L));
-    cache.cleanUp();
-    assertThat(cache.estimatedSize()).isEqualTo(0L);
-    // Ensure we can still read the unpersisted change
-    assertThat(cache.get(keyA)).isEqualTo(digestA);
-  }
-
-  @Test
-  public void testExplicitDelete() throws ExecutionException {
-    backingStore.put(keyA, digestA);
-
-    cache.get(keyA);
-    cache.asMap().remove(keyA);
-    // Propagate
-    writeBackTask.run();
-    assertThat(backingStore.size()).isEqualTo(0L);
-  }
-}
diff --git a/proxy/src/test/java/com/wavefront/agent/histogram/tape/TapeDeckTest.java b/proxy/src/test/java/com/wavefront/agent/histogram/tape/TapeDeckTest.java
deleted file mode 100644
index ac2a52ded..000000000
--- a/proxy/src/test/java/com/wavefront/agent/histogram/tape/TapeDeckTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-package com.wavefront.agent.histogram.tape;
-
-
-import com.google.common.collect.ImmutableList;
-
-import com.squareup.tape.ObjectQueue;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static com.google.common.truth.Truth.assertThat;
-
-/**
- * Unit tests around {@link TapeDeck}
- *
- * @author Tim Schmidt (tim@wavefront.com).
- */
-public class TapeDeckTest {
-  private TapeDeck> deck;
-  private File file;
-
-  @Before
-  public void setup() {
-    try {
-      file = File.createTempFile("test-file", ".tmp");
-      file.deleteOnExit();
-    } catch (IOException e) {
-      e.printStackTrace();
-    }
-    deck = new TapeDeck<>(TapeStringListConverter.getDefaultInstance(), true);
-  }
-
-  @After
-  public void cleanup() {
-    file.delete();
-  }
-
-  private void testTape(ObjectQueue> tape) {
-    final AtomicInteger addCalls = new AtomicInteger(0);
-    final AtomicInteger removeCalls = new AtomicInteger(0);
-
-    ObjectQueue.Listener> listener = new ObjectQueue.Listener>() {
-      @Override
-      public void onAdd(ObjectQueue> objectQueue, List strings) {
-        addCalls.incrementAndGet();
-      }
-
-      @Override
-      public void onRemove(ObjectQueue> objectQueue) {
-        removeCalls.incrementAndGet();
-      }
-    };
-
-    tape.setListener(listener);
-    assertThat(tape).isNotNull();
-    tape.add(ImmutableList.of("test"));
-    assertThat(tape.size()).isEqualTo(1);
-    assertThat(tape.peek()).containsExactly("test");
-    assertThat(addCalls.get()).isEqualTo(1);
-    assertThat(removeCalls.get()).isEqualTo(0);
-    tape.remove();
-    assertThat(addCalls.get()).isEqualTo(1);
-    assertThat(removeCalls.get()).isEqualTo(1);
-  }
-
-  @Test
-  public void testFileDoNotPersist() throws IOException {
-    file.delete();
-    deck = new TapeDeck<>(TapeStringListConverter.getDefaultInstance(), false);
-
-    ObjectQueue> q = deck.getTape(file);
-    testTape(q);
-  }
-
-  @Test
-  public void testFileDoesNotExist() throws IOException {
-    file.delete();
-    ObjectQueue> q = deck.getTape(file);
-    testTape(q);
-  }
-}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/JaegerThriftCollectorHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/JaegerThriftCollectorHandlerTest.java
deleted file mode 100644
index bbdc1c3bf..000000000
--- a/proxy/src/test/java/com/wavefront/agent/listeners/JaegerThriftCollectorHandlerTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-package com.wavefront.agent.listeners;
-
-import com.google.common.collect.ImmutableList;
-
-import com.uber.tchannel.messages.ThriftRequest;
-import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
-import com.wavefront.agent.handlers.ReportableEntityHandler;
-
-import org.junit.Test;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import io.jaegertracing.thriftjava.Batch;
-import io.jaegertracing.thriftjava.Collector;
-import io.jaegertracing.thriftjava.Process;
-import io.jaegertracing.thriftjava.Tag;
-import io.jaegertracing.thriftjava.TagType;
-import wavefront.report.Annotation;
-import wavefront.report.Span;
-
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
-import static org.easymock.EasyMock.verify;
-
-public class JaegerThriftCollectorHandlerTest {
-  private ReportableEntityHandler mockTraceHandler =
-      MockReportableEntityHandlerFactory.getMockTraceHandler();
-  private long startTime = System.currentTimeMillis();
-
-  @Test
-  public void testJaegerThriftCollector() throws Exception {
-    reset(mockTraceHandler);
-    mockTraceHandler.report(Span.newBuilder().setCustomer("dummy").setStartMillis(startTime)
-        .setDuration(1234)
-        .setName("HTTP GET")
-        .setSource("10.0.0.1")
-        .setSpanId("00000000-0000-0000-0000-00000012d687")
-        .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
-        // Note: Order of annotations list matters for this unit test.
-        .setAnnotations(ImmutableList.of(new Annotation("service", "frontend"),
-            new Annotation("application", "Jaeger")))
-        .build());
-    expectLastCall();
-
-    mockTraceHandler.report(Span.newBuilder().setCustomer("dummy").setStartMillis(startTime)
-        .setDuration(2345)
-        .setName("HTTP GET /")
-        .setSource("10.0.0.1")
-        .setSpanId("00000000-0000-0000-0000-00000023cace")
-        .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
-        // Note: Order of annotations list matters for this unit test.
-        .setAnnotations(ImmutableList.of(
-            new Annotation("service", "frontend"),
-            new Annotation("parent", "00000000-0000-0000-0000-00000012d687"),
-            new Annotation("application", "Jaeger")))
-        .build());
-    expectLastCall();
-
-    mockTraceHandler.report(Span.newBuilder().setCustomer("dummy").setStartMillis(startTime)
-        .setDuration(3456)
-        .setName("HTTP GET /")
-        .setSource("10.0.0.1")
-        .setSpanId("00000000-0000-0000-9a12-b85901d53397")
-        .setTraceId("00000000-0000-0000-fea4-87ee36e58cab")
-        // Note: Order of annotations list matters for this unit test.
-        .setAnnotations(ImmutableList.of(
-            new Annotation("service", "frontend"),
-            new Annotation("parent", "00000000-0000-0000-fea4-87ee36e58cab"),
-            new Annotation("application", "Jaeger")))
-        .build());
-    expectLastCall();
-
-
-    replay(mockTraceHandler);
-
-    JaegerThriftCollectorHandler handler = new JaegerThriftCollectorHandler("9876", mockTraceHandler,
-        new AtomicBoolean(false));
-
-    Tag tag1 = new Tag("ip", TagType.STRING);
-    tag1.setVStr("10.0.0.1");
-
-    io.jaegertracing.thriftjava.Span span1 = new io.jaegertracing.thriftjava.Span(1234567890123L, 1234567890L,
-        1234567L, 0L, "HTTP GET", 1, startTime * 1000, 1234 * 1000);
-
-    io.jaegertracing.thriftjava.Span span2 = new io.jaegertracing.thriftjava.Span(1234567890123L, 1234567890L,
-        2345678L, 1234567L, "HTTP GET /", 1, startTime * 1000, 2345 * 1000);
-
-    // check negative span IDs too
-    io.jaegertracing.thriftjava.Span span3 = new io.jaegertracing.thriftjava.Span(-97803834702328661L, 0L,
-        -7344605349865507945L, -97803834702328661L, "HTTP GET /", 1, startTime * 1000, 3456 * 1000);
-
-    Batch testBatch = new Batch();
-    testBatch.process = new Process();
-    testBatch.process.serviceName = "frontend";
-    testBatch.process.setTags(ImmutableList.of(tag1));
-
-    testBatch.setSpans(ImmutableList.of(span1, span2, span3));
-
-    Collector.submitBatches_args batches = new Collector.submitBatches_args();
-    batches.addToBatches(testBatch);
-    ThriftRequest request = new ThriftRequest.Builder(
-        "jaeger-collector", "Collector::submitBatches").setBody(batches).build();
-    handler.handleImpl(request);
-
-    verify(mockTraceHandler);
-  }
-}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/ZipkinPortUnificationHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/ZipkinPortUnificationHandlerTest.java
deleted file mode 100644
index ff5067bc5..000000000
--- a/proxy/src/test/java/com/wavefront/agent/listeners/ZipkinPortUnificationHandlerTest.java
+++ /dev/null
@@ -1,143 +0,0 @@
-package com.wavefront.agent.listeners;
-
-import com.google.common.collect.ImmutableList;
-
-import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
-import com.wavefront.agent.handlers.ReportableEntityHandler;
-
-import org.easymock.EasyMock;
-import org.junit.Test;
-
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.Unpooled;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.http.DefaultFullHttpRequest;
-import io.netty.handler.codec.http.FullHttpRequest;
-import io.netty.handler.codec.http.FullHttpResponse;
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpVersion;
-import wavefront.report.Annotation;
-import wavefront.report.Span;
-import zipkin2.Endpoint;
-import zipkin2.codec.SpanBytesEncoder;
-
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
-import static org.easymock.EasyMock.verify;
-
-public class ZipkinPortUnificationHandlerTest {
-  private ReportableEntityHandler mockTraceHandler =
-      MockReportableEntityHandlerFactory.getMockTraceHandler();
-  private long startTime = System.currentTimeMillis();
-
-  @Test
-  public void testZipkinHandler() throws Exception {
-    ZipkinPortUnificationHandler handler = new ZipkinPortUnificationHandler("9411",
-        mockTraceHandler,
-        new AtomicBoolean(false));
-
-    Endpoint localEndpoint1 = Endpoint.newBuilder().serviceName("frontend").ip("10.0.0.1").build();
-    zipkin2.Span spanServer1 = zipkin2.Span.newBuilder().
-        traceId("2822889fe47043bd").
-        id("2822889fe47043bd").
-        kind(zipkin2.Span.Kind.SERVER).
-        name("getservice").
-        timestamp(startTime * 1000).
-        duration(1234 * 1000).
-        localEndpoint(localEndpoint1).
-        putTag("http.method", "GET").
-        putTag("http.url", "none+h1c://localhost:8881/").
-        putTag("http.status_code", "200").
-        build();
-
-    Endpoint localEndpoint2 = Endpoint.newBuilder().serviceName("backend").ip("10.0.0.1").build();
-    zipkin2.Span spanServer2 = zipkin2.Span.newBuilder().
-        traceId("2822889fe47043bd").
-        id("d6ab73f8a3930ae8").
-        parentId("2822889fe47043bd").
-        kind(zipkin2.Span.Kind.SERVER).
-        name("getbackendservice").
-        timestamp(startTime * 1000).
-        duration(2234 * 1000).
-        localEndpoint(localEndpoint2).
-        putTag("http.method", "GET").
-        putTag("http.url", "none+h2c://localhost:9000/api").
-        putTag("http.status_code", "200").
-        build();
-
-    List zipkinSpanList = ImmutableList.of(spanServer1, spanServer2);
-
-    // Validate all codecs i.e. JSON_V1, JSON_V2, THRIFT and PROTO3.
-    for (SpanBytesEncoder encoder : SpanBytesEncoder.values()) {
-      ByteBuf content = Unpooled.copiedBuffer(encoder.encodeList(zipkinSpanList));
-      // take care of mocks.
-      doMockLifecycle(mockTraceHandler);
-      ChannelHandlerContext mockCtx = createNiceMock(ChannelHandlerContext.class);
-      doMockLifecycle(mockCtx);
-      FullHttpRequest httpRequest = new DefaultFullHttpRequest(
-          HttpVersion.HTTP_1_1,
-          HttpMethod.POST,
-          "http://localhost:9411/api/v1/spans",
-          content,
-          true
-      );
-      handler.handleHttpMessage(mockCtx, httpRequest);
-      verify(mockTraceHandler);
-    }
-  }
-
-  private void doMockLifecycle(ChannelHandlerContext mockCtx) {
-    reset(mockCtx);
-    EasyMock.expect(mockCtx.write(EasyMock.isA(FullHttpResponse.class))).andReturn(null);
-    EasyMock.replay(mockCtx);
-  }
-
-  private void doMockLifecycle(ReportableEntityHandler mockTraceHandler) {
-    // Reset mock
-    reset(mockTraceHandler);
-
-    // Set Expectation
-    mockTraceHandler.report(Span.newBuilder().setCustomer("dummy").setStartMillis(startTime).
-        setDuration(1234).
-        setName("getservice").
-        setSource("10.0.0.1").
-        setSpanId("00000000-0000-0000-2822-889fe47043bd").
-        setTraceId("00000000-0000-0000-2822-889fe47043bd").
-        // Note: Order of annotations list matters for this unit test.
-        setAnnotations(ImmutableList.of(
-            new Annotation("span.kind", "server"),
-            new Annotation("service", "frontend"),
-            new Annotation("application", "Zipkin"),
-            new Annotation("http.method", "GET"),
-            new Annotation("http.status_code", "200"),
-            new Annotation("http.url", "none+h1c://localhost:8881/"))).
-        build());
-    expectLastCall();
-
-    mockTraceHandler.report(Span.newBuilder().setCustomer("dummy").setStartMillis(startTime).
-        setDuration(2234).
-        setName("getbackendservice").
-        setSource("10.0.0.1").
-        setSpanId("00000000-0000-0000-d6ab-73f8a3930ae8").
-        setTraceId("00000000-0000-0000-2822-889fe47043bd").
-        // Note: Order of annotations list matters for this unit test.
-        setAnnotations(ImmutableList.of(
-            new Annotation("parent", "00000000-0000-0000-2822-889fe47043bd"),
-            new Annotation("span.kind", "server"),
-            new Annotation("service", "backend"),
-            new Annotation("application", "Zipkin"),
-            new Annotation("http.method", "GET"),
-            new Annotation("http.status_code", "200"),
-            new Annotation("http.url", "none+h2c://localhost:9000/api"))).
-        build());
-    expectLastCall();
-
-    // Replay
-    replay(mockTraceHandler);
-  }
-}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpGrpcMetricsHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpGrpcMetricsHandlerTest.java
new file mode 100644
index 000000000..e8c44bd4c
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpGrpcMetricsHandlerTest.java
@@ -0,0 +1,941 @@
+package com.wavefront.agent.listeners.otlp;
+
+import static com.wavefront.agent.listeners.otlp.OtlpMetricsUtils.MILLIS_IN_DAY;
+import static com.wavefront.agent.listeners.otlp.OtlpMetricsUtils.MILLIS_IN_HOUR;
+import static com.wavefront.agent.listeners.otlp.OtlpMetricsUtils.MILLIS_IN_MINUTE;
+import static com.wavefront.agent.listeners.otlp.OtlpTestHelpers.DEFAULT_SOURCE;
+import static org.junit.Assert.assertFalse;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import io.grpc.stub.StreamObserver;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse;
+import io.opentelemetry.proto.metrics.v1.AggregationTemporality;
+import io.opentelemetry.proto.metrics.v1.ExponentialHistogram;
+import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
+import io.opentelemetry.proto.metrics.v1.Gauge;
+import io.opentelemetry.proto.metrics.v1.Histogram;
+import io.opentelemetry.proto.metrics.v1.HistogramDataPoint;
+import io.opentelemetry.proto.metrics.v1.Metric;
+import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
+import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
+import io.opentelemetry.proto.metrics.v1.ScopeMetrics;
+import io.opentelemetry.proto.metrics.v1.Sum;
+import io.opentelemetry.proto.metrics.v1.Summary;
+import io.opentelemetry.proto.metrics.v1.SummaryDataPoint;
+import io.opentelemetry.proto.resource.v1.Resource;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+import wavefront.report.HistogramType;
+import wavefront.report.ReportPoint;
+
+public class OtlpGrpcMetricsHandlerTest {
+
+  public static final StreamObserver emptyStreamObserver =
+      new StreamObserver() {
+
+        @Override
+        public void onNext(ExportMetricsServiceResponse exportMetricsServiceResponse) {}
+
+        @Override
+        public void onError(Throwable throwable) {}
+
+        @Override
+        public void onCompleted() {}
+      };
+
+  private final ReportableEntityHandler mockReportPointHandler =
+      MockReportableEntityHandlerFactory.getMockReportPointHandler();
+  private final ReportableEntityHandler mockHistogramHandler =
+      MockReportableEntityHandlerFactory.getMockReportPointHandler();
+  private OtlpGrpcMetricsHandler subject;
+  private final Supplier preprocessorSupplier =
+      ReportableEntityPreprocessor::new;
+
+  @Before
+  public void setup() {
+    subject =
+        new OtlpGrpcMetricsHandler(
+            mockReportPointHandler,
+            mockHistogramHandler,
+            preprocessorSupplier,
+            DEFAULT_SOURCE,
+            false,
+            true);
+  }
+
+  @Test
+  public void simpleGauge() {
+    long epochTime = 1515151515L;
+    EasyMock.reset(mockReportPointHandler);
+    Gauge otelGauge =
+        Gauge.newBuilder()
+            .addDataPoints(
+                NumberDataPoint.newBuilder()
+                    .setAsDouble(12.3)
+                    .setTimeUnixNano(TimeUnit.SECONDS.toNanos(epochTime))
+                    .build())
+            .build();
+    Metric otelMetric = Metric.newBuilder().setGauge(otelGauge).setName("test-gauge").build();
+    wavefront.report.ReportPoint wfMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-gauge")
+            .setTimestamp(TimeUnit.SECONDS.toMillis(epochTime))
+            .setValue(12.3)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void monotonicCumulativeSum() {
+    long epochTime = 1515151515L;
+    EasyMock.reset(mockReportPointHandler);
+    Sum otelSum =
+        Sum.newBuilder()
+            .setIsMonotonic(true)
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addDataPoints(
+                NumberDataPoint.newBuilder()
+                    .setAsDouble(12.3)
+                    .setTimeUnixNano(TimeUnit.SECONDS.toNanos(epochTime))
+                    .build())
+            .build();
+    Metric otelMetric = Metric.newBuilder().setSum(otelSum).setName("test-sum").build();
+    wavefront.report.ReportPoint wfMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-sum")
+            .setTimestamp(TimeUnit.SECONDS.toMillis(epochTime))
+            .setValue(12.3)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void nonmonotonicCumulativeSum() {
+    long epochTime = 1515151515L;
+    EasyMock.reset(mockReportPointHandler);
+    Sum otelSum =
+        Sum.newBuilder()
+            .setIsMonotonic(false)
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addDataPoints(
+                NumberDataPoint.newBuilder()
+                    .setAsDouble(12.3)
+                    .setTimeUnixNano(TimeUnit.SECONDS.toNanos(epochTime))
+                    .build())
+            .build();
+    Metric otelMetric = Metric.newBuilder().setSum(otelSum).setName("test-sum").build();
+    wavefront.report.ReportPoint wfMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-sum")
+            .setTimestamp(TimeUnit.SECONDS.toMillis(epochTime))
+            .setValue(12.3)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void simpleSummary() {
+    long epochTime = 1515151515L;
+    EasyMock.reset(mockReportPointHandler);
+    SummaryDataPoint point =
+        SummaryDataPoint.newBuilder()
+            .setSum(12.3)
+            .setCount(21)
+            .addQuantileValues(
+                SummaryDataPoint.ValueAtQuantile.newBuilder()
+                    .setQuantile(.5)
+                    .setValue(242.3)
+                    .build())
+            .setTimeUnixNano(TimeUnit.SECONDS.toNanos(epochTime))
+            .build();
+    Summary otelSummary = Summary.newBuilder().addDataPoints(point).build();
+    Metric otelMetric = Metric.newBuilder().setSummary(otelSummary).setName("test-summary").build();
+    mockReportPointHandler.report(
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-summary_sum")
+            .setTimestamp(TimeUnit.SECONDS.toMillis(epochTime))
+            .setValue(12.3)
+            .setHost(DEFAULT_SOURCE)
+            .build());
+    EasyMock.expectLastCall();
+    mockReportPointHandler.report(
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-summary_count")
+            .setTimestamp(TimeUnit.SECONDS.toMillis(epochTime))
+            .setValue(21)
+            .setHost(DEFAULT_SOURCE)
+            .build());
+    EasyMock.expectLastCall();
+    mockReportPointHandler.report(
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-summary")
+            .setTimestamp(TimeUnit.SECONDS.toMillis(epochTime))
+            .setValue(242.3)
+            .setAnnotations(ImmutableMap.of("quantile", "0.5"))
+            .setHost(DEFAULT_SOURCE)
+            .build());
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void monotonicDeltaSum() {
+    long epochTime = 1515151515L;
+    EasyMock.reset(mockReportPointHandler);
+    Sum otelSum =
+        Sum.newBuilder()
+            .setIsMonotonic(true)
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA)
+            .addDataPoints(
+                NumberDataPoint.newBuilder()
+                    .setAsDouble(12.3)
+                    .setTimeUnixNano(TimeUnit.SECONDS.toNanos(epochTime))
+                    .build())
+            .build();
+    Metric otelMetric = Metric.newBuilder().setSum(otelSum).setName("test-sum").build();
+    wavefront.report.ReportPoint wfMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("∆test-sum")
+            .setTimestamp(TimeUnit.SECONDS.toMillis(epochTime))
+            .setValue(12.3)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void nonmonotonicDeltaSum() {
+    long epochTime = 1515151515L;
+    EasyMock.reset(mockReportPointHandler);
+    Sum otelSum =
+        Sum.newBuilder()
+            .setIsMonotonic(false)
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA)
+            .addDataPoints(
+                NumberDataPoint.newBuilder()
+                    .setAsDouble(12.3)
+                    .setTimeUnixNano(TimeUnit.SECONDS.toNanos(epochTime))
+                    .build())
+            .build();
+    Metric otelMetric = Metric.newBuilder().setSum(otelSum).setName("test-sum").build();
+    wavefront.report.ReportPoint wfMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("∆test-sum")
+            .setTimestamp(TimeUnit.SECONDS.toMillis(epochTime))
+            .setValue(12.3)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void setsSourceFromResourceAttributesNotPointAttributes() {
+    EasyMock.reset(mockReportPointHandler);
+
+    Map annotations = new HashMap<>();
+    annotations.put("_source", "at-point");
+
+    ReportPoint wfMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setAnnotations(annotations)
+            .setHost("at-resrc")
+            .build();
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    NumberDataPoint otelPoint =
+        NumberDataPoint.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute("source", "at-point"))
+            .build();
+    Metric otelMetric = OtlpTestHelpers.otlpGaugeGenerator(otelPoint).build();
+
+    Resource otelResource =
+        Resource.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute("source", "at-resrc"))
+            .build();
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .setResource(otelResource)
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void cumulativeHistogram() {
+    long epochTime = 1515151515L;
+    EasyMock.reset(mockReportPointHandler);
+
+    HistogramDataPoint point =
+        HistogramDataPoint.newBuilder()
+            .addAllExplicitBounds(ImmutableList.of(1.0, 2.0))
+            .addAllBucketCounts(ImmutableList.of(1L, 2L, 3L))
+            .setTimeUnixNano(epochTime)
+            .build();
+
+    Histogram otelHistogram =
+        Histogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addAllDataPoints(Collections.singletonList(point))
+            .build();
+
+    Metric otelMetric =
+        Metric.newBuilder()
+            .setHistogram(otelHistogram)
+            .setName("test-cumulative-histogram")
+            .build();
+
+    wavefront.report.ReportPoint wfMetric1 =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-cumulative-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(1)
+            .setHost(DEFAULT_SOURCE)
+            .setAnnotations(Collections.singletonMap("le", "1.0"))
+            .build();
+
+    wavefront.report.ReportPoint wfMetric2 =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-cumulative-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(3)
+            .setHost(DEFAULT_SOURCE)
+            .setAnnotations(Collections.singletonMap("le", "2.0"))
+            .build();
+
+    wavefront.report.ReportPoint wfMetric3 =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-cumulative-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(6)
+            .setHost(DEFAULT_SOURCE)
+            .setAnnotations(Collections.singletonMap("le", "+Inf"))
+            .build();
+
+    mockReportPointHandler.report(wfMetric1);
+    EasyMock.expectLastCall().once();
+
+    mockReportPointHandler.report(wfMetric2);
+    EasyMock.expectLastCall().once();
+
+    mockReportPointHandler.report(wfMetric3);
+    EasyMock.expectLastCall().once();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void deltaHistogram() {
+    long epochTime = 1515151515L;
+    EasyMock.reset(mockHistogramHandler);
+
+    HistogramDataPoint point =
+        HistogramDataPoint.newBuilder()
+            .addAllExplicitBounds(ImmutableList.of(1.0, 2.0, 3.0))
+            .addAllBucketCounts(ImmutableList.of(1L, 2L, 3L, 4L))
+            .setTimeUnixNano(epochTime)
+            .build();
+
+    Histogram otelHistogram =
+        Histogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA)
+            .addAllDataPoints(Collections.singletonList(point))
+            .build();
+
+    Metric otelMetric =
+        Metric.newBuilder().setHistogram(otelHistogram).setName("test-delta-histogram").build();
+
+    List bins = new ArrayList<>(Arrays.asList(1.0, 1.5, 2.5, 3.0));
+    List counts = new ArrayList<>(Arrays.asList(1, 2, 3, 4));
+
+    wavefront.report.Histogram minHistogram =
+        wavefront.report.Histogram.newBuilder()
+            .setType(HistogramType.TDIGEST)
+            .setBins(bins)
+            .setCounts(counts)
+            .setDuration(MILLIS_IN_MINUTE)
+            .build();
+
+    wavefront.report.ReportPoint minWFMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-delta-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(minHistogram)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+
+    wavefront.report.Histogram hourHistogram =
+        wavefront.report.Histogram.newBuilder()
+            .setType(HistogramType.TDIGEST)
+            .setBins(bins)
+            .setCounts(counts)
+            .setDuration(MILLIS_IN_HOUR)
+            .build();
+
+    wavefront.report.ReportPoint hourWFMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-delta-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(hourHistogram)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+
+    wavefront.report.Histogram dayHistogram =
+        wavefront.report.Histogram.newBuilder()
+            .setType(HistogramType.TDIGEST)
+            .setBins(bins)
+            .setCounts(counts)
+            .setDuration(MILLIS_IN_DAY)
+            .build();
+
+    wavefront.report.ReportPoint dayWFMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-delta-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(dayHistogram)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+
+    mockHistogramHandler.report(minWFMetric);
+    EasyMock.expectLastCall().once();
+
+    mockHistogramHandler.report(hourWFMetric);
+    EasyMock.expectLastCall().once();
+
+    mockHistogramHandler.report(dayWFMetric);
+    EasyMock.expectLastCall().once();
+
+    EasyMock.replay(mockHistogramHandler);
+
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockHistogramHandler);
+  }
+
+  @Test
+  public void resourceAttrsCanBeExcluded() {
+    String resourceAttrKey = "testKey";
+
+    EasyMock.reset(mockReportPointHandler);
+
+    ReportPoint wfMetric = OtlpTestHelpers.wfReportPointGenerator().build();
+    assertFalse(wfMetric.getAnnotations().containsKey(resourceAttrKey));
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    NumberDataPoint otelPoint = NumberDataPoint.newBuilder().build();
+    Metric otelMetric = OtlpTestHelpers.otlpGaugeGenerator(otelPoint).build();
+    Resource resource =
+        Resource.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute(resourceAttrKey, "testValue"))
+            .build();
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .setResource(resource)
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void resourceAttrsCanBeIncluded() {
+    boolean includeResourceAttrsForMetrics = true;
+    subject =
+        new OtlpGrpcMetricsHandler(
+            mockReportPointHandler,
+            mockHistogramHandler,
+            preprocessorSupplier,
+            DEFAULT_SOURCE,
+            includeResourceAttrsForMetrics,
+            true);
+    String resourceAttrKey = "testKey";
+
+    EasyMock.reset(mockReportPointHandler);
+
+    Map annotations = ImmutableMap.of(resourceAttrKey, "testValue");
+    ReportPoint wfMetric =
+        OtlpTestHelpers.wfReportPointGenerator().setAnnotations(annotations).build();
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    NumberDataPoint otelPoint = NumberDataPoint.newBuilder().build();
+    Metric otelMetric = OtlpTestHelpers.otlpGaugeGenerator(otelPoint).build();
+    Resource resource =
+        Resource.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute(resourceAttrKey, "testValue"))
+            .build();
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .setResource(resource)
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void resourceAttrsWithServiceNameCanBeIncluded() {
+    boolean includeResourceAttrsForMetrics = true;
+    subject =
+        new OtlpGrpcMetricsHandler(
+            mockReportPointHandler,
+            mockHistogramHandler,
+            preprocessorSupplier,
+            DEFAULT_SOURCE,
+            includeResourceAttrsForMetrics,
+            true);
+
+    EasyMock.reset(mockReportPointHandler);
+
+    Map annotations = ImmutableMap.of("service", "testValue");
+    ReportPoint wfMetric =
+        OtlpTestHelpers.wfReportPointGenerator().setAnnotations(annotations).build();
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    NumberDataPoint otelPoint = NumberDataPoint.newBuilder().build();
+    Metric otelMetric = OtlpTestHelpers.otlpGaugeGenerator(otelPoint).build();
+    Resource resource =
+        Resource.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute("service.name", "testValue"))
+            .build();
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .setResource(resource)
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void exponentialDeltaHistogram() {
+    long epochTime = 1515151515L;
+    EasyMock.reset(mockHistogramHandler);
+
+    ExponentialHistogramDataPoint point =
+        ExponentialHistogramDataPoint.newBuilder()
+            .setScale(0)
+            .setTimeUnixNano(epochTime)
+            .setPositive(
+                ExponentialHistogramDataPoint.Buckets.newBuilder()
+                    .setOffset(2)
+                    .addBucketCounts(1)
+                    .build())
+            .setZeroCount(2)
+            .setNegative(
+                ExponentialHistogramDataPoint.Buckets.newBuilder()
+                    .setOffset(2)
+                    .addBucketCounts(3)
+                    .build())
+            .build();
+
+    ExponentialHistogram otelHistogram =
+        ExponentialHistogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA)
+            .addDataPoints(point)
+            .build();
+
+    Metric otelMetric =
+        Metric.newBuilder()
+            .setExponentialHistogram(otelHistogram)
+            .setName("test-exp-delta-histogram")
+            .build();
+
+    List bins = new ArrayList<>(Arrays.asList(-6.0, 0.0, 6.0));
+    List counts = new ArrayList<>(Arrays.asList(3, 2, 1));
+
+    wavefront.report.Histogram minHistogram =
+        wavefront.report.Histogram.newBuilder()
+            .setType(HistogramType.TDIGEST)
+            .setBins(bins)
+            .setCounts(counts)
+            .setDuration(MILLIS_IN_MINUTE)
+            .build();
+
+    wavefront.report.ReportPoint minWFMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-exp-delta-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(minHistogram)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+
+    wavefront.report.Histogram hourHistogram =
+        wavefront.report.Histogram.newBuilder()
+            .setType(HistogramType.TDIGEST)
+            .setBins(bins)
+            .setCounts(counts)
+            .setDuration(MILLIS_IN_HOUR)
+            .build();
+
+    wavefront.report.ReportPoint hourWFMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-exp-delta-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(hourHistogram)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+
+    wavefront.report.Histogram dayHistogram =
+        wavefront.report.Histogram.newBuilder()
+            .setType(HistogramType.TDIGEST)
+            .setBins(bins)
+            .setCounts(counts)
+            .setDuration(MILLIS_IN_DAY)
+            .build();
+
+    wavefront.report.ReportPoint dayWFMetric =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-exp-delta-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(dayHistogram)
+            .setHost(DEFAULT_SOURCE)
+            .build();
+
+    mockHistogramHandler.report(minWFMetric);
+    EasyMock.expectLastCall().once();
+
+    mockHistogramHandler.report(hourWFMetric);
+    EasyMock.expectLastCall().once();
+
+    mockHistogramHandler.report(dayWFMetric);
+    EasyMock.expectLastCall().once();
+
+    EasyMock.replay(mockHistogramHandler);
+
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockHistogramHandler);
+  }
+
+  @Test
+  public void exponentialCumulativeHistogram() {
+    long epochTime = 1515151515L;
+    EasyMock.reset(mockReportPointHandler);
+
+    ExponentialHistogramDataPoint point =
+        ExponentialHistogramDataPoint.newBuilder()
+            .setScale(0)
+            .setTimeUnixNano(epochTime)
+            .setPositive(
+                ExponentialHistogramDataPoint.Buckets.newBuilder()
+                    .setOffset(2)
+                    .addBucketCounts(1)
+                    .build())
+            .setZeroCount(2)
+            .setNegative(
+                ExponentialHistogramDataPoint.Buckets.newBuilder()
+                    .setOffset(2)
+                    .addBucketCounts(3)
+                    .build())
+            .build();
+
+    ExponentialHistogram otelHistogram =
+        ExponentialHistogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addDataPoints(point)
+            .build();
+
+    Metric otelMetric =
+        Metric.newBuilder()
+            .setExponentialHistogram(otelHistogram)
+            .setName("test-exp-cumulative-histogram")
+            .build();
+
+    wavefront.report.ReportPoint wfMetric1 =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-exp-cumulative-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(3)
+            .setHost(DEFAULT_SOURCE)
+            .setAnnotations(Collections.singletonMap("le", "-4.0"))
+            .build();
+
+    wavefront.report.ReportPoint wfMetric2 =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-exp-cumulative-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(5)
+            .setHost(DEFAULT_SOURCE)
+            .setAnnotations(Collections.singletonMap("le", "4.0"))
+            .build();
+
+    wavefront.report.ReportPoint wfMetric3 =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-exp-cumulative-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(6)
+            .setHost(DEFAULT_SOURCE)
+            .setAnnotations(Collections.singletonMap("le", "8.0"))
+            .build();
+
+    wavefront.report.ReportPoint wfMetric4 =
+        OtlpTestHelpers.wfReportPointGenerator()
+            .setMetric("test-exp-cumulative-histogram")
+            .setTimestamp(TimeUnit.NANOSECONDS.toMillis(epochTime))
+            .setValue(6)
+            .setHost(DEFAULT_SOURCE)
+            .setAnnotations(Collections.singletonMap("le", "+Inf"))
+            .build();
+
+    mockReportPointHandler.report(wfMetric1);
+    EasyMock.expectLastCall().once();
+
+    mockReportPointHandler.report(wfMetric2);
+    EasyMock.expectLastCall().once();
+
+    mockReportPointHandler.report(wfMetric3);
+    EasyMock.expectLastCall().once();
+
+    mockReportPointHandler.report(wfMetric4);
+    EasyMock.expectLastCall().once();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void appRelatedResAttrsShouldBeIncluded() {
+    String applicationKey = "application";
+    String serviceKey = "service.name";
+    String shardKey = "shard";
+    String clusterKey = "cluster";
+
+    EasyMock.reset(mockReportPointHandler);
+
+    Map annotations =
+        ImmutableMap.builder()
+            .put(applicationKey, "some-app-name")
+            .put("service", "some-service-name")
+            .put(shardKey, "some-shard-name")
+            .put(clusterKey, "some-cluster-name")
+            .build();
+
+    ReportPoint wfMetric =
+        OtlpTestHelpers.wfReportPointGenerator().setAnnotations(annotations).build();
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    NumberDataPoint otelPoint = NumberDataPoint.newBuilder().build();
+    Metric otelMetric = OtlpTestHelpers.otlpGaugeGenerator(otelPoint).build();
+    Resource resource =
+        Resource.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute(applicationKey, "some-app-name"))
+            .addAttributes(OtlpTestHelpers.attribute(serviceKey, "some-service-name"))
+            .addAttributes(OtlpTestHelpers.attribute(shardKey, "some-shard-name"))
+            .addAttributes(OtlpTestHelpers.attribute(clusterKey, "some-cluster-name"))
+            .build();
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .setResource(resource)
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+
+  @Test
+  public void appRelatedResAttrsCanBeExcluded() {
+    boolean shouldIncludeOtlpAppTagsOnMetrics = false;
+
+    subject =
+        new OtlpGrpcMetricsHandler(
+            mockReportPointHandler,
+            mockHistogramHandler,
+            preprocessorSupplier,
+            DEFAULT_SOURCE,
+            false,
+            shouldIncludeOtlpAppTagsOnMetrics);
+    String applicationKey = "application";
+    String serviceNameKey = "service.name";
+    String shardKey = "shard";
+    String clusterKey = "cluster";
+
+    EasyMock.reset(mockReportPointHandler);
+
+    ReportPoint wfMetric = OtlpTestHelpers.wfReportPointGenerator().build();
+    assertFalse(wfMetric.getAnnotations().containsKey(applicationKey));
+    assertFalse(wfMetric.getAnnotations().containsKey(serviceNameKey));
+    assertFalse(wfMetric.getAnnotations().containsKey("service"));
+    assertFalse(wfMetric.getAnnotations().containsKey(shardKey));
+    assertFalse(wfMetric.getAnnotations().containsKey(clusterKey));
+    mockReportPointHandler.report(wfMetric);
+    EasyMock.expectLastCall();
+
+    EasyMock.replay(mockReportPointHandler);
+
+    NumberDataPoint otelPoint = NumberDataPoint.newBuilder().build();
+    Metric otelMetric = OtlpTestHelpers.otlpGaugeGenerator(otelPoint).build();
+    Resource resource =
+        Resource.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute(applicationKey, "some-app-name"))
+            .addAttributes(OtlpTestHelpers.attribute(serviceNameKey, "some-service-name"))
+            .addAttributes(OtlpTestHelpers.attribute(shardKey, "some-shard-name"))
+            .addAttributes(OtlpTestHelpers.attribute(clusterKey, "some-cluster-name"))
+            .build();
+    ResourceMetrics resourceMetrics =
+        ResourceMetrics.newBuilder()
+            .setResource(resource)
+            .addScopeMetrics(ScopeMetrics.newBuilder().addMetrics(otelMetric).build())
+            .build();
+    ExportMetricsServiceRequest request =
+        ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build();
+
+    subject.export(request, emptyStreamObserver);
+
+    EasyMock.verify(mockReportPointHandler);
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpGrpcTraceHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpGrpcTraceHandlerTest.java
new file mode 100644
index 000000000..81d697501
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpGrpcTraceHandlerTest.java
@@ -0,0 +1,118 @@
+package com.wavefront.agent.listeners.otlp;
+
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.HEART_BEAT_METRIC;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.junit.Assert.assertEquals;
+
+import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.sdk.common.WavefrontSender;
+import io.grpc.stub.StreamObserver;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse;
+import io.opentelemetry.proto.trace.v1.Span;
+import java.util.Arrays;
+import java.util.HashMap;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import wavefront.report.Annotation;
+
+/**
+ * @author Xiaochen Wang (xiaochenw@vmware.com).
+ * @author Glenn Oppegard (goppegard@vmware.com).
+ */
+public class OtlpGrpcTraceHandlerTest {
+  private final ReportableEntityHandler mockSpanHandler =
+      MockReportableEntityHandlerFactory.getMockTraceHandler();
+  private final ReportableEntityHandler mockSpanLogsHandler =
+      MockReportableEntityHandlerFactory.getMockTraceSpanLogsHandler();
+  private final SpanSampler mockSampler = EasyMock.createMock(SpanSampler.class);
+  private final WavefrontSender mockSender = EasyMock.createMock(WavefrontSender.class);
+
+  @Test
+  public void testMinimalSpanAndEventAndHeartbeat() throws Exception {
+    // 1. Arrange
+    EasyMock.reset(mockSpanHandler, mockSpanLogsHandler, mockSampler, mockSender);
+    expect(mockSampler.sample(anyObject(), anyObject())).andReturn(true);
+    Capture actualSpan = EasyMock.newCapture();
+    Capture actualLogs = EasyMock.newCapture();
+    mockSpanHandler.report(EasyMock.capture(actualSpan));
+    mockSpanLogsHandler.report(EasyMock.capture(actualLogs));
+
+    Capture> heartbeatTagsCapture = EasyMock.newCapture();
+    ;
+    mockSender.sendMetric(
+        eq(HEART_BEAT_METRIC),
+        eq(1.0),
+        anyLong(),
+        eq("test-source"),
+        EasyMock.capture(heartbeatTagsCapture));
+    expectLastCall().times(2);
+
+    EasyMock.replay(mockSampler, mockSpanHandler, mockSpanLogsHandler, mockSender);
+
+    Span.Event otlpEvent = OtlpTestHelpers.otlpSpanEvent(0);
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().addEvents(otlpEvent).build();
+    ExportTraceServiceRequest otlpRequest = OtlpTestHelpers.otlpTraceRequest(otlpSpan);
+
+    // 2. Act
+    OtlpGrpcTraceHandler otlpGrpcTraceHandler =
+        new OtlpGrpcTraceHandler(
+            "9876",
+            mockSpanHandler,
+            mockSpanLogsHandler,
+            mockSender,
+            null,
+            mockSampler,
+            () -> false,
+            () -> false,
+            "test-source",
+            null);
+    otlpGrpcTraceHandler.export(otlpRequest, emptyStreamObserver);
+    otlpGrpcTraceHandler.run();
+    otlpGrpcTraceHandler.close();
+
+    // 3. Assert
+    EasyMock.verify(mockSampler, mockSpanHandler, mockSpanLogsHandler, mockSender);
+
+    wavefront.report.Span expectedSpan =
+        OtlpTestHelpers.wfSpanGenerator(Arrays.asList(new Annotation("_spanLogs", "true"))).build();
+    wavefront.report.SpanLogs expectedLogs =
+        OtlpTestHelpers.wfSpanLogsGenerator(expectedSpan, 0, "_sampledByPolicy=NONE").build();
+
+    OtlpTestHelpers.assertWFSpanEquals(expectedSpan, actualSpan.getValue());
+    assertEquals(expectedLogs, actualLogs.getValue());
+
+    HashMap actualHeartbeatTags = heartbeatTagsCapture.getValue();
+    assertEquals(6, actualHeartbeatTags.size());
+    assertEquals("defaultApplication", actualHeartbeatTags.get(APPLICATION_TAG_KEY));
+    assertEquals("none", actualHeartbeatTags.get(CLUSTER_TAG_KEY));
+    assertEquals("otlp", actualHeartbeatTags.get(COMPONENT_TAG_KEY));
+    assertEquals("defaultService", actualHeartbeatTags.get(SERVICE_TAG_KEY));
+    assertEquals("none", actualHeartbeatTags.get(SHARD_TAG_KEY));
+    assertEquals("none", actualHeartbeatTags.get("span.kind"));
+  }
+
+  public static final StreamObserver emptyStreamObserver =
+      new StreamObserver() {
+        @Override
+        public void onNext(ExportTraceServiceResponse postSpansResponse) {}
+
+        @Override
+        public void onError(Throwable throwable) {}
+
+        @Override
+        public void onCompleted() {}
+      };
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpHttpHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpHttpHandlerTest.java
new file mode 100644
index 000000000..a582e8c2b
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpHttpHandlerTest.java
@@ -0,0 +1,116 @@
+package com.wavefront.agent.listeners.otlp;
+
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.HEART_BEAT_METRIC;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.junit.Assert.assertEquals;
+
+import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.sdk.common.WavefrontSender;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.DefaultFullHttpRequest;
+import io.netty.handler.codec.http.DefaultHttpHeaders;
+import io.netty.handler.codec.http.EmptyHttpHeaders;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpVersion;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import java.util.HashMap;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+import wavefront.report.Span;
+import wavefront.report.SpanLogs;
+
+/**
+ * Unit tests for {@link OtlpHttpHandler}.
+ *
+ * @author Glenn Oppegard (goppegard@vmware.com)
+ */
+public class OtlpHttpHandlerTest {
+  private final ReportableEntityHandler mockTraceHandler =
+      MockReportableEntityHandlerFactory.getMockTraceHandler();
+  private final ReportableEntityHandler mockSpanLogsHandler =
+      MockReportableEntityHandlerFactory.getMockTraceSpanLogsHandler();
+  private final SpanSampler mockSampler = EasyMock.createMock(SpanSampler.class);
+  private final WavefrontSender mockSender = EasyMock.createMock(WavefrontSender.class);
+  private final ReportableEntityHandlerFactory mockHandlerFactory =
+      MockReportableEntityHandlerFactory.createMockHandlerFactory(
+          null, null, null, mockTraceHandler, mockSpanLogsHandler, null);
+  private final ChannelHandlerContext mockCtx =
+      EasyMock.createNiceMock(ChannelHandlerContext.class);
+
+  @Before
+  public void setup() {
+    EasyMock.reset(mockTraceHandler, mockSpanLogsHandler, mockSampler, mockSender, mockCtx);
+  }
+
+  @Test
+  public void testHeartbeatEmitted() throws Exception {
+    EasyMock.expect(mockSampler.sample(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(true);
+    Capture> heartbeatTagsCapture = EasyMock.newCapture();
+    mockSender.sendMetric(
+        eq(HEART_BEAT_METRIC),
+        eq(1.0),
+        anyLong(),
+        eq("defaultSource"),
+        EasyMock.capture(heartbeatTagsCapture));
+    expectLastCall().times(2);
+    EasyMock.replay(mockSampler, mockSender, mockCtx);
+
+    OtlpHttpHandler handler =
+        new OtlpHttpHandler(
+            mockHandlerFactory,
+            null,
+            null,
+            "4318",
+            mockSender,
+            null,
+            mockSampler,
+            () -> false,
+            () -> false,
+            "defaultSource",
+            null,
+            false,
+            true);
+    io.opentelemetry.proto.trace.v1.Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().build();
+    ExportTraceServiceRequest otlpRequest = OtlpTestHelpers.otlpTraceRequest(otlpSpan);
+    ByteBuf body = Unpooled.copiedBuffer(otlpRequest.toByteArray());
+    HttpHeaders headers = new DefaultHttpHeaders().add("content-type", "application/x-protobuf");
+    FullHttpRequest request =
+        new DefaultFullHttpRequest(
+            HttpVersion.HTTP_1_1,
+            HttpMethod.POST,
+            "http://localhost:4318/v1/traces",
+            body,
+            headers,
+            EmptyHttpHeaders.INSTANCE);
+
+    handler.handleHttpMessage(mockCtx, request);
+    handler.run();
+    handler.close();
+
+    EasyMock.verify(mockSampler, mockSender);
+    HashMap actualHeartbeatTags = heartbeatTagsCapture.getValue();
+    assertEquals(6, actualHeartbeatTags.size());
+    assertEquals("defaultApplication", actualHeartbeatTags.get(APPLICATION_TAG_KEY));
+    assertEquals("none", actualHeartbeatTags.get(CLUSTER_TAG_KEY));
+    assertEquals("otlp", actualHeartbeatTags.get(COMPONENT_TAG_KEY));
+    assertEquals("defaultService", actualHeartbeatTags.get(SERVICE_TAG_KEY));
+    assertEquals("none", actualHeartbeatTags.get(SHARD_TAG_KEY));
+    assertEquals("none", actualHeartbeatTags.get("span.kind"));
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpMetricsUtilsTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpMetricsUtilsTest.java
new file mode 100644
index 000000000..a9e90d648
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpMetricsUtilsTest.java
@@ -0,0 +1,958 @@
+package com.wavefront.agent.listeners.otlp;
+
+import static com.wavefront.agent.listeners.otlp.OtlpMetricsUtils.MILLIS_IN_DAY;
+import static com.wavefront.agent.listeners.otlp.OtlpMetricsUtils.MILLIS_IN_HOUR;
+import static com.wavefront.agent.listeners.otlp.OtlpMetricsUtils.MILLIS_IN_MINUTE;
+import static com.wavefront.agent.listeners.otlp.OtlpMetricsUtils.replaceServiceNameKeyWithServiceKey;
+import static com.wavefront.agent.listeners.otlp.OtlpTestHelpers.DEFAULT_SOURCE;
+import static com.wavefront.agent.listeners.otlp.OtlpTestHelpers.assertAllPointsEqual;
+import static com.wavefront.agent.listeners.otlp.OtlpTestHelpers.attribute;
+import static com.wavefront.agent.listeners.otlp.OtlpTestHelpers.justThePointsNamed;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.wavefront.api.agent.preprocessor.PreprocessorRuleMetrics;
+import com.wavefront.api.agent.preprocessor.ReportPointAddTagIfNotExistsTransformer;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import io.opentelemetry.proto.common.v1.AnyValue;
+import io.opentelemetry.proto.common.v1.KeyValue;
+import io.opentelemetry.proto.metrics.v1.AggregationTemporality;
+import io.opentelemetry.proto.metrics.v1.ExponentialHistogram;
+import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
+import io.opentelemetry.proto.metrics.v1.Gauge;
+import io.opentelemetry.proto.metrics.v1.Histogram;
+import io.opentelemetry.proto.metrics.v1.HistogramDataPoint;
+import io.opentelemetry.proto.metrics.v1.Metric;
+import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
+import io.opentelemetry.proto.metrics.v1.Sum;
+import io.opentelemetry.proto.metrics.v1.Summary;
+import io.opentelemetry.proto.metrics.v1.SummaryDataPoint;
+import io.opentelemetry.proto.resource.v1.Resource;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.junit.Assert;
+import org.junit.Test;
+import wavefront.report.Annotation;
+import wavefront.report.HistogramType;
+import wavefront.report.ReportPoint;
+
+/** @author Sumit Deo (deosu@vmware.com) */
+public class OtlpMetricsUtilsTest {
+  private static final List emptyAttrs = Collections.unmodifiableList(new ArrayList<>());
+  private static final long startTimeMs = System.currentTimeMillis();
+
+  private List actualPoints;
+  private ImmutableList expectedPoints;
+
+  private static ImmutableList buildExpectedDeltaReportPoints(
+      List bins, List counts) {
+    wavefront.report.Histogram minHistogram =
+        wavefront.report.Histogram.newBuilder()
+            .setType(HistogramType.TDIGEST)
+            .setBins(bins)
+            .setCounts(counts)
+            .setDuration(MILLIS_IN_MINUTE)
+            .build();
+
+    wavefront.report.Histogram hourHistogram =
+        wavefront.report.Histogram.newBuilder()
+            .setType(HistogramType.TDIGEST)
+            .setBins(bins)
+            .setCounts(counts)
+            .setDuration(MILLIS_IN_HOUR)
+            .build();
+
+    wavefront.report.Histogram dayHistogram =
+        wavefront.report.Histogram.newBuilder()
+            .setType(HistogramType.TDIGEST)
+            .setBins(bins)
+            .setCounts(counts)
+            .setDuration(MILLIS_IN_DAY)
+            .build();
+
+    return ImmutableList.of(
+        OtlpTestHelpers.wfReportPointGenerator().setValue(minHistogram).build(),
+        OtlpTestHelpers.wfReportPointGenerator().setValue(hourHistogram).build(),
+        OtlpTestHelpers.wfReportPointGenerator().setValue(dayHistogram).build());
+  }
+
+  private static List buildExpectedCumulativeReportPoints(
+      List bins, List counts) {
+    List reportPoints = new ArrayList<>();
+
+    return reportPoints;
+  }
+
+  @Test
+  public void rejectsEmptyMetric() {
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().build();
+
+    Assert.assertThrows(
+        IllegalArgumentException.class,
+        () -> {
+          OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+        });
+  }
+
+  @Test
+  public void rejectsGaugeWithZeroDataPoints() {
+    Gauge emptyGauge = Gauge.newBuilder().build();
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setGauge(emptyGauge).build();
+
+    Assert.assertThrows(
+        IllegalArgumentException.class,
+        () -> {
+          OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+        });
+  }
+
+  @Test
+  public void transformsMinimalGauge() {
+    Gauge otlpGauge =
+        Gauge.newBuilder().addDataPoints(NumberDataPoint.newBuilder().build()).build();
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setGauge(otlpGauge).build();
+    expectedPoints = ImmutableList.of(OtlpTestHelpers.wfReportPointGenerator().build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformsGaugeTimestampToEpochMilliseconds() {
+    long timeInNanos = TimeUnit.MILLISECONDS.toNanos(startTimeMs);
+    Gauge otlpGauge =
+        Gauge.newBuilder()
+            .addDataPoints(NumberDataPoint.newBuilder().setTimeUnixNano(timeInNanos).build())
+            .build();
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setGauge(otlpGauge).build();
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator().setTimestamp(startTimeMs).build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void acceptsGaugeWithMultipleDataPoints() {
+    List points =
+        ImmutableList.of(
+            NumberDataPoint.newBuilder()
+                .setTimeUnixNano(TimeUnit.SECONDS.toNanos(1))
+                .setAsDouble(1.0)
+                .build(),
+            NumberDataPoint.newBuilder()
+                .setTimeUnixNano(TimeUnit.SECONDS.toNanos(2))
+                .setAsDouble(2.0)
+                .build());
+    Metric otlpMetric = OtlpTestHelpers.otlpGaugeGenerator(points).build();
+
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setTimestamp(TimeUnit.SECONDS.toMillis(1))
+                .setValue(1.0)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setTimestamp(TimeUnit.SECONDS.toMillis(2))
+                .setValue(2.0)
+                .build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void handlesGaugeAttributes() {
+    KeyValue booleanAttr =
+        KeyValue.newBuilder()
+            .setKey("a-boolean")
+            .setValue(AnyValue.newBuilder().setBoolValue(true).build())
+            .build();
+
+    Gauge otlpGauge =
+        Gauge.newBuilder()
+            .addDataPoints(NumberDataPoint.newBuilder().addAttributes(booleanAttr).build())
+            .build();
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setGauge(otlpGauge).build();
+
+    List wfAttrs =
+        Collections.singletonList(
+            Annotation.newBuilder().setKey("a-boolean").setValue("true").build());
+    expectedPoints = ImmutableList.of(OtlpTestHelpers.wfReportPointGenerator(wfAttrs).build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void rejectsSumWithZeroDataPoints() {
+    Sum emptySum = Sum.newBuilder().build();
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setSum(emptySum).build();
+
+    Assert.assertThrows(
+        IllegalArgumentException.class,
+        () -> {
+          OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+        });
+  }
+
+  @Test
+  public void transformsMinimalSum() {
+    Sum otlpSum =
+        Sum.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addDataPoints(NumberDataPoint.newBuilder().build())
+            .build();
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setSum(otlpSum).build();
+    expectedPoints = ImmutableList.of(OtlpTestHelpers.wfReportPointGenerator().build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformsSumTimestampToEpochMilliseconds() {
+    long timeInNanos = TimeUnit.MILLISECONDS.toNanos(startTimeMs);
+    Sum otlpSum =
+        Sum.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addDataPoints(NumberDataPoint.newBuilder().setTimeUnixNano(timeInNanos).build())
+            .build();
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setSum(otlpSum).build();
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator().setTimestamp(startTimeMs).build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void acceptsSumWithIntAndDoubleDataPoints() {
+    List points =
+        ImmutableList.of(
+            NumberDataPoint.newBuilder()
+                .setTimeUnixNano(TimeUnit.SECONDS.toNanos(1))
+                .setAsInt(1)
+                .build(),
+            NumberDataPoint.newBuilder()
+                .setTimeUnixNano(TimeUnit.SECONDS.toNanos(2))
+                .setAsDouble(2.0)
+                .build());
+    Metric otlpMetric = OtlpTestHelpers.otlpSumGenerator(points).build();
+
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setTimestamp(TimeUnit.SECONDS.toMillis(1))
+                .setValue(1)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setTimestamp(TimeUnit.SECONDS.toMillis(2))
+                .setValue(2.0)
+                .build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void handlesSumAttributes() {
+    KeyValue booleanAttr =
+        KeyValue.newBuilder()
+            .setKey("a-boolean")
+            .setValue(AnyValue.newBuilder().setBoolValue(true).build())
+            .build();
+
+    Sum otlpSum =
+        Sum.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addDataPoints(NumberDataPoint.newBuilder().addAttributes(booleanAttr).build())
+            .build();
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setSum(otlpSum).build();
+
+    List wfAttrs =
+        Collections.singletonList(
+            Annotation.newBuilder().setKey("a-boolean").setValue("true").build());
+    expectedPoints = ImmutableList.of(OtlpTestHelpers.wfReportPointGenerator(wfAttrs).build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void addsPrefixToDeltaSums() {
+    Sum otlpSum =
+        Sum.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA)
+            .addDataPoints(NumberDataPoint.newBuilder().build())
+            .build();
+    Metric otlpMetric =
+        OtlpTestHelpers.otlpMetricGenerator().setSum(otlpSum).setName("testSum").build();
+    ReportPoint reportPoint =
+        OtlpTestHelpers.wfReportPointGenerator().setMetric("∆testSum").build();
+    expectedPoints = ImmutableList.of(reportPoint);
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformsMinimalSummary() {
+    SummaryDataPoint point =
+        SummaryDataPoint.newBuilder()
+            .addQuantileValues(
+                SummaryDataPoint.ValueAtQuantile.newBuilder()
+                    .setQuantile(.5)
+                    .setValue(12.3)
+                    .build())
+            .setSum(24.5)
+            .setCount(3)
+            .build();
+    Metric otlpMetric = OtlpTestHelpers.otlpSummaryGenerator(point).setName("testSummary").build();
+
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setMetric("testSummary_sum")
+                .setValue(24.5)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setMetric("testSummary_count")
+                .setValue(3)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setMetric("testSummary")
+                .setValue(12.3)
+                .setAnnotations(ImmutableMap.of("quantile", "0.5"))
+                .build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformsSummaryTimestampToEpochMilliseconds() {
+    SummaryDataPoint point =
+        SummaryDataPoint.newBuilder()
+            .addQuantileValues(SummaryDataPoint.ValueAtQuantile.newBuilder().build())
+            .setTimeUnixNano(TimeUnit.MILLISECONDS.toNanos(startTimeMs))
+            .build();
+    Metric otlpMetric = OtlpTestHelpers.otlpSummaryGenerator(point).build();
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    for (ReportPoint p : actualPoints) {
+      assertEquals(startTimeMs, p.getTimestamp());
+    }
+  }
+
+  @Test
+  public void acceptsSummaryWithMultipleDataPoints() {
+    List points =
+        ImmutableList.of(
+            SummaryDataPoint.newBuilder()
+                .setTimeUnixNano(TimeUnit.SECONDS.toNanos(1))
+                .setSum(1.0)
+                .setCount(1)
+                .build(),
+            SummaryDataPoint.newBuilder()
+                .setTimeUnixNano(TimeUnit.SECONDS.toNanos(2))
+                .setSum(2.0)
+                .setCount(2)
+                .build());
+    Summary otlpSummary = Summary.newBuilder().addAllDataPoints(points).build();
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setSummary(otlpSummary).build();
+
+    expectedPoints =
+        ImmutableList.of(
+            // SummaryDataPoint 1
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setMetric("test_sum")
+                .setTimestamp(TimeUnit.SECONDS.toMillis(1))
+                .setValue(1.0)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setMetric("test_count")
+                .setTimestamp(TimeUnit.SECONDS.toMillis(1))
+                .setValue(1)
+                .build(),
+            // SummaryDataPoint 2
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setMetric("test_sum")
+                .setTimestamp(TimeUnit.SECONDS.toMillis(2))
+                .setValue(2.0)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setMetric("test_count")
+                .setTimestamp(TimeUnit.SECONDS.toMillis(2))
+                .setValue(2)
+                .build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void createsMetricsForEachSummaryQuantile() {
+    Metric otlpMetric =
+        OtlpTestHelpers.otlpSummaryGenerator(
+                ImmutableList.of(
+                    SummaryDataPoint.ValueAtQuantile.newBuilder()
+                        .setQuantile(.2)
+                        .setValue(2.2)
+                        .build(),
+                    SummaryDataPoint.ValueAtQuantile.newBuilder()
+                        .setQuantile(.4)
+                        .setValue(4.4)
+                        .build(),
+                    SummaryDataPoint.ValueAtQuantile.newBuilder()
+                        .setQuantile(.6)
+                        .setValue(6.6)
+                        .build()))
+            .build();
+
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setAnnotations(ImmutableMap.of("quantile", "0.2"))
+                .setValue(2.2)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setAnnotations(ImmutableMap.of("quantile", "0.4"))
+                .setValue(4.4)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator()
+                .setAnnotations(ImmutableMap.of("quantile", "0.6"))
+                .setValue(6.6)
+                .build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, justThePointsNamed("test", actualPoints));
+  }
+
+  @Test
+  public void preservesOverriddenQuantileTag() {
+    KeyValue quantileTag =
+        KeyValue.newBuilder()
+            .setKey("quantile")
+            .setValue(AnyValue.newBuilder().setStringValue("half").build())
+            .build();
+    SummaryDataPoint point =
+        SummaryDataPoint.newBuilder()
+            .addQuantileValues(
+                SummaryDataPoint.ValueAtQuantile.newBuilder()
+                    .setQuantile(.5)
+                    .setValue(12.3)
+                    .build())
+            .addAttributes(quantileTag)
+            .build();
+    Metric otlpMetric = OtlpTestHelpers.otlpSummaryGenerator(point).setName("testSummary").build();
+
+    for (ReportPoint p : OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE)) {
+      assertEquals("half", p.getAnnotations().get("_quantile"));
+      if (p.getMetric().equals("testSummary")) {
+        assertEquals("0.5", p.getAnnotations().get("quantile"));
+      }
+    }
+  }
+
+  @Test
+  public void handlesSummaryAttributes() {
+    KeyValue booleanAttr =
+        KeyValue.newBuilder()
+            .setKey("a-boolean")
+            .setValue(AnyValue.newBuilder().setBoolValue(true).build())
+            .build();
+
+    SummaryDataPoint dataPoint = SummaryDataPoint.newBuilder().addAttributes(booleanAttr).build();
+    Metric otlpMetric = OtlpTestHelpers.otlpSummaryGenerator(dataPoint).build();
+
+    for (ReportPoint p : OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE)) {
+      assertEquals("true", p.getAnnotations().get("a-boolean"));
+    }
+  }
+
+  @Test
+  public void transformsMinimalCumulativeHistogram() {
+    HistogramDataPoint point =
+        HistogramDataPoint.newBuilder()
+            .addAllExplicitBounds(ImmutableList.of(1.0, 2.0))
+            .addAllBucketCounts(ImmutableList.of(1L, 1L, 1L))
+            .build();
+    Histogram histo =
+        Histogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addAllDataPoints(Collections.singletonList(point))
+            .build();
+
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setHistogram(histo).build();
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "1.0")))
+                .setValue(1)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "2.0")))
+                .setValue(2)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "+Inf")))
+                .setValue(3)
+                .build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformsCumulativeHistogramWithoutBounds() {
+    HistogramDataPoint point =
+        HistogramDataPoint.newBuilder().addAllBucketCounts(ImmutableList.of(1L)).build();
+    Histogram histo =
+        Histogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addAllDataPoints(Collections.singletonList(point))
+            .build();
+
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setHistogram(histo).build();
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "+Inf")))
+                .setValue(1)
+                .build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformsCumulativeHistogramWithTagLe() {
+    HistogramDataPoint point =
+        HistogramDataPoint.newBuilder()
+            .addAllBucketCounts(ImmutableList.of(1L))
+            .addAttributes(OtlpTestHelpers.attribute("le", "someVal"))
+            .build();
+    Histogram histo =
+        Histogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addAllDataPoints(Collections.singletonList(point))
+            .build();
+
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setHistogram(histo).build();
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator(
+                    ImmutableList.of(
+                        new Annotation("le", "+Inf"), new Annotation("_le", "someVal")))
+                .setValue(1)
+                .build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformsCumulativeHistogramThrowsMalformedDataPointsError() {
+    HistogramDataPoint point =
+        HistogramDataPoint.newBuilder()
+            .addAllExplicitBounds(Collections.singletonList(1.0))
+            .addAllBucketCounts(ImmutableList.of(1L))
+            .build();
+    Histogram histo =
+        Histogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addAllDataPoints(Collections.singletonList(point))
+            .build();
+
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setHistogram(histo).build();
+
+    Assert.assertThrows(
+        IllegalArgumentException.class,
+        () -> OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE));
+  }
+
+  @Test
+  public void transformsMinimalDeltaHistogram() {
+    HistogramDataPoint point =
+        HistogramDataPoint.newBuilder()
+            .addAllExplicitBounds(ImmutableList.of(1.0, 2.0))
+            .addAllBucketCounts(ImmutableList.of(1L, 2L, 3L))
+            .build();
+    Histogram histo =
+        Histogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA)
+            .addAllDataPoints(Collections.singletonList(point))
+            .build();
+
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setHistogram(histo).build();
+
+    List bins = new ArrayList<>(Arrays.asList(1.0, 1.5, 2.0));
+    List counts = new ArrayList<>(Arrays.asList(1, 2, 3));
+
+    expectedPoints = buildExpectedDeltaReportPoints(bins, counts);
+
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformsDeltaHistogramWithoutBounds() {
+    HistogramDataPoint point =
+        HistogramDataPoint.newBuilder().addAllBucketCounts(ImmutableList.of(1L)).build();
+    Histogram histo =
+        Histogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA)
+            .addAllDataPoints(Collections.singletonList(point))
+            .build();
+
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setHistogram(histo).build();
+
+    List bins = new ArrayList<>(Collections.singletonList(0.0));
+    List counts = new ArrayList<>(Collections.singletonList(1));
+
+    expectedPoints = buildExpectedDeltaReportPoints(bins, counts);
+
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformsDeltaHistogramThrowsMalformedDataPointsError() {
+    HistogramDataPoint point =
+        HistogramDataPoint.newBuilder()
+            .addAllExplicitBounds(Collections.singletonList(1.0))
+            .addAllBucketCounts(ImmutableList.of(1L))
+            .build();
+    Histogram histo =
+        Histogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA)
+            .addAllDataPoints(Collections.singletonList(point))
+            .build();
+
+    Metric otlpMetric = OtlpTestHelpers.otlpMetricGenerator().setHistogram(histo).build();
+
+    Assert.assertThrows(
+        IllegalArgumentException.class,
+        () -> OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE));
+  }
+
+  @Test
+  public void transformExpDeltaHistogram() {
+    ExponentialHistogramDataPoint point =
+        ExponentialHistogramDataPoint.newBuilder()
+            .setScale(1)
+            .setPositive(
+                ExponentialHistogramDataPoint.Buckets.newBuilder()
+                    .setOffset(3)
+                    .addBucketCounts(2)
+                    .addBucketCounts(1)
+                    .addBucketCounts(4)
+                    .addBucketCounts(3)
+                    .build())
+            .setZeroCount(5)
+            .build();
+    ExponentialHistogram histo =
+        ExponentialHistogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA)
+            .addDataPoints(point)
+            .build();
+    Metric otlpMetric =
+        OtlpTestHelpers.otlpMetricGenerator().setExponentialHistogram(histo).build();
+
+    // Actual buckets: -1, 2.8284, 4, 5.6569, 8, 11.3137, but we average the lower and upper
+    // bound of
+    // each bucket when doing delta histogram centroids.
+    List bins = Arrays.asList(0.9142, 3.4142, 4.8284, 6.8284, 9.6569);
+    List counts = Arrays.asList(5, 2, 1, 4, 3);
+
+    expectedPoints = buildExpectedDeltaReportPoints(bins, counts);
+
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformExpDeltaHistogramWithNegativeValues() {
+    ExponentialHistogramDataPoint point =
+        ExponentialHistogramDataPoint.newBuilder()
+            .setScale(-1)
+            .setPositive(
+                ExponentialHistogramDataPoint.Buckets.newBuilder()
+                    .setOffset(2)
+                    .addBucketCounts(3)
+                    .addBucketCounts(2)
+                    .addBucketCounts(5)
+                    .build())
+            .setZeroCount(1)
+            .setNegative(
+                ExponentialHistogramDataPoint.Buckets.newBuilder()
+                    .setOffset(-1)
+                    .addBucketCounts(6)
+                    .addBucketCounts(4)
+                    .build())
+            .build();
+
+    ExponentialHistogram histo =
+        ExponentialHistogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA)
+            .addDataPoints(point)
+            .build();
+    Metric otlpMetric =
+        OtlpTestHelpers.otlpMetricGenerator().setExponentialHistogram(histo).build();
+
+    // actual buckets: -4, -1, -0.25, 16.0, 64.0, 256.0, 1024.0, but we average the lower and
+    // upper
+    // bound of
+    // each bucket when doing delta histogram centroids.
+    List bins = Arrays.asList(-2.5, -0.625, 7.875, 40.0, 160.0, 640.0);
+    List counts = Arrays.asList(4, 6, 1, 3, 2, 5);
+
+    expectedPoints = buildExpectedDeltaReportPoints(bins, counts);
+
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformExpCumulativeHistogram() {
+    ExponentialHistogramDataPoint point =
+        ExponentialHistogramDataPoint.newBuilder()
+            .setScale(0)
+            .setPositive(
+                ExponentialHistogramDataPoint.Buckets.newBuilder()
+                    .setOffset(2)
+                    .addBucketCounts(1)
+                    .addBucketCounts(2)
+                    .build())
+            .setZeroCount(3)
+            .build();
+    ExponentialHistogram histo =
+        ExponentialHistogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addDataPoints(point)
+            .build();
+    Metric otlpMetric =
+        OtlpTestHelpers.otlpMetricGenerator().setExponentialHistogram(histo).build();
+
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "4.0")))
+                .setValue(3)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "8.0")))
+                .setValue(4)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "16.0")))
+                .setValue(6)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "+Inf")))
+                .setValue(6)
+                .build());
+
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void transformExpCumulativeHistogramWithNegativeValues() {
+    ExponentialHistogramDataPoint point =
+        ExponentialHistogramDataPoint.newBuilder()
+            .setScale(0)
+            .setPositive(
+                ExponentialHistogramDataPoint.Buckets.newBuilder()
+                    .setOffset(2)
+                    .addBucketCounts(1)
+                    .build())
+            .setZeroCount(2)
+            .setNegative(
+                ExponentialHistogramDataPoint.Buckets.newBuilder()
+                    .setOffset(2)
+                    .addBucketCounts(3)
+                    .build())
+            .build();
+
+    ExponentialHistogram histo =
+        ExponentialHistogram.newBuilder()
+            .setAggregationTemporality(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE)
+            .addDataPoints(point)
+            .build();
+    Metric otlpMetric =
+        OtlpTestHelpers.otlpMetricGenerator().setExponentialHistogram(histo).build();
+
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "-4.0")))
+                .setValue(3)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "4.0")))
+                .setValue(5)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "8.0")))
+                .setValue(6)
+                .build(),
+            OtlpTestHelpers.wfReportPointGenerator(ImmutableList.of(new Annotation("le", "+Inf")))
+                .setValue(6)
+                .build());
+
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, DEFAULT_SOURCE);
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void convertsResourceAttributesToAnnotations() {
+    List resourceAttrs = Collections.singletonList(attribute("r-key", "r-value"));
+    expectedPoints =
+        ImmutableList.of(
+            OtlpTestHelpers.wfReportPointGenerator(
+                    Collections.singletonList(new Annotation("r-key", "r-value")))
+                .build());
+    NumberDataPoint point = NumberDataPoint.newBuilder().setTimeUnixNano(0).build();
+    Metric otlpMetric = OtlpTestHelpers.otlpGaugeGenerator(point).build();
+
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, resourceAttrs, null, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void dataPointAttributesHaveHigherPrecedenceThanResourceAttributes() {
+    String key = "the-key";
+    NumberDataPoint point =
+        NumberDataPoint.newBuilder().addAttributes(attribute(key, "gauge-value")).build();
+    Metric otlpMetric = OtlpTestHelpers.otlpGaugeGenerator(point).build();
+    List resourceAttrs = Collections.singletonList(attribute(key, "rsrc-value"));
+
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, resourceAttrs, null, DEFAULT_SOURCE);
+
+    assertEquals("gauge-value", actualPoints.get(0).getAnnotations().get(key));
+  }
+
+  @Test
+  public void setsSource() {
+    Metric otlpMetric =
+        OtlpTestHelpers.otlpGaugeGenerator(NumberDataPoint.newBuilder().build()).build();
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, null, "a-src");
+
+    assertEquals("a-src", actualPoints.get(0).getHost());
+  }
+
+  @Test
+  public void appliesPreprocessorRules() {
+    List dataPoints =
+        Collections.singletonList(NumberDataPoint.newBuilder().setTimeUnixNano(0).build());
+    Metric otlpMetric = OtlpTestHelpers.otlpGaugeGenerator(dataPoints).build();
+    List wfAttrs =
+        Collections.singletonList(
+            Annotation.newBuilder().setKey("my-key").setValue("my-value").build());
+    ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+    PreprocessorRuleMetrics preprocessorRuleMetrics = new PreprocessorRuleMetrics(null, null, null);
+    for (Annotation annotation : wfAttrs) {
+      preprocessor
+          .forReportPoint()
+          .addTransformer(
+              new ReportPointAddTagIfNotExistsTransformer(
+                  annotation.getKey(), annotation.getValue(), x -> true, preprocessorRuleMetrics));
+    }
+    expectedPoints = ImmutableList.of(OtlpTestHelpers.wfReportPointGenerator(wfAttrs).build());
+    actualPoints = OtlpMetricsUtils.transform(otlpMetric, emptyAttrs, preprocessor, DEFAULT_SOURCE);
+
+    assertAllPointsEqual(expectedPoints, actualPoints);
+  }
+
+  @Test
+  public void testAppTagsFromResourceAttrs() {
+    Resource otelResource =
+        Resource.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute("application", "some-app-name"))
+            .addAttributes(OtlpTestHelpers.attribute("service.name", "some-service-name"))
+            .addAttributes(OtlpTestHelpers.attribute("shard", "some-shard-name"))
+            .addAttributes(OtlpTestHelpers.attribute("cluster", "some-cluster-name"))
+            .build();
+
+    List attrList =
+        OtlpMetricsUtils.appTagsFromResourceAttrs(otelResource.getAttributesList());
+    assertEquals(
+        "some-app-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "application").getValue().getStringValue());
+    assertEquals(
+        "some-service-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "service").getValue().getStringValue());
+    assertEquals(
+        "some-shard-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "shard").getValue().getStringValue());
+    assertEquals(
+        "some-cluster-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "cluster").getValue().getStringValue());
+  }
+
+  @Test
+  public void testAppTagsFromResourceAttrsWhenServiceKeyIsPresent() {
+    Resource otelResource =
+        Resource.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute("application", "some-app-name"))
+            .addAttributes(OtlpTestHelpers.attribute("service", "some-service-name"))
+            .addAttributes(OtlpTestHelpers.attribute("service.name", "some-other-service-name"))
+            .addAttributes(OtlpTestHelpers.attribute("shard", "some-shard-name"))
+            .addAttributes(OtlpTestHelpers.attribute("cluster", "some-cluster-name"))
+            .build();
+
+    List attrList =
+        OtlpMetricsUtils.appTagsFromResourceAttrs(otelResource.getAttributesList());
+    assertEquals(
+        "some-app-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "application").getValue().getStringValue());
+    assertEquals(
+        "some-service-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "service").getValue().getStringValue());
+    assertEquals(
+        "some-shard-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "shard").getValue().getStringValue());
+    assertEquals(
+        "some-cluster-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "cluster").getValue().getStringValue());
+    assertNull(OtlpTraceUtils.getAttrByKey(attrList, "service.name"));
+  }
+
+  @Test
+  public void testAppTagsFromResourceAttrsWhenAttrsAreNotProvided() {
+    Resource otelResource = Resource.newBuilder().build();
+
+    List attrList =
+        OtlpMetricsUtils.appTagsFromResourceAttrs(otelResource.getAttributesList());
+    assertTrue(attrList.isEmpty());
+  }
+
+  @Test
+  public void testReplaceServiceNameKeyWithServiceKey() {
+    Resource otelResource =
+        Resource.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute("service.name", "some-service-name"))
+            .build();
+
+    List attrList = replaceServiceNameKeyWithServiceKey(otelResource.getAttributesList());
+    assertEquals(
+        "some-service-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "service").getValue().getStringValue());
+
+    otelResource =
+        Resource.newBuilder()
+            .addAttributes(OtlpTestHelpers.attribute("service", "some-service-name"))
+            .addAttributes(OtlpTestHelpers.attribute("service.name", "some-other-service-name"))
+            .build();
+
+    attrList = replaceServiceNameKeyWithServiceKey(otelResource.getAttributesList());
+    assertEquals(
+        "some-service-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "service").getValue().getStringValue());
+    assertEquals(
+        "some-other-service-name",
+        OtlpTraceUtils.getAttrByKey(attrList, "service.name").getValue().getStringValue());
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpTestHelpers.java b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpTestHelpers.java
new file mode 100644
index 000000000..f4a574816
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpTestHelpers.java
@@ -0,0 +1,354 @@
+package com.wavefront.agent.listeners.otlp;
+
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.Maps;
+import com.google.protobuf.ByteString;
+import com.wavefront.api.agent.preprocessor.PreprocessorRuleMetrics;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.preprocessor.SpanAddAnnotationIfNotExistsTransformer;
+import com.wavefront.api.agent.preprocessor.SpanBlockFilter;
+import com.wavefront.sdk.common.Pair;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.common.v1.AnyValue;
+import io.opentelemetry.proto.common.v1.KeyValue;
+import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
+import io.opentelemetry.proto.metrics.v1.SummaryDataPoint;
+import io.opentelemetry.proto.trace.v1.ResourceSpans;
+import io.opentelemetry.proto.trace.v1.ScopeSpans;
+import io.opentelemetry.proto.trace.v1.Status;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.apache.commons.compress.utils.Lists;
+import org.hamcrest.FeatureMatcher;
+import wavefront.report.Annotation;
+import wavefront.report.Histogram;
+import wavefront.report.Span;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
+
+/**
+ * @author Xiaochen Wang (xiaochenw@vmware.com).
+ * @author Glenn Oppegard (goppegard@vmware.com).
+ */
+public class OtlpTestHelpers {
+  public static final String DEFAULT_SOURCE = "test-source";
+  private static final long startTimeMs = System.currentTimeMillis();
+  private static final long durationMs = 50L;
+  private static final byte[] spanIdBytes = {0x9, 0x9, 0x9, 0x9, 0x9, 0x9, 0x9, 0x9};
+  private static final byte[] parentSpanIdBytes = {0x6, 0x6, 0x6, 0x6, 0x6, 0x6, 0x6, 0x6};
+  private static final byte[] traceIdBytes = {
+    0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1
+  };
+
+  public static FeatureMatcher, Iterable> hasKey(String key) {
+    return new FeatureMatcher, Iterable>(
+        hasItem(key), "Annotations with Keys", "Annotation Key") {
+      @Override
+      protected Iterable featureValueOf(List actual) {
+        return actual.stream().map(Annotation::getKey).collect(Collectors.toList());
+      }
+    };
+  }
+
+  public static Span.Builder wfSpanGenerator(@Nullable List extraAttrs) {
+    if (extraAttrs == null) {
+      extraAttrs = Collections.emptyList();
+    }
+    List annotations = Lists.newArrayList();
+    if (extraAttrs.stream().noneMatch(anno -> anno.getKey().equals(APPLICATION_TAG_KEY))) {
+      annotations.add(new Annotation(APPLICATION_TAG_KEY, "defaultApplication"));
+    }
+    if (extraAttrs.stream().noneMatch(anno -> anno.getKey().equals(SERVICE_TAG_KEY))) {
+      annotations.add(new Annotation(SERVICE_TAG_KEY, "defaultService"));
+    }
+    if (extraAttrs.stream().noneMatch(anno -> anno.getKey().equals("cluster"))) {
+      annotations.add(new Annotation("cluster", "none"));
+    }
+    if (extraAttrs.stream().noneMatch(anno -> anno.getKey().equals("shard"))) {
+      annotations.add(new Annotation("shard", "none"));
+    }
+    if (extraAttrs.stream().noneMatch(anno -> anno.getKey().equals("span.kind"))) {
+      annotations.add(new Annotation("span.kind", "unspecified"));
+    }
+
+    annotations.addAll(extraAttrs);
+
+    return wavefront.report.Span.newBuilder()
+        .setName("root")
+        .setSpanId("00000000-0000-0000-0909-090909090909")
+        .setTraceId("01010101-0101-0101-0101-010101010101")
+        .setStartMillis(startTimeMs)
+        .setDuration(durationMs)
+        .setAnnotations(annotations)
+        .setSource(DEFAULT_SOURCE)
+        .setCustomer("dummy");
+  }
+
+  public static SpanLogs.Builder wfSpanLogsGenerator(Span span, int droppedAttrsCount) {
+    return wfSpanLogsGenerator(span, droppedAttrsCount, null);
+  }
+
+  public static SpanLogs.Builder wfSpanLogsGenerator(
+      Span span, int droppedAttrsCount, String spanLine) {
+
+    long logTimestamp =
+        TimeUnit.MILLISECONDS.toMicros(span.getStartMillis() + (span.getDuration() / 2));
+    Map logFields =
+        new HashMap() {
+          {
+            put("name", "eventName");
+            put("attrKey", "attrValue");
+          }
+        };
+
+    // otel spec says it's invalid to add the tag if the count is zero
+    if (droppedAttrsCount > 0) {
+      logFields.put("otel.dropped_attributes_count", String.valueOf(droppedAttrsCount));
+    }
+    SpanLog spanLog = SpanLog.newBuilder().setFields(logFields).setTimestamp(logTimestamp).build();
+
+    return SpanLogs.newBuilder()
+        .setLogs(Collections.singletonList(spanLog))
+        .setSpanId(span.getSpanId())
+        .setTraceId(span.getTraceId())
+        .setCustomer(span.getCustomer())
+        .setSpan(spanLine == null ? null : spanLine);
+  }
+
+  public static io.opentelemetry.proto.trace.v1.Span.Builder otlpSpanGenerator() {
+    return io.opentelemetry.proto.trace.v1.Span.newBuilder()
+        .setName("root")
+        .setSpanId(ByteString.copyFrom(spanIdBytes))
+        .setTraceId(ByteString.copyFrom(traceIdBytes))
+        .setStartTimeUnixNano(TimeUnit.MILLISECONDS.toNanos(startTimeMs))
+        .setEndTimeUnixNano(TimeUnit.MILLISECONDS.toNanos(startTimeMs + durationMs));
+  }
+
+  public static io.opentelemetry.proto.trace.v1.Span otlpSpanWithKind(
+      io.opentelemetry.proto.trace.v1.Span.SpanKind kind) {
+    return otlpSpanGenerator().setKind(kind).build();
+  }
+
+  public static io.opentelemetry.proto.trace.v1.Span otlpSpanWithStatus(
+      Status.StatusCode code, String message) {
+    Status status = Status.newBuilder().setCode(code).setMessage(message).build();
+    return otlpSpanGenerator().setStatus(status).build();
+  }
+
+  public static io.opentelemetry.proto.common.v1.KeyValue attribute(String key, String value) {
+    return KeyValue.newBuilder()
+        .setKey(key)
+        .setValue(AnyValue.newBuilder().setStringValue(value).build())
+        .build();
+  }
+
+  public static io.opentelemetry.proto.trace.v1.Span.Event otlpSpanEvent(int droppedAttrsCount) {
+    long eventTimestamp = TimeUnit.MILLISECONDS.toNanos(startTimeMs + (durationMs / 2));
+    KeyValue attr = attribute("attrKey", "attrValue");
+    io.opentelemetry.proto.trace.v1.Span.Event.Builder builder =
+        io.opentelemetry.proto.trace.v1.Span.Event.newBuilder()
+            .setName("eventName")
+            .setTimeUnixNano(eventTimestamp)
+            .addAttributes(attr);
+
+    if (droppedAttrsCount > 0) {
+      builder.setDroppedAttributesCount(droppedAttrsCount);
+    }
+    return builder.build();
+  }
+
+  public static Pair parentSpanIdPair() {
+    return Pair.of(ByteString.copyFrom(parentSpanIdBytes), "00000000-0000-0000-0606-060606060606");
+  }
+
+  public static ReportableEntityPreprocessor addTagIfNotExistsPreprocessor(
+      List annotationList) {
+    ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+    PreprocessorRuleMetrics preprocessorRuleMetrics = new PreprocessorRuleMetrics(null, null, null);
+    for (Annotation annotation : annotationList) {
+      preprocessor
+          .forSpan()
+          .addTransformer(
+              new SpanAddAnnotationIfNotExistsTransformer(
+                  annotation.getKey(), annotation.getValue(), x -> true, preprocessorRuleMetrics));
+    }
+
+    return preprocessor;
+  }
+
+  public static ReportableEntityPreprocessor blockSpanPreprocessor() {
+    ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+    PreprocessorRuleMetrics preprocessorRuleMetrics = new PreprocessorRuleMetrics(null, null, null);
+    preprocessor
+        .forSpan()
+        .addFilter(
+            new SpanBlockFilter("sourceName", DEFAULT_SOURCE, x -> true, preprocessorRuleMetrics));
+
+    return preprocessor;
+  }
+
+  public static ReportableEntityPreprocessor rejectSpanPreprocessor() {
+    ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+    preprocessor
+        .forSpan()
+        .addFilter(
+            (input, messageHolder) -> {
+              if (messageHolder != null && messageHolder.length > 0) {
+                messageHolder[0] = "span rejected for testing purpose";
+              }
+              return false;
+            });
+
+    return preprocessor;
+  }
+
+  public static void assertWFSpanEquals(
+      wavefront.report.Span expected, wavefront.report.Span actual) {
+    assertEquals(expected.getName(), actual.getName());
+    assertEquals(expected.getSpanId(), actual.getSpanId());
+    assertEquals(expected.getTraceId(), actual.getTraceId());
+    assertEquals(expected.getStartMillis(), actual.getStartMillis());
+    assertEquals(expected.getDuration(), actual.getDuration());
+    assertEquals(expected.getSource(), actual.getSource());
+    assertEquals(expected.getCustomer(), actual.getCustomer());
+
+    assertThat(
+        "Annotations match in any order",
+        actual.getAnnotations(),
+        containsInAnyOrder(expected.getAnnotations().toArray()));
+  }
+
+  public static ExportTraceServiceRequest otlpTraceRequest(
+      io.opentelemetry.proto.trace.v1.Span otlpSpan) {
+    ScopeSpans scopeSpans = ScopeSpans.newBuilder().addSpans(otlpSpan).build();
+    ResourceSpans rSpans = ResourceSpans.newBuilder().addScopeSpans(scopeSpans).build();
+    return ExportTraceServiceRequest.newBuilder().addResourceSpans(rSpans).build();
+  }
+
+  private static void assertHistogramsEqual(Histogram expected, Histogram actual, double delta) {
+    String errorSuffix = " mismatched. Expected: " + expected + " ,Actual: " + actual;
+    assertEquals("Histogram duration" + errorSuffix, expected.getDuration(), actual.getDuration());
+    assertEquals("Histogram type" + errorSuffix, expected.getType(), actual.getType());
+    List expectedBins = expected.getBins();
+    List actualBins = actual.getBins();
+    assertEquals("Histogram bin size" + errorSuffix, expectedBins.size(), actualBins.size());
+    for (int i = 0; i < expectedBins.size(); i++) {
+      assertEquals(
+          "Histogram bin " + i + errorSuffix, expectedBins.get(i), actualBins.get(i), delta);
+    }
+    assertEquals("Histogram counts" + errorSuffix, expected.getCounts(), actual.getCounts());
+  }
+
+  public static void assertWFReportPointEquals(
+      wavefront.report.ReportPoint expected, wavefront.report.ReportPoint actual) {
+    assertEquals("metric name", expected.getMetric(), actual.getMetric());
+    Object expectedValue = expected.getValue();
+    Object actualValue = actual.getValue();
+    if ((expectedValue instanceof Histogram) && (actualValue instanceof Histogram)) {
+      assertHistogramsEqual((Histogram) expectedValue, (Histogram) actualValue, 0.0001);
+    } else {
+      assertEquals("value", expectedValue, actualValue);
+    }
+    assertEquals("timestamp", expected.getTimestamp(), actual.getTimestamp());
+    assertEquals(
+        "number of annotations", expected.getAnnotations().size(), actual.getAnnotations().size());
+    assertEquals("source/host", expected.getHost(), actual.getHost());
+    // TODO use a better assert instead of iterating manually?
+    for (String key : expected.getAnnotations().keySet()) {
+      assertTrue(actual.getAnnotations().containsKey(key));
+      assertEquals(expected.getAnnotations().get(key), actual.getAnnotations().get(key));
+    }
+  }
+
+  public static void assertAllPointsEqual(
+      List expected, List actual) {
+    assertEquals("same number of points", expected.size(), actual.size());
+    for (int i = 0; i < expected.size(); i++) {
+      assertWFReportPointEquals(expected.get(i), actual.get(i));
+    }
+  }
+
+  private static Map annotationListToMap(List annotationList) {
+    Map annotationMap = Maps.newHashMap();
+    for (Annotation annotation : annotationList) {
+      annotationMap.put(annotation.getKey(), annotation.getValue());
+    }
+    assertEquals(annotationList.size(), annotationMap.size());
+    return annotationMap;
+  }
+
+  public static io.opentelemetry.proto.metrics.v1.Metric.Builder otlpMetricGenerator() {
+    return io.opentelemetry.proto.metrics.v1.Metric.newBuilder().setName("test");
+  }
+
+  public static io.opentelemetry.proto.metrics.v1.Metric.Builder otlpGaugeGenerator(
+      List points) {
+    return otlpMetricGenerator()
+        .setGauge(
+            io.opentelemetry.proto.metrics.v1.Gauge.newBuilder().addAllDataPoints(points).build());
+  }
+
+  public static io.opentelemetry.proto.metrics.v1.Metric.Builder otlpGaugeGenerator(
+      NumberDataPoint point) {
+    return otlpMetricGenerator()
+        .setGauge(
+            io.opentelemetry.proto.metrics.v1.Gauge.newBuilder().addDataPoints(point).build());
+  }
+
+  public static io.opentelemetry.proto.metrics.v1.Metric.Builder otlpSumGenerator(
+      List points) {
+    return otlpMetricGenerator()
+        .setSum(
+            io.opentelemetry.proto.metrics.v1.Sum.newBuilder()
+                .setAggregationTemporality(
+                    io.opentelemetry.proto.metrics.v1.AggregationTemporality
+                        .AGGREGATION_TEMPORALITY_CUMULATIVE)
+                .setIsMonotonic(true)
+                .addAllDataPoints(points)
+                .build());
+  }
+
+  public static io.opentelemetry.proto.metrics.v1.Metric.Builder otlpSummaryGenerator(
+      SummaryDataPoint point) {
+    return otlpMetricGenerator()
+        .setSummary(
+            io.opentelemetry.proto.metrics.v1.Summary.newBuilder().addDataPoints(point).build());
+  }
+
+  public static io.opentelemetry.proto.metrics.v1.Metric.Builder otlpSummaryGenerator(
+      Collection quantiles) {
+    return otlpSummaryGenerator(
+        SummaryDataPoint.newBuilder().addAllQuantileValues(quantiles).build());
+  }
+
+  public static wavefront.report.ReportPoint.Builder wfReportPointGenerator() {
+    return wavefront.report.ReportPoint.newBuilder()
+        .setMetric("test")
+        .setHost(DEFAULT_SOURCE)
+        .setTimestamp(0)
+        .setValue(0.0);
+  }
+
+  public static wavefront.report.ReportPoint.Builder wfReportPointGenerator(
+      List annotations) {
+    return wfReportPointGenerator().setAnnotations(annotationListToMap(annotations));
+  }
+
+  public static List justThePointsNamed(
+      String name, Collection points) {
+    return points.stream().filter(p -> p.getMetric().equals(name)).collect(Collectors.toList());
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpTraceUtilsTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpTraceUtilsTest.java
new file mode 100644
index 000000000..4d1b9300f
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/otlp/OtlpTraceUtilsTest.java
@@ -0,0 +1,1024 @@
+package com.wavefront.agent.listeners.otlp;
+
+import static com.wavefront.agent.listeners.otlp.OtlpGrpcTraceHandlerTest.emptyStreamObserver;
+import static com.wavefront.agent.listeners.otlp.OtlpTestHelpers.assertWFSpanEquals;
+import static com.wavefront.agent.listeners.otlp.OtlpTestHelpers.attribute;
+import static com.wavefront.agent.listeners.otlp.OtlpTestHelpers.hasKey;
+import static com.wavefront.agent.listeners.otlp.OtlpTestHelpers.parentSpanIdPair;
+import static com.wavefront.agent.listeners.otlp.OtlpTraceUtils.OTEL_STATUS_DESCRIPTION_KEY;
+import static com.wavefront.agent.listeners.otlp.OtlpTraceUtils.transformAll;
+import static com.wavefront.internal.SpanDerivedMetricsUtils.ERROR_SPAN_TAG_VAL;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.ERROR_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.NULL_TAG_VAL;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.captureBoolean;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.newCapture;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.protobuf.ByteString;
+import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.internal.SpanDerivedMetricsUtils;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.sdk.common.Pair;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.yammer.metrics.core.Counter;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.common.v1.AnyValue;
+import io.opentelemetry.proto.common.v1.ArrayValue;
+import io.opentelemetry.proto.common.v1.InstrumentationScope;
+import io.opentelemetry.proto.common.v1.KeyValue;
+import io.opentelemetry.proto.trace.v1.Span;
+import io.opentelemetry.proto.trace.v1.Status;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import wavefront.report.Annotation;
+import wavefront.report.SpanLogs;
+
+/**
+ * @author Xiaochen Wang (xiaochenw@vmware.com).
+ * @author Glenn Oppegard (goppegard@vmware.com).
+ */
+@RunWith(PowerMockRunner.class)
+@PowerMockIgnore({"javax.management.*"})
+@PrepareForTest({SpanDerivedMetricsUtils.class, OtlpTraceUtils.class})
+public class OtlpTraceUtilsTest {
+
+  private static final List emptyAttrs = Collections.unmodifiableList(new ArrayList<>());
+  public static final String SERVICE_NAME = "service.name";
+  private final SpanSampler mockSampler = EasyMock.createMock(SpanSampler.class);
+  private final WavefrontSender mockSender = EasyMock.createMock(WavefrontSender.class);
+
+  private final ReportableEntityHandler mockSpanHandler =
+      MockReportableEntityHandlerFactory.getMockTraceHandler();
+  private ReportableEntityHandler mockTraceLogsHandler =
+      MockReportableEntityHandlerFactory.getMockTraceSpanLogsHandler();
+  private final wavefront.report.Span wfMinimalSpan = OtlpTestHelpers.wfSpanGenerator(null).build();
+  private wavefront.report.Span actualSpan;
+
+  private static Map getWfAnnotationAsMap(List wfAnnotations) {
+    Map wfAnnotationAsMap = Maps.newHashMap();
+    for (Annotation annotation : wfAnnotations) {
+      wfAnnotationAsMap.put(annotation.getKey(), annotation.getValue());
+    }
+    return wfAnnotationAsMap;
+  }
+
+  @Before
+  public void setup() {
+    actualSpan = null;
+    EasyMock.reset(mockSampler, mockSpanHandler);
+  }
+
+  @Test
+  public void exportToWavefrontDoesNotReportIfPreprocessorFilteredSpan() {
+    // Arrange
+    ReportableEntityPreprocessor mockPreprocessor =
+        EasyMock.createMock(ReportableEntityPreprocessor.class);
+    ExportTraceServiceRequest otlpRequest =
+        OtlpTestHelpers.otlpTraceRequest(OtlpTestHelpers.otlpSpanGenerator().build());
+
+    PowerMock.mockStaticPartial(
+        OtlpTraceUtils.class, "fromOtlpRequest", "wasFilteredByPreprocessor");
+    EasyMock.expect(OtlpTraceUtils.fromOtlpRequest(otlpRequest, mockPreprocessor, "test-source"))
+        .andReturn(
+            Arrays.asList(new OtlpTraceUtils.WavefrontSpanAndLogs(wfMinimalSpan, new SpanLogs())));
+    EasyMock.expect(
+            OtlpTraceUtils.wasFilteredByPreprocessor(
+                eq(wfMinimalSpan), eq(mockSpanHandler), eq(mockPreprocessor)))
+        .andReturn(true);
+
+    replay(mockPreprocessor, mockSpanHandler);
+    PowerMock.replay(OtlpTraceUtils.class);
+
+    // Act
+    OtlpTraceUtils.exportToWavefront(
+        otlpRequest,
+        mockSpanHandler,
+        null,
+        () -> mockPreprocessor,
+        null,
+        null,
+        "test-source",
+        null,
+        null,
+        null);
+
+    // Assert
+    verify(mockPreprocessor, mockSpanHandler);
+    PowerMock.verify(OtlpTraceUtils.class);
+  }
+
+  @Test
+  public void exportToWavefrontReportsSpanIfSamplerReturnsTrue() {
+    // Arrange
+    Counter mockCounter = EasyMock.createMock(Counter.class);
+    Capture samplerCapture = EasyMock.newCapture();
+    EasyMock.expect(mockSampler.sample(capture(samplerCapture), eq(mockCounter))).andReturn(true);
+
+    Capture handlerCapture = EasyMock.newCapture();
+    mockSpanHandler.report(capture(handlerCapture));
+    expectLastCall();
+
+    PowerMock.mockStaticPartial(OtlpTraceUtils.class, "reportREDMetrics");
+    Pair, String> heartbeat = Pair.of(ImmutableMap.of("foo", "bar"), "src");
+    EasyMock.expect(OtlpTraceUtils.reportREDMetrics(anyObject(), anyObject(), anyObject()))
+        .andReturn(heartbeat);
+
+    replay(mockCounter, mockSampler, mockSpanHandler);
+    PowerMock.replay(OtlpTraceUtils.class);
+
+    // Act
+    ExportTraceServiceRequest otlpRequest =
+        OtlpTestHelpers.otlpTraceRequest(OtlpTestHelpers.otlpSpanGenerator().build());
+    Set, String>> discoveredHeartbeats = Sets.newConcurrentHashSet();
+
+    OtlpTraceUtils.exportToWavefront(
+        otlpRequest,
+        mockSpanHandler,
+        null,
+        null,
+        null,
+        Pair.of(mockSampler, mockCounter),
+        "test-source",
+        discoveredHeartbeats,
+        null,
+        null);
+
+    // Assert
+    verify(mockCounter, mockSampler, mockSpanHandler);
+    PowerMock.verify(OtlpTraceUtils.class);
+    assertEquals(samplerCapture.getValue(), handlerCapture.getValue());
+    assertTrue(discoveredHeartbeats.contains(heartbeat));
+  }
+
+  @Test
+  public void exportToWavefrontReportsREDMetricsEvenWhenSpanNotSampled() {
+    // Arrange
+    EasyMock.expect(mockSampler.sample(anyObject(), anyObject())).andReturn(false);
+
+    PowerMock.mockStaticPartial(OtlpTraceUtils.class, "reportREDMetrics");
+    Pair, String> heartbeat = Pair.of(ImmutableMap.of("foo", "bar"), "src");
+    EasyMock.expect(OtlpTraceUtils.reportREDMetrics(anyObject(), anyObject(), anyObject()))
+        .andReturn(heartbeat);
+
+    replay(mockSampler, mockSpanHandler);
+    PowerMock.replay(OtlpTraceUtils.class);
+
+    // Act
+    ExportTraceServiceRequest otlpRequest =
+        OtlpTestHelpers.otlpTraceRequest(OtlpTestHelpers.otlpSpanGenerator().build());
+    Set, String>> discoveredHeartbeats = Sets.newConcurrentHashSet();
+
+    OtlpTraceUtils.exportToWavefront(
+        otlpRequest,
+        mockSpanHandler,
+        null,
+        null,
+        null,
+        Pair.of(mockSampler, null),
+        "test-source",
+        discoveredHeartbeats,
+        null,
+        null);
+
+    // Assert
+    verify(mockSampler, mockSpanHandler);
+    PowerMock.verify(OtlpTraceUtils.class);
+    assertTrue(discoveredHeartbeats.contains(heartbeat));
+  }
+
+  @Test
+  public void testAnnotationsFromSimpleAttributes() {
+    KeyValue emptyAttr = KeyValue.newBuilder().setKey("empty").build();
+    KeyValue booleanAttr =
+        KeyValue.newBuilder()
+            .setKey("a-boolean")
+            .setValue(AnyValue.newBuilder().setBoolValue(true).build())
+            .build();
+    KeyValue stringAttr =
+        KeyValue.newBuilder()
+            .setKey("a-string")
+            .setValue(AnyValue.newBuilder().setStringValue("a-value").build())
+            .build();
+    KeyValue intAttr =
+        KeyValue.newBuilder()
+            .setKey("a-int")
+            .setValue(AnyValue.newBuilder().setIntValue(1234).build())
+            .build();
+    KeyValue doubleAttr =
+        KeyValue.newBuilder()
+            .setKey("a-double")
+            .setValue(AnyValue.newBuilder().setDoubleValue(2.1138).build())
+            .build();
+    KeyValue bytesAttr =
+        KeyValue.newBuilder()
+            .setKey("a-bytes")
+            .setValue(
+                AnyValue.newBuilder()
+                    .setBytesValue(ByteString.copyFromUtf8("any + old & data"))
+                    .build())
+            .build();
+    KeyValue noValueAttr =
+        KeyValue.newBuilder().setKey("no-value").setValue(AnyValue.newBuilder().build()).build();
+
+    List attributes =
+        Arrays.asList(
+            emptyAttr, booleanAttr, stringAttr, intAttr, doubleAttr, noValueAttr, bytesAttr);
+
+    List wfAnnotations = OtlpTraceUtils.annotationsFromAttributes(attributes);
+    Map wfAnnotationAsMap = getWfAnnotationAsMap(wfAnnotations);
+
+    assertEquals(attributes.size(), wfAnnotationAsMap.size());
+    assertEquals("", wfAnnotationAsMap.get("empty"));
+    assertEquals("true", wfAnnotationAsMap.get("a-boolean"));
+    assertEquals("a-value", wfAnnotationAsMap.get("a-string"));
+    assertEquals("1234", wfAnnotationAsMap.get("a-int"));
+    assertEquals("2.1138", wfAnnotationAsMap.get("a-double"));
+    assertEquals("YW55ICsgb2xkICYgZGF0YQ==", wfAnnotationAsMap.get("a-bytes"));
+    assertEquals(
+        "",
+        wfAnnotationAsMap.get("no-value"));
+  }
+
+  @Test
+  public void testAnnotationsFromArrayAttributes() {
+    KeyValue intArrayAttr =
+        KeyValue.newBuilder()
+            .setKey("int-array")
+            .setValue(
+                AnyValue.newBuilder()
+                    .setArrayValue(
+                        ArrayValue.newBuilder()
+                            .addAllValues(
+                                Arrays.asList(
+                                    AnyValue.newBuilder().setIntValue(-1).build(),
+                                    AnyValue.newBuilder().setIntValue(0).build(),
+                                    AnyValue.newBuilder().setIntValue(1).build()))
+                            .build())
+                    .build())
+            .build();
+
+    KeyValue boolArrayAttr =
+        KeyValue.newBuilder()
+            .setKey("bool-array")
+            .setValue(
+                AnyValue.newBuilder()
+                    .setArrayValue(
+                        ArrayValue.newBuilder()
+                            .addAllValues(
+                                Arrays.asList(
+                                    AnyValue.newBuilder().setBoolValue(true).build(),
+                                    AnyValue.newBuilder().setBoolValue(false).build(),
+                                    AnyValue.newBuilder().setBoolValue(true).build()))
+                            .build())
+                    .build())
+            .build();
+
+    KeyValue dblArrayAttr =
+        KeyValue.newBuilder()
+            .setKey("dbl-array")
+            .setValue(
+                AnyValue.newBuilder()
+                    .setArrayValue(
+                        ArrayValue.newBuilder()
+                            .addAllValues(
+                                Arrays.asList(
+                                    AnyValue.newBuilder().setDoubleValue(-3.14).build(),
+                                    AnyValue.newBuilder().setDoubleValue(0.0).build(),
+                                    AnyValue.newBuilder().setDoubleValue(3.14).build()))
+                            .build())
+                    .build())
+            .build();
+
+    List attributes = Arrays.asList(intArrayAttr, boolArrayAttr, dblArrayAttr);
+
+    List wfAnnotations = OtlpTraceUtils.annotationsFromAttributes(attributes);
+    Map wfAnnotationAsMap = getWfAnnotationAsMap(wfAnnotations);
+
+    assertEquals("[-1, 0, 1]", wfAnnotationAsMap.get("int-array"));
+    assertEquals("[true, false, true]", wfAnnotationAsMap.get("bool-array"));
+    assertEquals("[-3.14, 0.0, 3.14]", wfAnnotationAsMap.get("dbl-array"));
+  }
+
+  @Test
+  public void handlesSpecialCaseAnnotations() {
+    /*
+    A `source` tag at the span-level will override an explicit source that is set via
+    `wfSpanBuilder.setSource(...)`, which arguably seems like a bug. Since we determine the WF
+    source in `sourceAndResourceAttrs()`, rename any remaining OTLP Attribute to `_source`.
+    */
+    List attrs = Collections.singletonList(attribute("source", "a-source"));
+
+    List actual = OtlpTraceUtils.annotationsFromAttributes(attrs);
+
+    assertThat(actual, hasItem(new Annotation("_source", "a-source")));
+  }
+
+  @Test
+  public void testRequiredTags() {
+    List wfAnnotations = OtlpTraceUtils.setRequiredTags(Collections.emptyList());
+    Map annotations = getWfAnnotationAsMap(wfAnnotations);
+
+    assertEquals(4, wfAnnotations.size());
+    assertFalse(annotations.containsKey(SERVICE_NAME));
+    assertEquals("defaultApplication", annotations.get(APPLICATION_TAG_KEY));
+    assertEquals("defaultService", annotations.get(SERVICE_TAG_KEY));
+    assertEquals(NULL_TAG_VAL, annotations.get(CLUSTER_TAG_KEY));
+    assertEquals(NULL_TAG_VAL, annotations.get(SHARD_TAG_KEY));
+  }
+
+  @Test
+  public void testSetRequiredTagsOtlpServiceNameTagIsUsed() {
+    Annotation serviceName =
+        Annotation.newBuilder().setKey(SERVICE_NAME).setValue("a-service").build();
+
+    List wfAnnotations =
+        OtlpTraceUtils.setRequiredTags(Collections.singletonList(serviceName));
+    Map annotations = getWfAnnotationAsMap(wfAnnotations);
+
+    assertFalse(annotations.containsKey(SERVICE_NAME));
+    assertEquals("a-service", annotations.get(SERVICE_TAG_KEY));
+  }
+
+  @Test
+  public void testSetRequireTagsOtlpServiceNameTagIsDroppedIfServiceIsSet() {
+    Annotation serviceName =
+        Annotation.newBuilder().setKey(SERVICE_NAME).setValue("otlp-service").build();
+    Annotation wfService =
+        Annotation.newBuilder().setKey(SERVICE_TAG_KEY).setValue("wf-service").build();
+
+    List wfAnnotations =
+        OtlpTraceUtils.setRequiredTags(Arrays.asList(serviceName, wfService));
+    Map annotations = getWfAnnotationAsMap(wfAnnotations);
+
+    assertFalse(annotations.containsKey(SERVICE_NAME));
+    assertEquals("wf-service", annotations.get(SERVICE_TAG_KEY));
+  }
+
+  @Test
+  public void testSetRequiredTagsDeduplicatesAnnotations() {
+    Annotation.Builder dupeBuilder = Annotation.newBuilder().setKey("shared-key");
+    Annotation first = dupeBuilder.setValue("first").build();
+    Annotation middle = dupeBuilder.setValue("middle").build();
+    Annotation last = dupeBuilder.setValue("last").build();
+    List duplicates = Arrays.asList(first, middle, last);
+
+    List actual = OtlpTraceUtils.setRequiredTags(duplicates);
+
+    // We care that the last item "wins" and is preserved when de-duping
+    assertThat(actual, hasItem(last));
+    assertThat(actual, not(hasItems(first, middle)));
+  }
+
+  @Test
+  public void transformSpanHandlesMinimalSpan() {
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().build();
+    wavefront.report.Span expectedSpan = OtlpTestHelpers.wfSpanGenerator(null).build();
+
+    actualSpan = OtlpTraceUtils.transformSpan(otlpSpan, emptyAttrs, null, null, "test-source");
+
+    assertWFSpanEquals(expectedSpan, actualSpan);
+  }
+
+  @Test
+  public void transformSpanHandlesZeroDuration() {
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().setEndTimeUnixNano(0).build();
+    wavefront.report.Span expectedSpan =
+        OtlpTestHelpers.wfSpanGenerator(null).setDuration(0).build();
+
+    actualSpan = OtlpTraceUtils.transformSpan(otlpSpan, emptyAttrs, null, null, "test-source");
+
+    assertWFSpanEquals(expectedSpan, actualSpan);
+  }
+
+  @Test
+  public void transformSpanHandlesSpanAttributes() {
+    Pair parentSpanIdPair = parentSpanIdPair();
+    KeyValue booleanAttr =
+        KeyValue.newBuilder()
+            .setKey("a-boolean")
+            .setValue(AnyValue.newBuilder().setBoolValue(true).build())
+            .build();
+    Span otlpSpan =
+        OtlpTestHelpers.otlpSpanGenerator()
+            .addAttributes(booleanAttr)
+            .setParentSpanId(parentSpanIdPair._1)
+            .build();
+    List wfAttrs =
+        Arrays.asList(
+            Annotation.newBuilder().setKey("parent").setValue(parentSpanIdPair._2).build(),
+            Annotation.newBuilder().setKey("a-boolean").setValue("true").build());
+    wavefront.report.Span expectedSpan = OtlpTestHelpers.wfSpanGenerator(wfAttrs).build();
+
+    actualSpan = OtlpTraceUtils.transformSpan(otlpSpan, emptyAttrs, null, null, "test-source");
+
+    assertWFSpanEquals(expectedSpan, actualSpan);
+  }
+
+  @Test
+  public void transformSpanConvertsResourceAttributesToAnnotations() {
+    List resourceAttrs = Collections.singletonList(attribute("r-key", "r-value"));
+    wavefront.report.Span expectedSpan =
+        OtlpTestHelpers.wfSpanGenerator(
+                Collections.singletonList(new Annotation("r-key", "r-value")))
+            .build();
+
+    actualSpan =
+        OtlpTraceUtils.transformSpan(
+            OtlpTestHelpers.otlpSpanGenerator().build(), resourceAttrs, null, null, "test-source");
+
+    assertWFSpanEquals(expectedSpan, actualSpan);
+  }
+
+  @Test
+  public void transformSpanGivesSpanAttributesHigherPrecedenceThanResourceAttributes() {
+    String key = "the-key";
+    Span otlpSpan =
+        OtlpTestHelpers.otlpSpanGenerator().addAttributes(attribute(key, "span-value")).build();
+    List resourceAttrs = Collections.singletonList(attribute(key, "rsrc-value"));
+
+    actualSpan = OtlpTraceUtils.transformSpan(otlpSpan, resourceAttrs, null, null, "test-source");
+
+    assertThat(actualSpan.getAnnotations(), not(hasItem(new Annotation(key, "rsrc-value"))));
+    assertThat(actualSpan.getAnnotations(), hasItem(new Annotation(key, "span-value")));
+  }
+
+  @Test
+  public void transformSpanHandlesInstrumentationScope() {
+    InstrumentationScope scope =
+        InstrumentationScope.newBuilder().setName("grpc").setVersion("1.0").build();
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().build();
+
+    actualSpan = OtlpTraceUtils.transformSpan(otlpSpan, emptyAttrs, scope, null, "test-source");
+
+    assertThat(actualSpan.getAnnotations(), hasItem(new Annotation("otel.scope.name", "grpc")));
+    assertThat(actualSpan.getAnnotations(), hasItem(new Annotation("otel.scope.version", "1.0")));
+  }
+
+  @Test
+  public void transformSpanAddsDroppedCountTags() {
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().setDroppedEventsCount(1).build();
+
+    actualSpan = OtlpTraceUtils.transformSpan(otlpSpan, emptyAttrs, null, null, "test-source");
+
+    assertThat(
+        actualSpan.getAnnotations(), hasItem(new Annotation("otel.dropped_events_count", "1")));
+  }
+
+  @Test
+  public void transformSpanAppliesPreprocessorRules() {
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().build();
+    List wfAttrs =
+        Collections.singletonList(
+            Annotation.newBuilder().setKey("my-key").setValue("my-value").build());
+    ReportableEntityPreprocessor preprocessor =
+        OtlpTestHelpers.addTagIfNotExistsPreprocessor(wfAttrs);
+    wavefront.report.Span expectedSpan = OtlpTestHelpers.wfSpanGenerator(wfAttrs).build();
+
+    actualSpan =
+        OtlpTraceUtils.transformSpan(otlpSpan, emptyAttrs, null, preprocessor, "test-source");
+
+    assertWFSpanEquals(expectedSpan, actualSpan);
+  }
+
+  @Test
+  public void transformSpanAppliesPreprocessorBeforeSettingRequiredTags() {
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().build();
+    List wfAttrs =
+        Collections.singletonList(
+            Annotation.newBuilder().setKey(APPLICATION_TAG_KEY).setValue("an-app").build());
+    ReportableEntityPreprocessor preprocessor =
+        OtlpTestHelpers.addTagIfNotExistsPreprocessor(wfAttrs);
+    wavefront.report.Span expectedSpan = OtlpTestHelpers.wfSpanGenerator(wfAttrs).build();
+
+    actualSpan =
+        OtlpTraceUtils.transformSpan(otlpSpan, emptyAttrs, null, preprocessor, "test-source");
+
+    assertWFSpanEquals(expectedSpan, actualSpan);
+  }
+
+  @Test
+  public void transformSpanTranslatesSpanKindToAnnotation() {
+    wavefront.report.Span clientSpan =
+        OtlpTraceUtils.transformSpan(
+            OtlpTestHelpers.otlpSpanWithKind(Span.SpanKind.SPAN_KIND_CLIENT),
+            emptyAttrs,
+            null,
+            null,
+            "test-source");
+    assertThat(clientSpan.getAnnotations(), hasItem(new Annotation("span.kind", "client")));
+
+    wavefront.report.Span consumerSpan =
+        OtlpTraceUtils.transformSpan(
+            OtlpTestHelpers.otlpSpanWithKind(Span.SpanKind.SPAN_KIND_CONSUMER),
+            emptyAttrs,
+            null,
+            null,
+            "test-source");
+    assertThat(consumerSpan.getAnnotations(), hasItem(new Annotation("span.kind", "consumer")));
+
+    wavefront.report.Span internalSpan =
+        OtlpTraceUtils.transformSpan(
+            OtlpTestHelpers.otlpSpanWithKind(Span.SpanKind.SPAN_KIND_INTERNAL),
+            emptyAttrs,
+            null,
+            null,
+            "test-source");
+    assertThat(internalSpan.getAnnotations(), hasItem(new Annotation("span.kind", "internal")));
+
+    wavefront.report.Span producerSpan =
+        OtlpTraceUtils.transformSpan(
+            OtlpTestHelpers.otlpSpanWithKind(Span.SpanKind.SPAN_KIND_PRODUCER),
+            emptyAttrs,
+            null,
+            null,
+            "test-source");
+    assertThat(producerSpan.getAnnotations(), hasItem(new Annotation("span.kind", "producer")));
+
+    wavefront.report.Span serverSpan =
+        OtlpTraceUtils.transformSpan(
+            OtlpTestHelpers.otlpSpanWithKind(Span.SpanKind.SPAN_KIND_SERVER),
+            emptyAttrs,
+            null,
+            null,
+            "test-source");
+    assertThat(serverSpan.getAnnotations(), hasItem(new Annotation("span.kind", "server")));
+
+    wavefront.report.Span unspecifiedSpan =
+        OtlpTraceUtils.transformSpan(
+            OtlpTestHelpers.otlpSpanWithKind(Span.SpanKind.SPAN_KIND_UNSPECIFIED),
+            emptyAttrs,
+            null,
+            null,
+            "test-source");
+    assertThat(
+        unspecifiedSpan.getAnnotations(), hasItem(new Annotation("span.kind", "unspecified")));
+
+    wavefront.report.Span noKindSpan =
+        OtlpTraceUtils.transformSpan(
+            OtlpTestHelpers.otlpSpanGenerator().build(), emptyAttrs, null, null, "test-source");
+    assertThat(noKindSpan.getAnnotations(), hasItem(new Annotation("span.kind", "unspecified")));
+  }
+
+  @Test
+  public void transformSpanHandlesSpanStatusIfError() {
+    // Error Status without Message
+    Span errorSpan = OtlpTestHelpers.otlpSpanWithStatus(Status.StatusCode.STATUS_CODE_ERROR, "");
+
+    actualSpan = OtlpTraceUtils.transformSpan(errorSpan, emptyAttrs, null, null, "test-source");
+
+    assertThat(
+        actualSpan.getAnnotations(), hasItem(new Annotation(ERROR_TAG_KEY, ERROR_SPAN_TAG_VAL)));
+    assertThat(actualSpan.getAnnotations(), not(hasKey(OTEL_STATUS_DESCRIPTION_KEY)));
+
+    // Error Status with Message
+    Span errorSpanWithMessage =
+        OtlpTestHelpers.otlpSpanWithStatus(Status.StatusCode.STATUS_CODE_ERROR, "a description");
+
+    actualSpan =
+        OtlpTraceUtils.transformSpan(errorSpanWithMessage, emptyAttrs, null, null, "test-source");
+
+    assertThat(
+        actualSpan.getAnnotations(), hasItem(new Annotation(ERROR_TAG_KEY, ERROR_SPAN_TAG_VAL)));
+    assertThat(
+        actualSpan.getAnnotations(),
+        hasItem(new Annotation(OTEL_STATUS_DESCRIPTION_KEY, "a description")));
+  }
+
+  @Test
+  public void transformSpanIgnoresSpanStatusIfNotError() {
+    // Ok Status
+    Span okSpan = OtlpTestHelpers.otlpSpanWithStatus(Status.StatusCode.STATUS_CODE_OK, "");
+
+    actualSpan = OtlpTraceUtils.transformSpan(okSpan, emptyAttrs, null, null, "test-source");
+
+    assertThat(actualSpan.getAnnotations(), not(hasKey(ERROR_TAG_KEY)));
+    assertThat(actualSpan.getAnnotations(), not(hasKey(OTEL_STATUS_DESCRIPTION_KEY)));
+
+    // Unset Status
+    Span unsetSpan = OtlpTestHelpers.otlpSpanWithStatus(Status.StatusCode.STATUS_CODE_UNSET, "");
+
+    actualSpan = OtlpTraceUtils.transformSpan(unsetSpan, emptyAttrs, null, null, "test-source");
+
+    assertThat(actualSpan.getAnnotations(), not(hasKey(ERROR_TAG_KEY)));
+    assertThat(actualSpan.getAnnotations(), not(hasKey(OTEL_STATUS_DESCRIPTION_KEY)));
+  }
+
+  @Test
+  public void transformSpanSetsSourceFromResourceAttributesNotSpanAttributes() {
+    List resourceAttrs = Collections.singletonList(attribute("source", "a-src"));
+    Span otlpSpan =
+        OtlpTestHelpers.otlpSpanGenerator()
+            .addAttributes(attribute("source", "span-level"))
+            .build();
+
+    actualSpan = OtlpTraceUtils.transformSpan(otlpSpan, resourceAttrs, null, null, "ignored");
+
+    assertEquals("a-src", actualSpan.getSource());
+    assertThat(actualSpan.getAnnotations(), not(hasItem(new Annotation("source", "a-src"))));
+    assertThat(actualSpan.getAnnotations(), hasItem(new Annotation("_source", "span-level")));
+  }
+
+  @Test
+  public void transformSpanUsesDefaultSourceWhenNoAttributesMatch() {
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().build();
+
+    actualSpan = OtlpTraceUtils.transformSpan(otlpSpan, emptyAttrs, null, null, "defaultSource");
+
+    assertEquals("defaultSource", actualSpan.getSource());
+  }
+
+  @Test
+  public void transformSpanHandlesTraceState() {
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().setTraceState("key=val").build();
+
+    actualSpan = OtlpTraceUtils.transformSpan(otlpSpan, emptyAttrs, null, null, "defaultSource");
+
+    assertThat(actualSpan.getAnnotations(), hasItem(new Annotation("w3c.tracestate", "key=val")));
+  }
+
+  @Test
+  public void transformEventsConvertsToWFSpanLogs() {
+    int droppedAttrsCount = 1;
+    Span.Event otlpEvent = OtlpTestHelpers.otlpSpanEvent(droppedAttrsCount);
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().addEvents(otlpEvent).build();
+
+    wavefront.report.SpanLogs expected =
+        OtlpTestHelpers.wfSpanLogsGenerator(wfMinimalSpan, droppedAttrsCount).build();
+
+    SpanLogs actual = OtlpTraceUtils.transformEvents(otlpSpan, wfMinimalSpan);
+
+    assertEquals(expected, actual);
+  }
+
+  @Test
+  public void transformEventsDoesNotReturnNullWhenGivenZeroOTLPEvents() {
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().build();
+    assertEquals(0, otlpSpan.getEventsCount());
+
+    SpanLogs actual = OtlpTraceUtils.transformEvents(otlpSpan, wfMinimalSpan);
+
+    assertNotNull(actual);
+    assertEquals(0, actual.getLogs().size());
+  }
+
+  @Test
+  public void transformAllSetsAttributeWhenOtlpEventsExists() {
+    Span.Event otlpEvent = OtlpTestHelpers.otlpSpanEvent(0);
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().addEvents(otlpEvent).build();
+
+    OtlpTraceUtils.WavefrontSpanAndLogs actual =
+        transformAll(otlpSpan, emptyAttrs, null, null, "test-source");
+
+    assertThat(actual.getSpan().getAnnotations(), hasKey("_spanLogs"));
+    assertThat(actual.getSpanLogs().getLogs(), not(empty()));
+  }
+
+  @Test
+  public void transformAllDoesNotSetAttributeWhenNoOtlpEventsExists() {
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().build();
+    assertThat(otlpSpan.getEventsList(), empty());
+
+    OtlpTraceUtils.WavefrontSpanAndLogs actual =
+        transformAll(otlpSpan, emptyAttrs, null, null, "test-source");
+
+    assertThat(actual.getSpan().getAnnotations(), not(hasKey("_spanLogs")));
+    assertThat(actual.getSpanLogs().getLogs(), empty());
+  }
+
+  @Test
+  public void wasFilteredByPreprocessorHandlesNullPreprocessor() {
+    ReportableEntityPreprocessor preprocessor = null;
+
+    assertFalse(
+        OtlpTraceUtils.wasFilteredByPreprocessor(wfMinimalSpan, mockSpanHandler, preprocessor));
+  }
+
+  @Test
+  public void wasFilteredByPreprocessorCanReject() {
+    ReportableEntityPreprocessor preprocessor = OtlpTestHelpers.rejectSpanPreprocessor();
+    mockSpanHandler.reject(wfMinimalSpan, "span rejected for testing purpose");
+    expectLastCall();
+    replay(mockSpanHandler);
+
+    assertTrue(
+        OtlpTraceUtils.wasFilteredByPreprocessor(wfMinimalSpan, mockSpanHandler, preprocessor));
+    verify(mockSpanHandler);
+  }
+
+  @Test
+  public void wasFilteredByPreprocessorCanBlock() {
+    ReportableEntityPreprocessor preprocessor = OtlpTestHelpers.blockSpanPreprocessor();
+    mockSpanHandler.block(wfMinimalSpan);
+    expectLastCall();
+    replay(mockSpanHandler);
+
+    assertTrue(
+        OtlpTraceUtils.wasFilteredByPreprocessor(wfMinimalSpan, mockSpanHandler, preprocessor));
+    verify(mockSpanHandler);
+  }
+
+  @Test
+  public void sourceFromAttributesSetsSourceAccordingToPrecedenceRules() {
+    Pair> actual;
+
+    // "source" attribute has highest precedence
+    actual =
+        OtlpTraceUtils.sourceFromAttributes(
+            Arrays.asList(
+                attribute("hostname", "a-hostname"),
+                attribute("host.id", "a-host.id"),
+                attribute("source", "a-src"),
+                attribute("host.name", "a-host.name")),
+            "ignore");
+    assertEquals("a-src", actual._1);
+
+    // "host.name" next highest
+    actual =
+        OtlpTraceUtils.sourceFromAttributes(
+            Arrays.asList(
+                attribute("hostname", "a-hostname"),
+                attribute("host.id", "a-host.id"),
+                attribute("host.name", "a-host.name")),
+            "ignore");
+    assertEquals("a-host.name", actual._1);
+
+    // "hostname" next highest
+    actual =
+        OtlpTraceUtils.sourceFromAttributes(
+            Arrays.asList(attribute("hostname", "a-hostname"), attribute("host.id", "a-host.id")),
+            "ignore");
+    assertEquals("a-hostname", actual._1);
+
+    // "host.id" has lowest precedence
+    actual =
+        OtlpTraceUtils.sourceFromAttributes(
+            Arrays.asList(attribute("host.id", "a-host.id")), "ignore");
+    assertEquals("a-host.id", actual._1);
+  }
+
+  @Test
+  public void sourceFromAttributesUsesDefaultWhenNoCandidateExists() {
+    Pair> actual =
+        OtlpTraceUtils.sourceFromAttributes(emptyAttrs, "a-default");
+
+    assertEquals("a-default", actual._1);
+    assertEquals(emptyAttrs, actual._2);
+  }
+
+  @Test
+  public void sourceFromAttributesDeletesCandidateUsedAsSource() {
+    List attrs =
+        Arrays.asList(
+            attribute("hostname", "a-hostname"),
+            attribute("some-key", "some-val"),
+            attribute("host.id", "a-host.id"));
+
+    Pair> actual = OtlpTraceUtils.sourceFromAttributes(attrs, "ignore");
+
+    assertEquals("a-hostname", actual._1);
+
+    List expectedAttrs =
+        Arrays.asList(attribute("some-key", "some-val"), attribute("host.id", "a-host.id"));
+    assertEquals(expectedAttrs, actual._2);
+  }
+
+  @Test
+  public void reportREDMetricsCallsDerivedMetricsUtils() {
+    PowerMock.mockStatic(SpanDerivedMetricsUtils.class);
+    WavefrontInternalReporter mockInternalReporter =
+        EasyMock.niceMock(WavefrontInternalReporter.class);
+
+    List wfAttrs =
+        Arrays.asList(
+            new Annotation(APPLICATION_TAG_KEY, "app"),
+            new Annotation(SERVICE_TAG_KEY, "svc"),
+            new Annotation(CLUSTER_TAG_KEY, "east1"),
+            new Annotation(SHARD_TAG_KEY, "az1"),
+            new Annotation(COMPONENT_TAG_KEY, "comp"),
+            new Annotation(ERROR_TAG_KEY, "true"));
+    wavefront.report.Span wfSpan = OtlpTestHelpers.wfSpanGenerator(wfAttrs).build();
+
+    List> spanTags =
+        wfSpan.getAnnotations().stream()
+            .map(a -> Pair.of(a.getKey(), a.getValue()))
+            .collect(Collectors.toList());
+    HashSet customKeys = Sets.newHashSet("a", "b", "c");
+    Pair, String> mockReturn = Pair.of(ImmutableMap.of("key", "val"), "foo");
+
+    EasyMock.expect(
+            SpanDerivedMetricsUtils.reportWavefrontGeneratedData(
+                eq(mockInternalReporter),
+                eq("root"),
+                eq("app"),
+                eq("svc"),
+                eq("east1"),
+                eq("az1"),
+                eq("test-source"),
+                eq("comp"),
+                eq(true),
+                eq(TimeUnit.MILLISECONDS.toMicros(wfSpan.getDuration())),
+                eq(customKeys),
+                eq(spanTags)))
+        .andReturn(mockReturn);
+    PowerMock.replay(SpanDerivedMetricsUtils.class);
+
+    Pair, String> actual =
+        OtlpTraceUtils.reportREDMetrics(wfSpan, mockInternalReporter, customKeys);
+
+    assertEquals(mockReturn, actual);
+    PowerMock.verify(SpanDerivedMetricsUtils.class);
+  }
+
+  @Test
+  public void reportREDMetricsProvidesDefaults() {
+    PowerMock.mockStatic(SpanDerivedMetricsUtils.class);
+
+    Capture isError = EasyMock.newCapture();
+    Capture componentTag = EasyMock.newCapture();
+    EasyMock.expect(
+            SpanDerivedMetricsUtils.reportWavefrontGeneratedData(
+                anyObject(),
+                anyObject(),
+                anyObject(),
+                anyObject(),
+                anyObject(),
+                anyObject(),
+                anyObject(),
+                capture(componentTag),
+                captureBoolean(isError),
+                anyLong(),
+                anyObject(),
+                anyObject()))
+        .andReturn(Pair.of(null, null));
+    PowerMock.replay(SpanDerivedMetricsUtils.class);
+
+    assertThat(wfMinimalSpan.getAnnotations(), not(hasKey(ERROR_TAG_KEY)));
+    assertThat(wfMinimalSpan.getAnnotations(), not(hasKey(COMPONENT_TAG_KEY)));
+    OtlpTraceUtils.reportREDMetrics(wfMinimalSpan, null, null);
+
+    assertFalse(isError.getValue());
+    assertEquals(NULL_TAG_VAL, componentTag.getValue());
+    PowerMock.verify(SpanDerivedMetricsUtils.class);
+  }
+
+  @Test
+  public void exportToWavefrontWithSpanLine() {
+    // Arrange
+    Counter mockCounter = EasyMock.createMock(Counter.class);
+
+    expect(mockSampler.sample(anyObject(), anyObject())).andReturn(true);
+
+    Capture spanCapture = newCapture();
+    mockSpanHandler.report(capture(spanCapture));
+    expectLastCall();
+
+    Capture spanLogsCapture = newCapture();
+    mockTraceLogsHandler.report(capture(spanLogsCapture));
+    expectLastCall();
+
+    replay(mockCounter, mockSampler, mockSpanHandler, mockTraceLogsHandler);
+    PowerMock.replay(OtlpTraceUtils.class);
+
+    Span.Event otlpEvent = OtlpTestHelpers.otlpSpanEvent(0);
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().addEvents(otlpEvent).build();
+    ExportTraceServiceRequest otlpRequest = OtlpTestHelpers.otlpTraceRequest(otlpSpan);
+
+    // Act
+    OtlpGrpcTraceHandler otlpGrpcTraceHandler =
+        new OtlpGrpcTraceHandler(
+            "9876",
+            mockSpanHandler,
+            mockTraceLogsHandler,
+            mockSender,
+            null,
+            mockSampler,
+            () -> false,
+            () -> false,
+            "test-source",
+            null);
+    otlpGrpcTraceHandler.export(otlpRequest, emptyStreamObserver);
+    otlpGrpcTraceHandler.run();
+    otlpGrpcTraceHandler.close();
+
+    // Assert
+    verify(mockCounter, mockSampler, mockSpanHandler);
+    PowerMock.verify(OtlpTraceUtils.class);
+    assertFalse(spanCapture.getValue().getAnnotations().contains("_sampledByPolicy"));
+    assertEquals("_sampledByPolicy=NONE", spanLogsCapture.getValue().getSpan());
+  }
+
+  @Test
+  public void annotationsFromInstrumentationScopeWithNullOrEmptyScope() {
+    assertEquals(Collections.emptyList(), OtlpTraceUtils.annotationsFromInstrumentationScope(null));
+
+    InstrumentationScope emptyScope = InstrumentationScope.newBuilder().build();
+    assertEquals(
+        Collections.emptyList(), OtlpTraceUtils.annotationsFromInstrumentationScope(emptyScope));
+  }
+
+  @Test
+  public void annotationsFromInstrumentationScopeWithScopeData() {
+    InstrumentationScope scope = InstrumentationScope.newBuilder().setName("net/http").build();
+
+    assertEquals(
+        Collections.singletonList(new Annotation("otel.scope.name", "net/http")),
+        OtlpTraceUtils.annotationsFromInstrumentationScope(scope));
+
+    scope = scope.toBuilder().setVersion("1.0.0").build();
+
+    assertEquals(
+        Arrays.asList(
+            new Annotation("otel.scope.name", "net/http"),
+            new Annotation("otel.scope.version", "1.0.0")),
+        OtlpTraceUtils.annotationsFromInstrumentationScope(scope));
+  }
+
+  @Test
+  public void annotationsFromDroppedCountsWithZeroValues() {
+    Span otlpSpan = OtlpTestHelpers.otlpSpanGenerator().build();
+
+    assertEquals(0, otlpSpan.getDroppedAttributesCount());
+    assertEquals(0, otlpSpan.getDroppedEventsCount());
+    assertEquals(0, otlpSpan.getDroppedLinksCount());
+
+    assertThat(OtlpTraceUtils.annotationsFromDroppedCounts(otlpSpan), empty());
+  }
+
+  @Test
+  public void annotationsFromDroppedCountsWithNonZeroValues() {
+    Span otlpSpan =
+        OtlpTestHelpers.otlpSpanGenerator()
+            .setDroppedAttributesCount(1)
+            .setDroppedEventsCount(2)
+            .setDroppedLinksCount(3)
+            .build();
+
+    List actual = OtlpTraceUtils.annotationsFromDroppedCounts(otlpSpan);
+    assertThat(actual, hasSize(3));
+    assertThat(actual, hasItem(new Annotation("otel.dropped_attributes_count", "1")));
+    assertThat(actual, hasItem(new Annotation("otel.dropped_events_count", "2")));
+    assertThat(actual, hasItem(new Annotation("otel.dropped_links_count", "3")));
+  }
+
+  @Test
+  public void shouldReportSpanLogsFalseIfZeroLogs() {
+    assertFalse(OtlpTraceUtils.shouldReportSpanLogs(0, null));
+  }
+
+  @Test
+  public void shouldReportSpanLogsFalseIfNonZeroLogsAndFeatureDisabled() {
+    Supplier spanLogsFeatureDisabled = () -> true;
+    assertFalse(OtlpTraceUtils.shouldReportSpanLogs(1, Pair.of(spanLogsFeatureDisabled, null)));
+  }
+
+  @Test
+  public void shouldReportSpanLogsTrueIfNonZeroLogsAndFeatureEnabled() {
+    Supplier spanLogsFeatureDisabled = () -> false;
+    assertTrue(OtlpTraceUtils.shouldReportSpanLogs(1, Pair.of(spanLogsFeatureDisabled, null)));
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/tracing/CustomTracingPortUnificationHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/CustomTracingPortUnificationHandlerTest.java
new file mode 100644
index 000000000..42d730655
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/CustomTracingPortUnificationHandlerTest.java
@@ -0,0 +1,59 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.captureLong;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.newCapture;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.internal.reporter.WavefrontInternalReporter;
+import com.wavefront.internal_reporter_java.io.dropwizard.metrics5.DeltaCounter;
+import com.wavefront.internal_reporter_java.io.dropwizard.metrics5.WavefrontHistogram;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Test;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+
+public class CustomTracingPortUnificationHandlerTest {
+  @Test
+  public void reportsCorrectDuration() {
+    WavefrontInternalReporter reporter = EasyMock.niceMock(WavefrontInternalReporter.class);
+    expect(reporter.newDeltaCounter(anyObject())).andReturn(new DeltaCounter()).anyTimes();
+    WavefrontHistogram histogram = EasyMock.niceMock(WavefrontHistogram.class);
+    expect(reporter.newWavefrontHistogram(anyObject(), anyObject())).andReturn(histogram);
+    Capture duration = newCapture();
+    histogram.update(captureLong(duration));
+    expectLastCall();
+    ReportableEntityHandler handler =
+        MockReportableEntityHandlerFactory.getMockTraceHandler();
+    CustomTracingPortUnificationHandler subject =
+        new CustomTracingPortUnificationHandler(
+            null, null, null, null, null, null, handler, null, null, null, null, null, reporter,
+            null, null, null);
+    replay(reporter, histogram);
+
+    Span span = getSpan();
+    span.setDuration(1000); // milliseconds
+    subject.report(span);
+    verify(reporter, histogram);
+    long value = duration.getValue();
+    assertEquals(1000000, value); // microseconds
+  }
+
+  @NotNull
+  private Span getSpan() {
+    Span span = new Span();
+    span.setAnnotations(
+        ImmutableList.of(
+            new Annotation("application", "application"), new Annotation("service", "service")));
+    return span;
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/tracing/JaegerGrpcCollectorHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/JaegerGrpcCollectorHandlerTest.java
new file mode 100644
index 000000000..f6fbcfd5e
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/JaegerGrpcCollectorHandlerTest.java
@@ -0,0 +1,1718 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.google.protobuf.util.Timestamps.fromMillis;
+import static com.wavefront.agent.TestUtils.verifyWithTimeout;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.HEART_BEAT_METRIC;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Duration;
+import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.PreprocessorRuleMetrics;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.preprocessor.SpanReplaceRegexTransformer;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.api.agent.SpanSamplingPolicy;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.entities.tracing.sampling.DurationSampler;
+import com.wavefront.sdk.entities.tracing.sampling.RateSampler;
+import io.grpc.stub.StreamObserver;
+import io.opentelemetry.exporter.jaeger.proto.api_v2.Collector;
+import io.opentelemetry.exporter.jaeger.proto.api_v2.Model;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.function.Supplier;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
+
+/**
+ * Unit tests for {@link JaegerGrpcCollectorHandler}
+ *
+ * @author Hao Song (songhao@vmware.com)
+ */
+public class JaegerGrpcCollectorHandlerTest {
+  private static final String DEFAULT_SOURCE = "jaeger";
+  private final ReportableEntityHandler mockTraceHandler =
+      MockReportableEntityHandlerFactory.getMockTraceHandler();
+  private final ReportableEntityHandler mockTraceLogsHandler =
+      MockReportableEntityHandlerFactory.getMockTraceSpanLogsHandler();
+  private final WavefrontSender mockWavefrontSender = EasyMock.createMock(WavefrontSender.class);
+  private final long startTime = System.currentTimeMillis();
+
+  // Derived RED metrics related.
+  private final String PREPROCESSED_APPLICATION_TAG_VALUE = "preprocessedApplication";
+  private final String PREPROCESSED_SERVICE_TAG_VALUE = "preprocessedService";
+  private final String PREPROCESSED_CLUSTER_TAG_VALUE = "preprocessedCluster";
+  private final String PREPROCESSED_SHARD_TAG_VALUE = "preprocessedShard";
+  private final String PREPROCESSED_SOURCE_VALUE = "preprocessedSource";
+
+  @Test
+  public void testJaegerGrpcCollector() throws Exception {
+
+    reset(mockTraceHandler, mockTraceLogsHandler);
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(1000)
+            .setName("HTTP GET")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("component", "db"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("_spanLogs", "true")))
+            .build());
+    expectLastCall();
+
+    mockTraceLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("default")
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(startTime * 1000)
+                        .setFields(
+                            ImmutableMap.of("event", "error", "exception", "NullPointerException"))
+                        .build()))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(2000)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("component", "db"),
+                    new Annotation("application", "Custom-JaegerApp"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(2000)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-9a12-b85901d53397")
+            .setTraceId("00000000-0000-0000-fea4-87ee36e58cab")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("parent", "00000000-0000-0000-fea4-87ee36e58cab")))
+            .build());
+    expectLastCall();
+
+    // Test filtering empty tags
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(2000)
+            .setName("HTTP GET /test")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-0051759bfc69")
+            .setTraceId("0000011e-ab2a-9944-0000-000049631900")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerGrpcCollectorHandler handler =
+        new JaegerGrpcCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Model.KeyValue ipTag =
+        Model.KeyValue.newBuilder()
+            .setKey("ip")
+            .setVStr("10.0.0.1")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue componentTag =
+        Model.KeyValue.newBuilder()
+            .setKey("component")
+            .setVStr("db")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customApplicationTag =
+        Model.KeyValue.newBuilder()
+            .setKey("application")
+            .setVStr("Custom-JaegerApp")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue emptyTag =
+        Model.KeyValue.newBuilder()
+            .setKey("empty")
+            .setVStr("")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1234567890L);
+    buffer.putLong(1234567890123L);
+    ByteString traceId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(0L);
+    buffer.putLong(-97803834702328661L);
+    ByteString trace3Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1231232342340L);
+    buffer.putLong(1231231232L);
+    ByteString trace4Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(1234567L);
+    ByteString span1Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(2345678L);
+    ByteString span2Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(-7344605349865507945L);
+    ByteString span3Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(-97803834702328661L);
+    ByteString span3ParentId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(349865507945L);
+    ByteString span4Id = ByteString.copyFrom(buffer.array());
+
+    Model.Span span1 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span1Id)
+            .setDuration(Duration.newBuilder().setSeconds(1L).build())
+            .setOperationName("HTTP GET")
+            .setStartTime(fromMillis(startTime))
+            .addTags(componentTag)
+            .addLogs(
+                Model.Log.newBuilder()
+                    .addFields(
+                        Model.KeyValue.newBuilder()
+                            .setKey("event")
+                            .setVStr("error")
+                            .setVType(Model.ValueType.STRING)
+                            .build())
+                    .addFields(
+                        Model.KeyValue.newBuilder()
+                            .setKey("exception")
+                            .setVStr("NullPointerException")
+                            .setVType(Model.ValueType.STRING)
+                            .build())
+                    .setTimestamp(fromMillis(startTime)))
+            .build();
+
+    Model.Span span2 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span2Id)
+            .setDuration(Duration.newBuilder().setSeconds(2L).build())
+            .setOperationName("HTTP GET /")
+            .setStartTime(fromMillis(startTime))
+            .addTags(componentTag)
+            .addTags(customApplicationTag)
+            .addReferences(
+                Model.SpanRef.newBuilder()
+                    .setRefType(Model.SpanRefType.CHILD_OF)
+                    .setSpanId(span1Id)
+                    .setTraceId(traceId)
+                    .build())
+            .build();
+
+    // check negative span IDs too
+    Model.Span span3 =
+        Model.Span.newBuilder()
+            .setTraceId(trace3Id)
+            .setSpanId(span3Id)
+            .setDuration(Duration.newBuilder().setSeconds(2L).build())
+            .setOperationName("HTTP GET /")
+            .setStartTime(fromMillis(startTime))
+            .addReferences(
+                Model.SpanRef.newBuilder()
+                    .setRefType(Model.SpanRefType.CHILD_OF)
+                    .setSpanId(span3ParentId)
+                    .setTraceId(traceId)
+                    .build())
+            .build();
+
+    Model.Span span4 =
+        Model.Span.newBuilder()
+            .setTraceId(trace4Id)
+            .setSpanId(span4Id)
+            .setDuration(Duration.newBuilder().setSeconds(2L).build())
+            .setOperationName("HTTP GET /test")
+            .setStartTime(fromMillis(startTime))
+            .addTags(emptyTag)
+            .build();
+
+    Model.Batch testBatch =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder().setServiceName("frontend").addTags(ipTag).build())
+            .addAllSpans(ImmutableList.of(span1, span2, span3, span4))
+            .build();
+
+    Collector.PostSpansRequest batches =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatch).build();
+
+    handler.postSpans(batches, emptyStreamObserver);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testApplicationTagPriority() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    // Span to verify span level tags precedence
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(1000)
+            .setName("HTTP GET")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("component", "db"),
+                    new Annotation("application", "SpanLevelAppTag"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    // Span to verify process level tags precedence
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(2000)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("component", "db"),
+                    new Annotation("application", "ProcessLevelAppTag"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687")))
+            .build());
+    expectLastCall();
+
+    // Span to verify Proxy level tags precedence
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(3000)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-9a12-b85901d53397")
+            .setTraceId("00000000-0000-0000-fea4-87ee36e58cab")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "ProxyLevelAppTag"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("parent", "00000000-0000-0000-fea4-87ee36e58cab")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    // Verify span level "application" tags precedence
+    JaegerGrpcCollectorHandler handler =
+        new JaegerGrpcCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            "ProxyLevelAppTag",
+            null);
+
+    Model.KeyValue ipTag =
+        Model.KeyValue.newBuilder()
+            .setKey("ip")
+            .setVStr("10.0.0.1")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue componentTag =
+        Model.KeyValue.newBuilder()
+            .setKey("component")
+            .setVStr("db")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue spanLevelAppTag =
+        Model.KeyValue.newBuilder()
+            .setKey("application")
+            .setVStr("SpanLevelAppTag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue processLevelAppTag =
+        Model.KeyValue.newBuilder()
+            .setKey("application")
+            .setVStr("ProcessLevelAppTag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1234567890L);
+    buffer.putLong(1234567890123L);
+    ByteString traceId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(0L);
+    buffer.putLong(-97803834702328661L);
+    ByteString trace2Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(1234567L);
+    ByteString span1Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(2345678L);
+    ByteString span2Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(-7344605349865507945L);
+    ByteString span3Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(-97803834702328661L);
+    ByteString span3ParentId = ByteString.copyFrom(buffer.array());
+
+    // Span1 to verify span level tags precedence
+    Model.Span span1 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span1Id)
+            .setDuration(Duration.newBuilder().setSeconds(1L).build())
+            .setOperationName("HTTP GET")
+            .setStartTime(fromMillis(startTime))
+            .addTags(componentTag)
+            .addTags(spanLevelAppTag)
+            .setFlags(1)
+            .build();
+
+    Model.Span span2 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span2Id)
+            .setDuration(Duration.newBuilder().setSeconds(2L).build())
+            .setOperationName("HTTP GET /")
+            .setStartTime(fromMillis(startTime))
+            .addTags(componentTag)
+            .setFlags(1)
+            .addReferences(
+                Model.SpanRef.newBuilder()
+                    .setRefType(Model.SpanRefType.CHILD_OF)
+                    .setSpanId(span1Id)
+                    .setTraceId(traceId)
+                    .build())
+            .build();
+
+    // check negative span IDs too
+    Model.Span span3 =
+        Model.Span.newBuilder()
+            .setTraceId(trace2Id)
+            .setSpanId(span3Id)
+            .setDuration(Duration.newBuilder().setSeconds(3L).build())
+            .setOperationName("HTTP GET /")
+            .setStartTime(fromMillis(startTime))
+            .setFlags(1)
+            .addReferences(
+                Model.SpanRef.newBuilder()
+                    .setRefType(Model.SpanRefType.CHILD_OF)
+                    .setSpanId(span3ParentId)
+                    .setTraceId(traceId)
+                    .build())
+            .build();
+
+    Model.Batch testBatch =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder()
+                    .setServiceName("frontend")
+                    .addTags(ipTag)
+                    .addTags(processLevelAppTag)
+                    .build())
+            .addAllSpans(ImmutableList.of(span1, span2))
+            .build();
+
+    Collector.PostSpansRequest batches =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatch).build();
+
+    handler.postSpans(batches, emptyStreamObserver);
+
+    Model.Batch testBatchForProxyLevel =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder().setServiceName("frontend").addTags(ipTag).build())
+            .addAllSpans(ImmutableList.of(span3))
+            .build();
+
+    Collector.PostSpansRequest batchesForProxyLevel =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatchForProxyLevel).build();
+
+    handler.postSpans(batchesForProxyLevel, emptyStreamObserver);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testJaegerDurationSampler() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9000)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687")))
+            .build());
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerGrpcCollectorHandler handler =
+        new JaegerGrpcCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new DurationSampler(5 * 1000), () -> null),
+            null,
+            null);
+
+    Model.KeyValue ipTag =
+        Model.KeyValue.newBuilder()
+            .setKey("ip")
+            .setVStr("10.0.0.1")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1234567890L);
+    buffer.putLong(1234567890123L);
+    ByteString traceId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(1234567L);
+    ByteString span1Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(2345678L);
+    ByteString span2Id = ByteString.copyFrom(buffer.array());
+
+    Model.Span span1 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span1Id)
+            .setDuration(Duration.newBuilder().setSeconds(4L).build())
+            .setOperationName("HTTP GET")
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Span span2 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span2Id)
+            .setDuration(Duration.newBuilder().setSeconds(9L).build())
+            .setOperationName("HTTP GET /")
+            .setStartTime(fromMillis(startTime))
+            .addReferences(
+                Model.SpanRef.newBuilder()
+                    .setRefType(Model.SpanRefType.CHILD_OF)
+                    .setSpanId(span1Id)
+                    .setTraceId(traceId)
+                    .build())
+            .build();
+
+    Model.Batch testBatch =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder().setServiceName("frontend").addTags(ipTag).build())
+            .addAllSpans(ImmutableList.of(span1, span2))
+            .build();
+
+    Collector.PostSpansRequest batches =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatch).build();
+
+    handler.postSpans(batches, emptyStreamObserver);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testJaegerDebugOverride() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9000)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("debug", "true"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4000)
+            .setName("HTTP GET")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("sampling.priority", "0.3"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("_sampledByPolicy", "test")))
+            .build());
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerGrpcCollectorHandler handler =
+        new JaegerGrpcCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(
+                new DurationSampler(10 * 1000),
+                () ->
+                    ImmutableList.of(
+                        new SpanSamplingPolicy("test", "{{sampling.priority}}='0.3'", 100))),
+            null,
+            null);
+
+    Model.KeyValue ipTag =
+        Model.KeyValue.newBuilder()
+            .setKey("ip")
+            .setVStr("10.0.0.1")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue debugTag =
+        Model.KeyValue.newBuilder()
+            .setKey("debug")
+            .setVStr("true")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue samplePriorityTag =
+        Model.KeyValue.newBuilder()
+            .setKey("sampling.priority")
+            .setVFloat64(0.3)
+            .setVType(Model.ValueType.FLOAT64)
+            .build();
+
+    ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1234567890L);
+    buffer.putLong(1234567890123L);
+    ByteString traceId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(2345678L);
+    ByteString span1Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(1234567L);
+    ByteString span2Id = ByteString.copyFrom(buffer.array());
+
+    Model.Span span1 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span1Id)
+            .setDuration(Duration.newBuilder().setSeconds(9L).build())
+            .setOperationName("HTTP GET /")
+            .addTags(debugTag)
+            .addReferences(
+                Model.SpanRef.newBuilder()
+                    .setRefType(Model.SpanRefType.CHILD_OF)
+                    .setSpanId(span2Id)
+                    .setTraceId(traceId)
+                    .build())
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Span span2 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span2Id)
+            .setDuration(Duration.newBuilder().setSeconds(4L).build())
+            .setOperationName("HTTP GET")
+            .addTags(samplePriorityTag)
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Batch testBatch =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder().setServiceName("frontend").addTags(ipTag).build())
+            .addAllSpans(ImmutableList.of(span1, span2))
+            .build();
+
+    Collector.PostSpansRequest batches =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatch).build();
+
+    handler.postSpans(batches, emptyStreamObserver);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testSourceTagPriority() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9000)
+            .setName("HTTP GET /")
+            .setSource("source-spantag")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4000)
+            .setName("HTTP GET")
+            .setSource("source-processtag")
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(3000)
+            .setName("HTTP GET /test")
+            .setSource("hostname-processtag")
+            .setSpanId("00000000-0000-0000-0000-0051759bfc69")
+            .setTraceId("0000011e-ab2a-9944-0000-000049631900")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerGrpcCollectorHandler handler =
+        new JaegerGrpcCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Model.KeyValue ipTag =
+        Model.KeyValue.newBuilder()
+            .setKey("ip")
+            .setVStr("10.0.0.1")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue hostNameProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("hostname")
+            .setVStr("hostname-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customSourceProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("source")
+            .setVStr("source-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customSourceSpanTag =
+        Model.KeyValue.newBuilder()
+            .setKey("source")
+            .setVStr("source-spantag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1234567890L);
+    buffer.putLong(1234567890123L);
+    ByteString traceId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1231232342340L);
+    buffer.putLong(1231231232L);
+    ByteString trace2Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(2345678L);
+    ByteString span1Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(1234567L);
+    ByteString span2Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(349865507945L);
+    ByteString span3Id = ByteString.copyFrom(buffer.array());
+
+    Model.Span span1 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span1Id)
+            .setDuration(Duration.newBuilder().setSeconds(9L).build())
+            .setOperationName("HTTP GET /")
+            .addTags(customSourceSpanTag)
+            .addReferences(
+                Model.SpanRef.newBuilder()
+                    .setRefType(Model.SpanRefType.CHILD_OF)
+                    .setSpanId(span2Id)
+                    .setTraceId(traceId)
+                    .build())
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Span span2 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span2Id)
+            .setDuration(Duration.newBuilder().setSeconds(4L).build())
+            .setOperationName("HTTP GET")
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Span span3 =
+        Model.Span.newBuilder()
+            .setTraceId(trace2Id)
+            .setSpanId(span3Id)
+            .setDuration(Duration.newBuilder().setSeconds(3L).build())
+            .setOperationName("HTTP GET /test")
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Batch testBatch =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder()
+                    .setServiceName("frontend")
+                    .addTags(ipTag)
+                    .addTags(hostNameProcessTag)
+                    .addTags(customSourceProcessTag)
+                    .build())
+            .addAllSpans(ImmutableList.of(span1, span2))
+            .build();
+
+    Collector.PostSpansRequest batches =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatch).build();
+
+    handler.postSpans(batches, emptyStreamObserver);
+
+    Model.Batch testBatchForProxyLevel =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder()
+                    .setServiceName("frontend")
+                    .addTags(ipTag)
+                    .addTags(hostNameProcessTag)
+                    .build())
+            .addAllSpans(ImmutableList.of(span3))
+            .build();
+
+    Collector.PostSpansRequest batchesSourceAsProcessTagHostName =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatchForProxyLevel).build();
+
+    handler.postSpans(batchesSourceAsProcessTagHostName, emptyStreamObserver);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testIgnoresServiceTags() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9000)
+            .setName("HTTP GET /")
+            .setSource("source-processtag")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4000)
+            .setName("HTTP GET")
+            .setSource("source-processtag")
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(3000)
+            .setName("HTTP GET /test")
+            .setSource("source-processtag")
+            .setSpanId("00000000-0000-0000-0000-0051759bfc69")
+            .setTraceId("0000011e-ab2a-9944-0000-000049631900")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerGrpcCollectorHandler handler =
+        new JaegerGrpcCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Model.KeyValue ipTag =
+        Model.KeyValue.newBuilder()
+            .setKey("ip")
+            .setVStr("10.0.0.1")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue sourceProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("source")
+            .setVStr("source-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customServiceProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("service")
+            .setVStr("service-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customServiceSpanTag =
+        Model.KeyValue.newBuilder()
+            .setKey("service")
+            .setVStr("service-spantag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1234567890L);
+    buffer.putLong(1234567890123L);
+    ByteString traceId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1231232342340L);
+    buffer.putLong(1231231232L);
+    ByteString trace2Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(2345678L);
+    ByteString span1Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(1234567L);
+    ByteString span2Id = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(349865507945L);
+    ByteString span3Id = ByteString.copyFrom(buffer.array());
+
+    Model.Span span1 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span1Id)
+            .setDuration(Duration.newBuilder().setSeconds(9L).build())
+            .setOperationName("HTTP GET /")
+            .addTags(customServiceSpanTag)
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Span span2 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span2Id)
+            .setDuration(Duration.newBuilder().setSeconds(4L).build())
+            .setOperationName("HTTP GET")
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Span span3 =
+        Model.Span.newBuilder()
+            .setTraceId(trace2Id)
+            .setSpanId(span3Id)
+            .setDuration(Duration.newBuilder().setSeconds(3L).build())
+            .setOperationName("HTTP GET /test")
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Batch testBatch =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder()
+                    .setServiceName("frontend")
+                    .addTags(ipTag)
+                    .addTags(sourceProcessTag)
+                    .addTags(customServiceProcessTag)
+                    .build())
+            .addAllSpans(ImmutableList.of(span1, span2))
+            .build();
+
+    Collector.PostSpansRequest batches =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatch).build();
+    handler.postSpans(batches, emptyStreamObserver);
+
+    Model.Batch testBatchForProxyLevel =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder()
+                    .setServiceName("frontend")
+                    .addTags(ipTag)
+                    .addTags(sourceProcessTag)
+                    .build())
+            .addAllSpans(ImmutableList.of(span3))
+            .build();
+
+    handler.postSpans(
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatchForProxyLevel).build(),
+        emptyStreamObserver);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testProtectedTagsSpanOverridesProcess() throws Exception {
+    // cluster, shard and service are special tags, because they're indexed by wavefront
+    // The priority order is:
+    // Span Level > Process Level > Proxy Level > Default
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9000)
+            .setName("HTTP GET /")
+            .setSource("source-spantag")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "application-spantag"),
+                    new Annotation("cluster", "cluster-spantag"),
+                    new Annotation("shard", "shard-spantag")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerGrpcCollectorHandler handler =
+        new JaegerGrpcCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Model.KeyValue customSourceSpanTag =
+        Model.KeyValue.newBuilder()
+            .setKey("source")
+            .setVStr("source-spantag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customApplicationProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("application")
+            .setVStr("application-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customApplicationSpanTag =
+        Model.KeyValue.newBuilder()
+            .setKey("application")
+            .setVStr("application-spantag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customClusterProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("cluster")
+            .setVStr("cluster-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customClusterSpanTag =
+        Model.KeyValue.newBuilder()
+            .setKey("cluster")
+            .setVStr("cluster-spantag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customShardProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("shard")
+            .setVStr("shard-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customShardSpanTag =
+        Model.KeyValue.newBuilder()
+            .setKey("shard")
+            .setVStr("shard-spantag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1234567890L);
+    buffer.putLong(1234567890123L);
+    ByteString traceId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(2345678L);
+    ByteString spanId = ByteString.copyFrom(buffer.array());
+
+    Model.Span span =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(spanId)
+            .setDuration(Duration.newBuilder().setSeconds(9L).build())
+            .setOperationName("HTTP GET /")
+            .addTags(customSourceSpanTag)
+            .addTags(customApplicationSpanTag)
+            .addTags(customClusterSpanTag)
+            .addTags(customShardSpanTag)
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Batch testBatch =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder()
+                    .setServiceName("frontend")
+                    .addTags(customApplicationProcessTag)
+                    .addTags(customClusterProcessTag)
+                    .addTags(customShardProcessTag)
+                    .build())
+            .addAllSpans(ImmutableList.of(span))
+            .build();
+
+    Collector.PostSpansRequest batches =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatch).build();
+    handler.postSpans(batches, emptyStreamObserver);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testProtectedTagsProcessOverridesProxyConfig() throws Exception {
+    // cluster, shard and service are special tags, because they're indexed by wavefront
+    // The priority order is:
+    // Span Level > Process Level > Proxy Level > Default
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9000)
+            .setName("HTTP GET /")
+            .setSource("source-spantag")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "application-processtag"),
+                    new Annotation("cluster", "cluster-processtag"),
+                    new Annotation("shard", "shard-processtag")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerGrpcCollectorHandler handler =
+        new JaegerGrpcCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Model.KeyValue customSourceSpanTag =
+        Model.KeyValue.newBuilder()
+            .setKey("source")
+            .setVStr("source-spantag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customApplicationProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("application")
+            .setVStr("application-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customClusterProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("cluster")
+            .setVStr("cluster-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customShardProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("shard")
+            .setVStr("shard-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1234567890L);
+    buffer.putLong(1234567890123L);
+    ByteString traceId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(2345678L);
+    ByteString spanId = ByteString.copyFrom(buffer.array());
+
+    Model.Span span =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(spanId)
+            .setDuration(Duration.newBuilder().setSeconds(9L).build())
+            .setOperationName("HTTP GET /")
+            .addTags(customSourceSpanTag)
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Batch testBatch =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder()
+                    .setServiceName("frontend")
+                    .addTags(customApplicationProcessTag)
+                    .addTags(customClusterProcessTag)
+                    .addTags(customShardProcessTag)
+                    .build())
+            .addAllSpans(ImmutableList.of(span))
+            .build();
+
+    Collector.PostSpansRequest batches =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatch).build();
+    handler.postSpans(batches, emptyStreamObserver);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testAllProcessTagsPropagated() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9000)
+            .setName("HTTP GET /")
+            .setSource("source-spantag")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("processTag1", "one"),
+                    new Annotation("processTag2", "two"),
+                    new Annotation("processTag3", "three"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerGrpcCollectorHandler handler =
+        new JaegerGrpcCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Model.KeyValue ipTag =
+        Model.KeyValue.newBuilder()
+            .setKey("ip")
+            .setVStr("10.0.0.1")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue hostNameProcessTag =
+        Model.KeyValue.newBuilder()
+            .setKey("hostname")
+            .setVStr("hostname-processtag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customProcessTag1 =
+        Model.KeyValue.newBuilder()
+            .setKey("processTag1")
+            .setVStr("one")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customProcessTag2 =
+        Model.KeyValue.newBuilder()
+            .setKey("processTag2")
+            .setVStr("two")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customProcessTag3 =
+        Model.KeyValue.newBuilder()
+            .setKey("processTag3")
+            .setVStr("three")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    Model.KeyValue customSourceSpanTag =
+        Model.KeyValue.newBuilder()
+            .setKey("source")
+            .setVStr("source-spantag")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1234567890L);
+    buffer.putLong(1234567890123L);
+    ByteString traceId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(2345678L);
+    ByteString spanId = ByteString.copyFrom(buffer.array());
+
+    Model.Span span =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(spanId)
+            .setDuration(Duration.newBuilder().setSeconds(9L).build())
+            .setOperationName("HTTP GET /")
+            .addTags(customSourceSpanTag)
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Batch testBatch =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder()
+                    .setServiceName("frontend")
+                    .addTags(ipTag)
+                    .addTags(hostNameProcessTag)
+                    .addTags(customProcessTag1)
+                    .addTags(customProcessTag2)
+                    .addTags(customProcessTag3)
+                    .build())
+            .addAllSpans(ImmutableList.of(span))
+            .build();
+
+    Collector.PostSpansRequest batches =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatch).build();
+
+    handler.postSpans(batches, emptyStreamObserver);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  /**
+   * Test for derived metrics emitted from Jaeger trace listeners. Derived metrics should report tag
+   * values post applying preprocessing rules to the span.
+   */
+  @Test
+  public void testJaegerPreprocessedDerivedMetrics() throws Exception {
+    reset(mockTraceHandler, mockWavefrontSender);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4000)
+            .setName("HTTP GET")
+            .setSource(PREPROCESSED_SOURCE_VALUE)
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("service", PREPROCESSED_SERVICE_TAG_VALUE),
+                    new Annotation("application", PREPROCESSED_APPLICATION_TAG_VALUE),
+                    new Annotation("cluster", PREPROCESSED_CLUSTER_TAG_VALUE),
+                    new Annotation("shard", PREPROCESSED_SHARD_TAG_VALUE)))
+            .build());
+    expectLastCall();
+
+    Capture> tagsCapture = EasyMock.newCapture();
+    mockWavefrontSender.sendMetric(
+        eq(HEART_BEAT_METRIC),
+        eq(1.0),
+        anyLong(),
+        eq(PREPROCESSED_SOURCE_VALUE),
+        EasyMock.capture(tagsCapture));
+    expectLastCall().anyTimes();
+
+    replay(mockTraceHandler, mockWavefrontSender);
+
+    Supplier preprocessorSupplier =
+        () -> {
+          ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+          PreprocessorRuleMetrics preprocessorRuleMetrics =
+              new PreprocessorRuleMetrics(null, null, null);
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      APPLICATION_TAG_KEY,
+                      "^Jaeger.*",
+                      PREPROCESSED_APPLICATION_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      SERVICE_TAG_KEY,
+                      "^test.*",
+                      PREPROCESSED_SERVICE_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      "sourceName",
+                      "^jaeger.*",
+                      PREPROCESSED_SOURCE_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      CLUSTER_TAG_KEY,
+                      "^none.*",
+                      PREPROCESSED_CLUSTER_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      SHARD_TAG_KEY,
+                      "^none.*",
+                      PREPROCESSED_SHARD_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          return preprocessor;
+        };
+
+    JaegerGrpcCollectorHandler handler =
+        new JaegerGrpcCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            mockWavefrontSender,
+            () -> false,
+            () -> false,
+            preprocessorSupplier,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Model.KeyValue ipTag =
+        Model.KeyValue.newBuilder()
+            .setKey("ip")
+            .setVStr("10.0.0.1")
+            .setVType(Model.ValueType.STRING)
+            .build();
+
+    ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
+    buffer.putLong(1234567890L);
+    buffer.putLong(1234567890123L);
+    ByteString traceId = ByteString.copyFrom(buffer.array());
+
+    buffer = ByteBuffer.allocate(Long.BYTES);
+    buffer.putLong(1234567L);
+    ByteString span2Id = ByteString.copyFrom(buffer.array());
+
+    Model.Span span2 =
+        Model.Span.newBuilder()
+            .setTraceId(traceId)
+            .setSpanId(span2Id)
+            .setDuration(Duration.newBuilder().setSeconds(4L).build())
+            .setOperationName("HTTP GET")
+            .setStartTime(fromMillis(startTime))
+            .build();
+
+    Model.Batch testBatch =
+        Model.Batch.newBuilder()
+            .setProcess(
+                Model.Process.newBuilder().setServiceName("testService").addTags(ipTag).build())
+            .addAllSpans(ImmutableList.of(span2))
+            .build();
+
+    Collector.PostSpansRequest batches =
+        Collector.PostSpansRequest.newBuilder().setBatch(testBatch).build();
+
+    handler.postSpans(batches, emptyStreamObserver);
+    handler.run();
+
+    verifyWithTimeout(500, mockTraceHandler, mockWavefrontSender);
+    HashMap tagsReturned = tagsCapture.getValue();
+    assertEquals(PREPROCESSED_APPLICATION_TAG_VALUE, tagsReturned.get(APPLICATION_TAG_KEY));
+    assertEquals(PREPROCESSED_SERVICE_TAG_VALUE, tagsReturned.get(SERVICE_TAG_KEY));
+    assertEquals(PREPROCESSED_CLUSTER_TAG_VALUE, tagsReturned.get(CLUSTER_TAG_KEY));
+    assertEquals(PREPROCESSED_SHARD_TAG_VALUE, tagsReturned.get(SHARD_TAG_KEY));
+  }
+
+  private final StreamObserver emptyStreamObserver =
+      new StreamObserver() {
+        @Override
+        public void onNext(Collector.PostSpansResponse postSpansResponse) {}
+
+        @Override
+        public void onError(Throwable throwable) {}
+
+        @Override
+        public void onCompleted() {}
+      };
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/tracing/JaegerPortUnificationHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/JaegerPortUnificationHandlerTest.java
new file mode 100644
index 000000000..153f918a2
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/JaegerPortUnificationHandlerTest.java
@@ -0,0 +1,458 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.agent.TestUtils.verifyWithTimeout;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.HEART_BEAT_METRIC;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.wavefront.agent.auth.TokenAuthenticatorBuilder;
+import com.wavefront.agent.channel.NoopHealthCheckManager;
+import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.PreprocessorRuleMetrics;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.preprocessor.SpanReplaceRegexTransformer;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.entities.tracing.sampling.RateSampler;
+import io.jaegertracing.thriftjava.Batch;
+import io.jaegertracing.thriftjava.Log;
+import io.jaegertracing.thriftjava.Process;
+import io.jaegertracing.thriftjava.Tag;
+import io.jaegertracing.thriftjava.TagType;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.DefaultFullHttpRequest;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpVersion;
+import java.util.HashMap;
+import java.util.function.Supplier;
+import org.apache.thrift.TSerializer;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
+
+/**
+ * Unit tests for {@link JaegerPortUnificationHandler}.
+ *
+ * @author Han Zhang (zhanghan@vmware.com)
+ */
+public class JaegerPortUnificationHandlerTest {
+  private static final String DEFAULT_SOURCE = "jaeger";
+  private ReportableEntityHandler mockTraceHandler =
+      MockReportableEntityHandlerFactory.getMockTraceHandler();
+  private ReportableEntityHandler mockTraceSpanLogsHandler =
+      MockReportableEntityHandlerFactory.getMockTraceSpanLogsHandler();
+  private WavefrontSender mockWavefrontSender = EasyMock.createMock(WavefrontSender.class);
+  private ChannelHandlerContext mockCtx = createNiceMock(ChannelHandlerContext.class);
+
+  private long startTime = System.currentTimeMillis();
+
+  // Derived RED metrics related.
+  private final String PREPROCESSED_APPLICATION_TAG_VALUE = "preprocessedApplication";
+  private final String PREPROCESSED_SERVICE_TAG_VALUE = "preprocessedService";
+  private final String PREPROCESSED_CLUSTER_TAG_VALUE = "preprocessedCluster";
+  private final String PREPROCESSED_SHARD_TAG_VALUE = "preprocessedShard";
+  private final String PREPROCESSED_SOURCE_VALUE = "preprocessedSource";
+
+  /**
+   * Test for derived metrics emitted from Jaeger trace listeners. Derived metrics should report tag
+   * values post applying preprocessing rules to the span.
+   */
+  @Test
+  public void testJaegerPreprocessedDerivedMetrics() throws Exception {
+    Supplier preprocessorSupplier =
+        () -> {
+          ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+          PreprocessorRuleMetrics preprocessorRuleMetrics =
+              new PreprocessorRuleMetrics(null, null, null);
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      APPLICATION_TAG_KEY,
+                      "^Jaeger.*",
+                      PREPROCESSED_APPLICATION_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      SERVICE_TAG_KEY,
+                      "^test.*",
+                      PREPROCESSED_SERVICE_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      "sourceName",
+                      "^jaeger.*",
+                      PREPROCESSED_SOURCE_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      CLUSTER_TAG_KEY,
+                      "^none.*",
+                      PREPROCESSED_CLUSTER_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      SHARD_TAG_KEY,
+                      "^none.*",
+                      PREPROCESSED_SHARD_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          return preprocessor;
+        };
+
+    JaegerPortUnificationHandler handler =
+        new JaegerPortUnificationHandler(
+            "14268",
+            TokenAuthenticatorBuilder.create().build(),
+            new NoopHealthCheckManager(),
+            mockTraceHandler,
+            mockTraceSpanLogsHandler,
+            mockWavefrontSender,
+            () -> false,
+            () -> false,
+            preprocessorSupplier,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    io.jaegertracing.thriftjava.Span span1 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            1234567L,
+            0L,
+            "HTTP GET",
+            1,
+            startTime * 1000,
+            1234 * 1000);
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "testService";
+    testBatch.setSpans(ImmutableList.of(span1));
+
+    // Reset mock
+    reset(mockCtx, mockTraceHandler, mockWavefrontSender);
+
+    // Set Expectation
+    Span expectedSpan1 =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(1234)
+            .setName("HTTP GET")
+            .setSource(PREPROCESSED_SOURCE_VALUE)
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            .
+            // Note: Order of annotations list matters for this unit test.
+            setAnnotations(
+                ImmutableList.of(
+                    new Annotation("jaegerSpanId", "12d687"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", PREPROCESSED_SERVICE_TAG_VALUE),
+                    new Annotation("application", PREPROCESSED_APPLICATION_TAG_VALUE),
+                    new Annotation("cluster", PREPROCESSED_CLUSTER_TAG_VALUE),
+                    new Annotation("shard", PREPROCESSED_SHARD_TAG_VALUE)))
+            .build();
+    mockTraceHandler.report(expectedSpan1);
+    expectLastCall();
+
+    Capture> tagsCapture = EasyMock.newCapture();
+    mockWavefrontSender.sendMetric(
+        eq(HEART_BEAT_METRIC),
+        eq(1.0),
+        anyLong(),
+        eq(PREPROCESSED_SOURCE_VALUE),
+        EasyMock.capture(tagsCapture));
+    expectLastCall().anyTimes();
+
+    replay(mockCtx, mockTraceHandler, mockWavefrontSender);
+
+    ByteBuf content = Unpooled.copiedBuffer(new TSerializer().serialize(testBatch));
+
+    FullHttpRequest httpRequest =
+        new DefaultFullHttpRequest(
+            HttpVersion.HTTP_1_1,
+            HttpMethod.POST,
+            "http://localhost:14268/api/traces",
+            content,
+            true);
+    handler.handleHttpMessage(mockCtx, httpRequest);
+    handler.run();
+    verifyWithTimeout(500, mockTraceHandler, mockWavefrontSender);
+    HashMap tagsReturned = tagsCapture.getValue();
+    assertEquals(PREPROCESSED_APPLICATION_TAG_VALUE, tagsReturned.get(APPLICATION_TAG_KEY));
+    assertEquals(PREPROCESSED_SERVICE_TAG_VALUE, tagsReturned.get(SERVICE_TAG_KEY));
+    assertEquals(PREPROCESSED_CLUSTER_TAG_VALUE, tagsReturned.get(CLUSTER_TAG_KEY));
+    assertEquals(PREPROCESSED_SHARD_TAG_VALUE, tagsReturned.get(SHARD_TAG_KEY));
+  }
+
+  @Test
+  public void testJaegerPortUnificationHandler() throws Exception {
+    reset(mockTraceHandler, mockTraceSpanLogsHandler, mockCtx);
+    Span expectedSpan1 =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(1234)
+            .setName("HTTP GET")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "12d687"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("component", "db"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("_spanLogs", "true")))
+            .build();
+    mockTraceHandler.report(expectedSpan1);
+    expectLastCall();
+
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("default")
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(startTime * 1000)
+                        .setFields(
+                            ImmutableMap.of("event", "error", "exception", "NullPointerException"))
+                        .build()))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(2345)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687"),
+                    new Annotation("component", "db"),
+                    new Annotation("application", "Custom-JaegerApp"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(3456)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-9a12-b85901d53397")
+            .setTraceId("00000000-0000-0000-fea4-87ee36e58cab")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "9a12b85901d53397"),
+                    new Annotation("jaegerTraceId", "fea487ee36e58cab"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-fea4-87ee36e58cab"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    // Test filtering empty tags
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(3456)
+            .setName("HTTP GET /test")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-0051759bfc69")
+            .setTraceId("0000011e-ab2a-9944-0000-000049631900")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "51759bfc69"),
+                    new Annotation("jaegerTraceId", "11eab2a99440000000049631900"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    expect(mockCtx.write(EasyMock.isA(FullHttpResponse.class))).andReturn(null).anyTimes();
+
+    replay(mockTraceHandler, mockTraceSpanLogsHandler, mockCtx);
+
+    JaegerPortUnificationHandler handler =
+        new JaegerPortUnificationHandler(
+            "14268",
+            TokenAuthenticatorBuilder.create().build(),
+            new NoopHealthCheckManager(),
+            mockTraceHandler,
+            mockTraceSpanLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Tag ipTag = new Tag("ip", TagType.STRING);
+    ipTag.setVStr("10.0.0.1");
+
+    Tag componentTag = new Tag("component", TagType.STRING);
+    componentTag.setVStr("db");
+
+    Tag customApplicationTag = new Tag("application", TagType.STRING);
+    customApplicationTag.setVStr("Custom-JaegerApp");
+
+    Tag emptyTag = new Tag("empty", TagType.STRING);
+    emptyTag.setVStr("");
+
+    io.jaegertracing.thriftjava.Span span1 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            1234567L,
+            0L,
+            "HTTP GET",
+            1,
+            startTime * 1000,
+            1234 * 1000);
+
+    io.jaegertracing.thriftjava.Span span2 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            2345678L,
+            1234567L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            2345 * 1000);
+
+    // check negative span IDs too
+    io.jaegertracing.thriftjava.Span span3 =
+        new io.jaegertracing.thriftjava.Span(
+            -97803834702328661L,
+            0L,
+            -7344605349865507945L,
+            -97803834702328661L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            3456 * 1000);
+
+    io.jaegertracing.thriftjava.Span span4 =
+        new io.jaegertracing.thriftjava.Span(
+            1231231232L,
+            1231232342340L,
+            349865507945L,
+            0,
+            "HTTP GET /test",
+            1,
+            startTime * 1000,
+            3456 * 1000);
+
+    span1.setTags(ImmutableList.of(componentTag));
+    span2.setTags(ImmutableList.of(componentTag, customApplicationTag));
+    span4.setTags(ImmutableList.of(emptyTag));
+
+    Tag tag1 = new Tag("event", TagType.STRING);
+    tag1.setVStr("error");
+    Tag tag2 = new Tag("exception", TagType.STRING);
+    tag2.setVStr("NullPointerException");
+    span1.setLogs(ImmutableList.of(new Log(startTime * 1000, ImmutableList.of(tag1, tag2))));
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+    testBatch.process.setTags(ImmutableList.of(ipTag));
+
+    testBatch.setSpans(ImmutableList.of(span1, span2, span3, span4));
+
+    ByteBuf content = Unpooled.copiedBuffer(new TSerializer().serialize(testBatch));
+
+    FullHttpRequest httpRequest =
+        new DefaultFullHttpRequest(
+            HttpVersion.HTTP_1_1,
+            HttpMethod.POST,
+            "http://localhost:14268/api/traces",
+            content,
+            true);
+    handler.handleHttpMessage(mockCtx, httpRequest);
+    verify(mockTraceHandler, mockTraceSpanLogsHandler);
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/tracing/JaegerTChannelCollectorHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/JaegerTChannelCollectorHandlerTest.java
new file mode 100644
index 000000000..ee37ab8d1
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/JaegerTChannelCollectorHandlerTest.java
@@ -0,0 +1,1366 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.uber.tchannel.messages.ThriftRequest;
+import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.api.agent.SpanSamplingPolicy;
+import com.wavefront.sdk.entities.tracing.sampling.DurationSampler;
+import com.wavefront.sdk.entities.tracing.sampling.RateSampler;
+import io.jaegertracing.thriftjava.Batch;
+import io.jaegertracing.thriftjava.Collector;
+import io.jaegertracing.thriftjava.Log;
+import io.jaegertracing.thriftjava.Process;
+import io.jaegertracing.thriftjava.Tag;
+import io.jaegertracing.thriftjava.TagType;
+import org.easymock.Capture;
+import org.junit.Test;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
+
+public class JaegerTChannelCollectorHandlerTest {
+  private static final String DEFAULT_SOURCE = "jaeger";
+  private ReportableEntityHandler mockTraceHandler =
+      MockReportableEntityHandlerFactory.getMockTraceHandler();
+  private ReportableEntityHandler mockTraceLogsHandler =
+      MockReportableEntityHandlerFactory.getMockTraceSpanLogsHandler();
+  private long startTime = System.currentTimeMillis();
+
+  @Test
+  public void testJaegerTChannelCollector() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+    Span expectedSpan1 =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(1234)
+            .setName("HTTP GET")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "12d687"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("component", "db"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("_spanLogs", "true")))
+            .build();
+    mockTraceHandler.report(expectedSpan1);
+    expectLastCall();
+
+    mockTraceLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("default")
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(startTime * 1000)
+                        .setFields(
+                            ImmutableMap.of("event", "error", "exception", "NullPointerException"))
+                        .build()))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(2345)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687"),
+                    new Annotation("component", "db"),
+                    new Annotation("application", "Custom-JaegerApp"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(3456)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-9a12-b85901d53397")
+            .setTraceId("00000000-0000-0000-fea4-87ee36e58cab")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "9a12b85901d53397"),
+                    new Annotation("jaegerTraceId", "fea487ee36e58cab"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-fea4-87ee36e58cab"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    // Test filtering empty tags
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(3456)
+            .setName("HTTP GET /test")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-0051759bfc69")
+            .setTraceId("0000011e-ab2a-9944-0000-000049631900")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "51759bfc69"),
+                    new Annotation("jaegerTraceId", "11eab2a99440000000049631900"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerTChannelCollectorHandler handler =
+        new JaegerTChannelCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Tag ipTag = new Tag("ip", TagType.STRING);
+    ipTag.setVStr("10.0.0.1");
+
+    Tag componentTag = new Tag("component", TagType.STRING);
+    componentTag.setVStr("db");
+
+    Tag customApplicationTag = new Tag("application", TagType.STRING);
+    customApplicationTag.setVStr("Custom-JaegerApp");
+
+    Tag emptyTag = new Tag("empty", TagType.STRING);
+    emptyTag.setVStr("");
+
+    io.jaegertracing.thriftjava.Span span1 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            1234567L,
+            0L,
+            "HTTP GET",
+            1,
+            startTime * 1000,
+            1234 * 1000);
+
+    io.jaegertracing.thriftjava.Span span2 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            2345678L,
+            1234567L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            2345 * 1000);
+
+    // check negative span IDs too
+    io.jaegertracing.thriftjava.Span span3 =
+        new io.jaegertracing.thriftjava.Span(
+            -97803834702328661L,
+            0L,
+            -7344605349865507945L,
+            -97803834702328661L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            3456 * 1000);
+
+    io.jaegertracing.thriftjava.Span span4 =
+        new io.jaegertracing.thriftjava.Span(
+            1231231232L,
+            1231232342340L,
+            349865507945L,
+            0,
+            "HTTP GET /test",
+            1,
+            startTime * 1000,
+            3456 * 1000);
+
+    span1.setTags(ImmutableList.of(componentTag));
+    span2.setTags(ImmutableList.of(componentTag, customApplicationTag));
+    span4.setTags(ImmutableList.of(emptyTag));
+
+    Tag tag1 = new Tag("event", TagType.STRING);
+    tag1.setVStr("error");
+    Tag tag2 = new Tag("exception", TagType.STRING);
+    tag2.setVStr("NullPointerException");
+    span1.setLogs(ImmutableList.of(new Log(startTime * 1000, ImmutableList.of(tag1, tag2))));
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+    testBatch.process.setTags(ImmutableList.of(ipTag));
+
+    testBatch.setSpans(ImmutableList.of(span1, span2, span3, span4));
+
+    Collector.submitBatches_args batches = new Collector.submitBatches_args();
+    batches.addToBatches(testBatch);
+    ThriftRequest request =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batches)
+            .build();
+    handler.handleImpl(request);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testApplicationTagPriority() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    // Span to verify span level tags precedence
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(1234)
+            .setName("HTTP GET")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "12d687"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("component", "db"),
+                    new Annotation("application", "SpanLevelAppTag"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    // Span to verify process level tags precedence
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(2345)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687"),
+                    new Annotation("component", "db"),
+                    new Annotation("application", "ProcessLevelAppTag"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    // Span to verify Proxy level tags precedence
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(3456)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-9a12-b85901d53397")
+            .setTraceId("00000000-0000-0000-fea4-87ee36e58cab")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "9a12b85901d53397"),
+                    new Annotation("jaegerTraceId", "fea487ee36e58cab"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-fea4-87ee36e58cab"),
+                    new Annotation("application", "ProxyLevelAppTag"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    // Verify span level "application" tags precedence
+    JaegerTChannelCollectorHandler handler =
+        new JaegerTChannelCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            "ProxyLevelAppTag",
+            null);
+
+    Tag ipTag = new Tag("ip", TagType.STRING);
+    ipTag.setVStr("10.0.0.1");
+
+    Tag componentTag = new Tag("component", TagType.STRING);
+    componentTag.setVStr("db");
+
+    Tag spanLevelAppTag = new Tag("application", TagType.STRING);
+    spanLevelAppTag.setVStr("SpanLevelAppTag");
+
+    Tag processLevelAppTag = new Tag("application", TagType.STRING);
+    processLevelAppTag.setVStr("ProcessLevelAppTag");
+
+    io.jaegertracing.thriftjava.Span span1 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            1234567L,
+            0L,
+            "HTTP GET",
+            1,
+            startTime * 1000,
+            1234 * 1000);
+
+    io.jaegertracing.thriftjava.Span span2 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            2345678L,
+            1234567L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            2345 * 1000);
+
+    // check negative span IDs too
+    io.jaegertracing.thriftjava.Span span3 =
+        new io.jaegertracing.thriftjava.Span(
+            -97803834702328661L,
+            0L,
+            -7344605349865507945L,
+            -97803834702328661L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            3456 * 1000);
+
+    // Span1 to verify span level tags precedence
+    span1.setTags(ImmutableList.of(componentTag, spanLevelAppTag));
+    span2.setTags(ImmutableList.of(componentTag));
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+    // Span2 to verify process level tags precedence
+    testBatch.process.setTags(ImmutableList.of(ipTag, processLevelAppTag));
+
+    testBatch.setSpans(ImmutableList.of(span1, span2));
+
+    Collector.submitBatches_args batches = new Collector.submitBatches_args();
+    batches.addToBatches(testBatch);
+    ThriftRequest request =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batches)
+            .build();
+    handler.handleImpl(request);
+
+    // Span3 to verify process level tags precedence. So do not set any process level tag.
+    Batch testBatchForProxyLevel = new Batch();
+    testBatchForProxyLevel.process = new Process();
+    testBatchForProxyLevel.process.serviceName = "frontend";
+    testBatchForProxyLevel.process.setTags(ImmutableList.of(ipTag));
+
+    testBatchForProxyLevel.setSpans(ImmutableList.of(span3));
+
+    Collector.submitBatches_args batchesForProxyLevel = new Collector.submitBatches_args();
+    batchesForProxyLevel.addToBatches(testBatchForProxyLevel);
+    ThriftRequest requestForProxyLevel =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batchesForProxyLevel)
+            .build();
+    handler.handleImpl(requestForProxyLevel);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testJaegerDurationSampler() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    Span expectedSpan2 =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("_spanLogs", "true")))
+            .build();
+    mockTraceHandler.report(expectedSpan2);
+    expectLastCall();
+
+    mockTraceLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("default")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(startTime * 1000)
+                        .setFields(
+                            ImmutableMap.of("event", "error", "exception", "NullPointerException"))
+                        .build()))
+            .build());
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerTChannelCollectorHandler handler =
+        new JaegerTChannelCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new DurationSampler(5), () -> null),
+            null,
+            null);
+
+    Tag ipTag = new Tag("ip", TagType.STRING);
+    ipTag.setVStr("10.0.0.1");
+
+    io.jaegertracing.thriftjava.Span span1 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L, 1234567890L, 1234567L, 0L, "HTTP GET", 1, startTime * 1000, 4 * 1000);
+
+    io.jaegertracing.thriftjava.Span span2 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            2345678L,
+            1234567L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            9 * 1000);
+
+    Tag tag1 = new Tag("event", TagType.STRING);
+    tag1.setVStr("error");
+    Tag tag2 = new Tag("exception", TagType.STRING);
+    tag2.setVStr("NullPointerException");
+    span1.setLogs(ImmutableList.of(new Log(startTime * 1000, ImmutableList.of(tag1, tag2))));
+    span2.setLogs(ImmutableList.of(new Log(startTime * 1000, ImmutableList.of(tag1, tag2))));
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+    testBatch.process.setTags(ImmutableList.of(ipTag));
+
+    testBatch.setSpans(ImmutableList.of(span1, span2));
+
+    Collector.submitBatches_args batches = new Collector.submitBatches_args();
+    batches.addToBatches(testBatch);
+    ThriftRequest request =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batches)
+            .build();
+    handler.handleImpl(request);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testJaegerDebugOverride() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    Span expectedSpan1 =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687"),
+                    new Annotation("debug", "true"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("_spanLogs", "true")))
+            .build();
+    mockTraceHandler.report(expectedSpan1);
+    expectLastCall();
+
+    mockTraceLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("default")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(startTime * 1000)
+                        .setFields(
+                            ImmutableMap.of("event", "error", "exception", "NullPointerException"))
+                        .build()))
+            .build());
+    expectLastCall();
+
+    Span expectedSpan2 =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4)
+            .setName("HTTP GET")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "12d687"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("sampling.priority", "0.3"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("_spanLogs", "true"),
+                    new Annotation("_sampledByPolicy", "test")))
+            .build();
+    mockTraceHandler.report(expectedSpan2);
+    expectLastCall();
+
+    mockTraceLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("default")
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            .setSpan("_sampledByPolicy=test")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(startTime * 1000)
+                        .setFields(
+                            ImmutableMap.of("event", "error", "exception", "NullPointerException"))
+                        .build()))
+            .build());
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerTChannelCollectorHandler handler =
+        new JaegerTChannelCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(
+                new DurationSampler(10),
+                () ->
+                    ImmutableList.of(
+                        new SpanSamplingPolicy("test", "{{sampling.priority}}='0.3'", 100))),
+            null,
+            null);
+
+    Tag ipTag = new Tag("ip", TagType.STRING);
+    ipTag.setVStr("10.0.0.1");
+
+    Tag debugTag = new Tag("debug", TagType.STRING);
+    debugTag.setVStr("true");
+
+    io.jaegertracing.thriftjava.Span span1 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            2345678L,
+            1234567L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            9 * 1000);
+    span1.setTags(ImmutableList.of(debugTag));
+
+    Tag samplePriorityTag = new Tag("sampling.priority", TagType.DOUBLE);
+    samplePriorityTag.setVDouble(0.3);
+    io.jaegertracing.thriftjava.Span span2 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L, 1234567890L, 1234567L, 0L, "HTTP GET", 1, startTime * 1000, 4 * 1000);
+    span2.setTags(ImmutableList.of(samplePriorityTag));
+
+    Tag tag1 = new Tag("event", TagType.STRING);
+    tag1.setVStr("error");
+    Tag tag2 = new Tag("exception", TagType.STRING);
+    tag2.setVStr("NullPointerException");
+    span1.setLogs(ImmutableList.of(new Log(startTime * 1000, ImmutableList.of(tag1, tag2))));
+    span2.setLogs(ImmutableList.of(new Log(startTime * 1000, ImmutableList.of(tag1, tag2))));
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+    testBatch.process.setTags(ImmutableList.of(ipTag));
+
+    testBatch.setSpans(ImmutableList.of(span1, span2));
+
+    Collector.submitBatches_args batches = new Collector.submitBatches_args();
+    batches.addToBatches(testBatch);
+    ThriftRequest request =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batches)
+            .build();
+    handler.handleImpl(request);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testSourceTagPriority() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("HTTP GET /")
+            .setSource("source-spantag")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4)
+            .setName("HTTP GET")
+            .setSource("source-processtag")
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "12d687"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(3456)
+            .setName("HTTP GET /test")
+            .setSource("hostname-processtag")
+            .setSpanId("00000000-0000-0000-0000-0051759bfc69")
+            .setTraceId("0000011e-ab2a-9944-0000-000049631900")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "51759bfc69"),
+                    new Annotation("jaegerTraceId", "11eab2a99440000000049631900"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerTChannelCollectorHandler handler =
+        new JaegerTChannelCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Tag ipTag = new Tag("ip", TagType.STRING);
+    ipTag.setVStr("10.0.0.1");
+
+    Tag hostNameProcessTag = new Tag("hostname", TagType.STRING);
+    hostNameProcessTag.setVStr("hostname-processtag");
+
+    Tag customSourceProcessTag = new Tag("source", TagType.STRING);
+    customSourceProcessTag.setVStr("source-processtag");
+
+    Tag customSourceSpanTag = new Tag("source", TagType.STRING);
+    customSourceSpanTag.setVStr("source-spantag");
+
+    io.jaegertracing.thriftjava.Span span1 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            2345678L,
+            1234567L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            9 * 1000);
+    span1.setTags(ImmutableList.of(customSourceSpanTag));
+
+    io.jaegertracing.thriftjava.Span span2 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L, 1234567890L, 1234567L, 0L, "HTTP GET", 1, startTime * 1000, 4 * 1000);
+
+    io.jaegertracing.thriftjava.Span span3 =
+        new io.jaegertracing.thriftjava.Span(
+            1231231232L,
+            1231232342340L,
+            349865507945L,
+            0,
+            "HTTP GET /test",
+            1,
+            startTime * 1000,
+            3456 * 1000);
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+    testBatch.process.setTags(ImmutableList.of(ipTag, hostNameProcessTag, customSourceProcessTag));
+
+    testBatch.setSpans(ImmutableList.of(span1, span2));
+
+    Collector.submitBatches_args batches = new Collector.submitBatches_args();
+    batches.addToBatches(testBatch);
+    ThriftRequest request =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batches)
+            .build();
+    handler.handleImpl(request);
+
+    // Span3 to verify hostname process level tags precedence. So do not set any process level
+    // source tag.
+    Batch testBatchSourceAsProcessTagHostName = new Batch();
+    testBatchSourceAsProcessTagHostName.process = new Process();
+    testBatchSourceAsProcessTagHostName.process.serviceName = "frontend";
+    testBatchSourceAsProcessTagHostName.process.setTags(
+        ImmutableList.of(ipTag, hostNameProcessTag));
+
+    testBatchSourceAsProcessTagHostName.setSpans(ImmutableList.of(span3));
+
+    Collector.submitBatches_args batchesSourceAsProcessTagHostName =
+        new Collector.submitBatches_args();
+    batchesSourceAsProcessTagHostName.addToBatches(testBatchSourceAsProcessTagHostName);
+    ThriftRequest requestForProxyLevel =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batchesSourceAsProcessTagHostName)
+            .build();
+    handler.handleImpl(requestForProxyLevel);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testIgnoresServiceTags() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("HTTP GET /")
+            .setSource("source-processtag")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4)
+            .setName("HTTP GET")
+            .setSource("source-processtag")
+            .setSpanId("00000000-0000-0000-0000-00000012d687")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "12d687"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(3456)
+            .setName("HTTP GET /test")
+            .setSource("source-processtag")
+            .setSpanId("00000000-0000-0000-0000-0051759bfc69")
+            .setTraceId("0000011e-ab2a-9944-0000-000049631900")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "51759bfc69"),
+                    new Annotation("jaegerTraceId", "11eab2a99440000000049631900"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerTChannelCollectorHandler handler =
+        new JaegerTChannelCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Tag ipTag = new Tag("ip", TagType.STRING);
+    ipTag.setVStr("10.0.0.1");
+
+    Tag sourceProcessTag = new Tag("source", TagType.STRING);
+    sourceProcessTag.setVStr("source-processtag");
+
+    Tag customServiceProcessTag = new Tag("service", TagType.STRING);
+    customServiceProcessTag.setVStr("service-processtag");
+
+    Tag customServiceSpanTag = new Tag("service", TagType.STRING);
+    customServiceSpanTag.setVStr("service-spantag");
+
+    io.jaegertracing.thriftjava.Span span1 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L, 1234567890L, 2345678L, 0, "HTTP GET /", 1, startTime * 1000, 9 * 1000);
+    span1.setTags(ImmutableList.of(customServiceSpanTag));
+
+    io.jaegertracing.thriftjava.Span span2 =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L, 1234567890L, 1234567L, 0, "HTTP GET", 1, startTime * 1000, 4 * 1000);
+
+    io.jaegertracing.thriftjava.Span span3 =
+        new io.jaegertracing.thriftjava.Span(
+            1231231232L,
+            1231232342340L,
+            349865507945L,
+            0,
+            "HTTP GET /test",
+            1,
+            startTime * 1000,
+            3456 * 1000);
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+    testBatch.process.setTags(ImmutableList.of(ipTag, sourceProcessTag, customServiceProcessTag));
+
+    testBatch.setSpans(ImmutableList.of(span1, span2));
+
+    Collector.submitBatches_args batches = new Collector.submitBatches_args();
+    batches.addToBatches(testBatch);
+    ThriftRequest request =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batches)
+            .build();
+    handler.handleImpl(request);
+
+    Batch testBatchWithoutProcessTag = new Batch();
+    testBatchWithoutProcessTag.process = new Process();
+    testBatchWithoutProcessTag.process.serviceName = "frontend";
+    testBatchWithoutProcessTag.process.setTags(ImmutableList.of(ipTag, sourceProcessTag));
+    testBatchWithoutProcessTag.setSpans(ImmutableList.of(span3));
+
+    Collector.submitBatches_args batchesWithoutProcessTags = new Collector.submitBatches_args();
+    batchesWithoutProcessTags.addToBatches(testBatchWithoutProcessTag);
+    ThriftRequest requestForProxyLevel =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batchesWithoutProcessTags)
+            .build();
+    handler.handleImpl(requestForProxyLevel);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testProtectedTagsSpanOverridesProcess() throws Exception {
+    // cluster, shard and service are special tags, because they're indexed by wavefront
+    // The priority order is:
+    // Span Level > Process Level > Proxy Level > Default
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("HTTP GET /")
+            .setSource("source-processtag")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "application-spantag"),
+                    new Annotation("cluster", "cluster-spantag"),
+                    new Annotation("shard", "shard-spantag")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerTChannelCollectorHandler handler =
+        new JaegerTChannelCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Tag ipTag = new Tag("ip", TagType.STRING);
+    ipTag.setVStr("10.0.0.1");
+
+    Tag sourceProcessTag = new Tag("source", TagType.STRING);
+    sourceProcessTag.setVStr("source-processtag");
+
+    Tag customApplicationProcessTag = new Tag("application", TagType.STRING);
+    customApplicationProcessTag.setVStr("application-processtag");
+
+    Tag customApplicationSpanTag = new Tag("application", TagType.STRING);
+    customApplicationSpanTag.setVStr("application-spantag");
+
+    Tag customClusterProcessTag = new Tag("cluster", TagType.STRING);
+    customClusterProcessTag.setVStr("cluster-processtag");
+
+    Tag customClusterSpanTag = new Tag("cluster", TagType.STRING);
+    customClusterSpanTag.setVStr("cluster-spantag");
+
+    Tag customShardProcessTag = new Tag("shard", TagType.STRING);
+    customShardProcessTag.setVStr("shard-processtag");
+
+    Tag customShardSpanTag = new Tag("shard", TagType.STRING);
+    customShardSpanTag.setVStr("shard-spantag");
+
+    io.jaegertracing.thriftjava.Span span =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L, 1234567890L, 2345678L, 0, "HTTP GET /", 1, startTime * 1000, 9 * 1000);
+    span.setTags(
+        ImmutableList.of(customApplicationSpanTag, customClusterSpanTag, customShardSpanTag));
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+    testBatch.process.setTags(
+        ImmutableList.of(
+            ipTag,
+            sourceProcessTag,
+            customApplicationProcessTag,
+            customClusterProcessTag,
+            customShardProcessTag));
+
+    testBatch.setSpans(ImmutableList.of(span));
+
+    Collector.submitBatches_args batches = new Collector.submitBatches_args();
+    batches.addToBatches(testBatch);
+    ThriftRequest request =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batches)
+            .build();
+    handler.handleImpl(request);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testProtectedTagsProcessOverridesProxyConfig() throws Exception {
+    // cluster, shard and service are special tags, because they're indexed by wavefront
+    // The priority order is:
+    // Span Level > Process Level > Proxy Level > Default
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("HTTP GET /")
+            .setSource("source-processtag")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("application", "application-processtag"),
+                    new Annotation("cluster", "cluster-processtag"),
+                    new Annotation("shard", "shard-processtag")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerTChannelCollectorHandler handler =
+        new JaegerTChannelCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Tag ipTag = new Tag("ip", TagType.STRING);
+    ipTag.setVStr("10.0.0.1");
+
+    Tag sourceProcessTag = new Tag("source", TagType.STRING);
+    sourceProcessTag.setVStr("source-processtag");
+
+    Tag customApplicationProcessTag = new Tag("application", TagType.STRING);
+    customApplicationProcessTag.setVStr("application-processtag");
+
+    Tag customClusterProcessTag = new Tag("cluster", TagType.STRING);
+    customClusterProcessTag.setVStr("cluster-processtag");
+
+    Tag customShardProcessTag = new Tag("shard", TagType.STRING);
+    customShardProcessTag.setVStr("shard-processtag");
+
+    io.jaegertracing.thriftjava.Span span =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L, 1234567890L, 2345678L, 0, "HTTP GET /", 1, startTime * 1000, 9 * 1000);
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+    testBatch.process.setTags(
+        ImmutableList.of(
+            ipTag,
+            sourceProcessTag,
+            customApplicationProcessTag,
+            customClusterProcessTag,
+            customShardProcessTag));
+
+    testBatch.setSpans(ImmutableList.of(span));
+
+    Collector.submitBatches_args batches = new Collector.submitBatches_args();
+    batches.addToBatches(testBatch);
+    ThriftRequest request =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batches)
+            .build();
+    handler.handleImpl(request);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testAllProcessTagsPropagated() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("HTTP GET /")
+            .setSource("source-spantag")
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("ip", "10.0.0.1"),
+                    new Annotation("processTag1", "one"),
+                    new Annotation("processTag2", "two"),
+                    new Annotation("processTag3", "three"),
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none")))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerTChannelCollectorHandler handler =
+        new JaegerTChannelCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Tag ipTag = new Tag("ip", TagType.STRING);
+    ipTag.setVStr("10.0.0.1");
+
+    Tag hostNameProcessTag = new Tag("hostname", TagType.STRING);
+    hostNameProcessTag.setVStr("hostname-processtag");
+
+    Tag customProcessTag1 = new Tag("processTag1", TagType.STRING);
+    customProcessTag1.setVStr("one");
+
+    Tag customProcessTag2 = new Tag("processTag2", TagType.STRING);
+    customProcessTag2.setVStr("two");
+
+    Tag customProcessTag3 = new Tag("processTag3", TagType.STRING);
+    customProcessTag3.setVStr("three");
+
+    Tag customSourceSpanTag = new Tag("source", TagType.STRING);
+    customSourceSpanTag.setVStr("source-spantag");
+
+    io.jaegertracing.thriftjava.Span span =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            2345678L,
+            1234567L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            9 * 1000);
+    span.setTags(ImmutableList.of(customSourceSpanTag));
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+    testBatch.process.setTags(
+        ImmutableList.of(
+            ipTag, hostNameProcessTag, customProcessTag1, customProcessTag2, customProcessTag3));
+
+    testBatch.setSpans(ImmutableList.of(span));
+
+    Collector.submitBatches_args batches = new Collector.submitBatches_args();
+    batches.addToBatches(testBatch);
+    ThriftRequest request =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batches)
+            .build();
+    handler.handleImpl(request);
+
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+
+  @Test
+  public void testJaegerSamplerSync() throws Exception {
+    reset(mockTraceHandler, mockTraceLogsHandler);
+
+    Span expectedSpan =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("HTTP GET /")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-0000-00000023cace")
+            .setTraceId("00000000-4996-02d2-0000-011f71fb04cb")
+            // Note: Order of annotations list matters for this unit test.
+            .setAnnotations(
+                ImmutableList.of(
+                    new Annotation("jaegerSpanId", "23cace"),
+                    new Annotation("jaegerTraceId", "499602d20000011f71fb04cb"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("parent", "00000000-0000-0000-0000-00000012d687"),
+                    new Annotation("application", "Jaeger"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("_spanLogs", "true")))
+            .build();
+    mockTraceHandler.report(expectedSpan);
+    expectLastCall();
+
+    Capture spanLogsCapture = newCapture();
+    mockTraceLogsHandler.report(capture(spanLogsCapture));
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceLogsHandler);
+
+    JaegerTChannelCollectorHandler handler =
+        new JaegerTChannelCollectorHandler(
+            "9876",
+            mockTraceHandler,
+            mockTraceLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new DurationSampler(5), () -> null),
+            null,
+            null);
+
+    io.jaegertracing.thriftjava.Span span =
+        new io.jaegertracing.thriftjava.Span(
+            1234567890123L,
+            1234567890L,
+            2345678L,
+            1234567L,
+            "HTTP GET /",
+            1,
+            startTime * 1000,
+            9 * 1000);
+
+    Tag tag = new Tag("event", TagType.STRING);
+    tag.setVStr("error");
+
+    span.setLogs(ImmutableList.of(new Log(startTime * 1000, ImmutableList.of(tag))));
+
+    Batch testBatch = new Batch();
+    testBatch.process = new Process();
+    testBatch.process.serviceName = "frontend";
+
+    testBatch.setSpans(ImmutableList.of(span));
+
+    Collector.submitBatches_args batches = new Collector.submitBatches_args();
+    batches.addToBatches(testBatch);
+    ThriftRequest request =
+        new ThriftRequest.Builder(
+                "jaeger-collector", "Collector::submitBatches")
+            .setBody(batches)
+            .build();
+    handler.handleImpl(request);
+
+    assertEquals("_sampledByPolicy=NONE", spanLogsCapture.getValue().getSpan());
+    verify(mockTraceHandler, mockTraceLogsHandler);
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/tracing/SpanUtilsTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/SpanUtilsTest.java
new file mode 100644
index 000000000..24ab13c77
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/SpanUtilsTest.java
@@ -0,0 +1,407 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.agent.listeners.tracing.SpanUtils.*;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.ImmutableList;
+import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.LineBasedAllowFilter;
+import com.wavefront.api.agent.preprocessor.LineBasedBlockFilter;
+import com.wavefront.api.agent.preprocessor.PreprocessorRuleMetrics;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.preprocessor.SpanBlockFilter;
+import com.wavefront.api.agent.ValidationConfiguration;
+import com.wavefront.ingester.ReportableEntityDecoder;
+import com.wavefront.ingester.SpanDecoder;
+import com.wavefront.ingester.SpanLogsDecoder;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.function.Supplier;
+import org.junit.Before;
+import org.junit.Test;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
+
+/**
+ * Unit tests for {@link SpanUtils}.
+ *
+ * @author Shipeng Xie (xshipeng@vmware.com)
+ */
+public class SpanUtilsTest {
+  private ReportableEntityDecoder spanDecoder = new SpanDecoder("localdev");
+  private ReportableEntityDecoder spanLogsDocoder = new SpanLogsDecoder();
+
+  private ReportableEntityHandler mockTraceHandler =
+      MockReportableEntityHandlerFactory.getMockTraceHandler();
+  private ReportableEntityHandler mockTraceSpanLogsHandler =
+      MockReportableEntityHandlerFactory.getMockTraceSpanLogsHandler();
+  private ValidationConfiguration validationConfiguration = new ValidationConfiguration();
+  private long startTime = System.currentTimeMillis();
+
+  @Before
+  public void setUp() {
+    reset(mockTraceHandler, mockTraceSpanLogsHandler);
+  }
+
+  @Test
+  public void testSpanLineDataBlockPreprocessor() {
+    Supplier preprocessorSupplier =
+        () -> {
+          ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+          PreprocessorRuleMetrics preprocessorRuleMetrics =
+              new PreprocessorRuleMetrics(null, null, null);
+          preprocessor
+              .forPointLine()
+              .addFilter(new LineBasedAllowFilter("^valid.*", preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addFilter(
+                  new SpanBlockFilter(SERVICE_TAG_KEY, "^test.*", null, preprocessorRuleMetrics));
+          return preprocessor;
+        };
+    String spanLine =
+        "\"valid.metric\" \"source\"=\"localdev\" "
+            + "\"spanId\"=\"4217104a-690d-4927-baff-d9aa779414c2\" "
+            + "\"traceId\"=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" "
+            + "\"application\"=\"app\" \"service\"=\"svc\" "
+            + startTime
+            + " 100";
+
+    mockTraceHandler.block(null, spanLine);
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+    preprocessAndHandleSpan(
+        spanLine,
+        spanDecoder,
+        mockTraceHandler,
+        mockTraceHandler::report,
+        preprocessorSupplier,
+        null,
+        span -> true);
+    verify(mockTraceHandler);
+  }
+
+  @Test
+  public void testSpanDecodeRejectPreprocessor() {
+    String spanLine =
+        "\"valid.metric\" \"source\"=\"localdev\" "
+            + "\"spanId\"=\"4217104a-690d-4927-baff-d9aa779414c2\" "
+            + "\"traceId\"=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" "
+            + "\"application\"=\"app\" \"service\"=\"svc\" "
+            + startTime;
+
+    mockTraceHandler.reject(
+        spanLine, spanLine + "; reason: \"Expected timestamp, found end of " + "line\"");
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+    preprocessAndHandleSpan(
+        spanLine,
+        spanDecoder,
+        mockTraceHandler,
+        mockTraceHandler::report,
+        null,
+        null,
+        span -> true);
+    verify(mockTraceHandler);
+  }
+
+  @Test
+  public void testSpanTagBlockPreprocessor() {
+    Supplier preprocessorSupplier =
+        () -> {
+          ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+          PreprocessorRuleMetrics preprocessorRuleMetrics =
+              new PreprocessorRuleMetrics(null, null, null);
+          preprocessor
+              .forSpan()
+              .addFilter(
+                  new SpanBlockFilter(SERVICE_TAG_KEY, "^test.*", null, preprocessorRuleMetrics));
+          return preprocessor;
+        };
+    String spanLine =
+        "\"valid.metric\" \"source\"=\"localdev\" "
+            + "\"spanId\"=\"4217104a-690d-4927-baff-d9aa779414c2\" "
+            + "\"traceId\"=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" "
+            + "\"application\"=\"app\" \"service\"=\"test\" "
+            + startTime
+            + " 100";
+
+    mockTraceHandler.block(
+        new Span(
+            "valid.metric",
+            "4217104a-690d-4927-baff" + "-d9aa779414c2",
+            "d5355bf7-fc8d-48d1-b761-75b170f396e0",
+            startTime,
+            100L,
+            "localdev",
+            "dummy",
+            ImmutableList.of(
+                new Annotation("application", "app"), new Annotation("service", "test"))));
+
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+    preprocessAndHandleSpan(
+        spanLine,
+        spanDecoder,
+        mockTraceHandler,
+        mockTraceHandler::report,
+        preprocessorSupplier,
+        null,
+        span -> true);
+    verify(mockTraceHandler);
+  }
+
+  @Test
+  public void testSpanLogsLineDataBlockPreprocessor() {
+    Supplier preprocessorSupplier =
+        () -> {
+          ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+          PreprocessorRuleMetrics preprocessorRuleMetrics =
+              new PreprocessorRuleMetrics(null, null, null);
+          preprocessor
+              .forPointLine()
+              .addFilter(new LineBasedBlockFilter(".*invalid.*", preprocessorRuleMetrics));
+          return preprocessor;
+        };
+
+    String spanLine =
+        "\"invalid.metric\" \"source\"=\"localdev\" "
+            + "\"spanId\"=\"4217104a-690d-4927-baff-d9aa779414c2\" "
+            + "\"traceId\"=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" "
+            + "\"application\"=\"app\" \"service\"=\"svc\" "
+            + startTime
+            + " 100";
+    String spanLogsLine =
+        "{"
+            + "\"customer\":\"dummy\","
+            + "\"traceId\":\"d5355bf7-fc8d-48d1-b761-75b170f396e0\","
+            + "\"spanId\":\"4217104a-690d-4927-baff-d9aa779414c2\","
+            + "\"logs\":[{\"timestamp\":"
+            + startTime
+            + ",\"fields\":{\"error"
+            + ".kind\":\"exception\", \"event\":\"error\"}}],"
+            + "\"span\":\"\\\"invalid.metric\\\" \\\"source\\\"=\\\"localdev\\\" "
+            + "\\\"spanId\\\"=\\\"4217104a-690d-4927-baff-d9aa779414c2\\\" "
+            + "\\\"traceId\\\"=\\\"d5355bf7-fc8d-48d1-b761-75b170f396e0\\\" "
+            + "\\\"application\\\"=\\\"app\\\" \\\"service\\\"=\\\"svc\\\" "
+            + startTime
+            + " 100\""
+            + "}";
+    SpanLogs spanLogs =
+        SpanLogs.newBuilder()
+            .setSpan(spanLine)
+            .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0")
+            .setSpanId("4217104a-690d-4927-baff-d9aa779414c2")
+            .setCustomer("dummy")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setFields(
+                            new HashMap() {
+                              {
+                                put("error.kind", "exception");
+                                put("event", "error");
+                              }
+                            })
+                        .setTimestamp(startTime)
+                        .build()))
+            .build();
+    mockTraceSpanLogsHandler.block(spanLogs);
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+    handleSpanLogs(
+        spanLogsLine,
+        spanLogsDocoder,
+        spanDecoder,
+        mockTraceSpanLogsHandler,
+        preprocessorSupplier,
+        null,
+        span -> true);
+    verify(mockTraceSpanLogsHandler);
+  }
+
+  @Test
+  public void testSpanLogsTagBlockPreprocessor() {
+    Supplier preprocessorSupplier =
+        () -> {
+          ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+          PreprocessorRuleMetrics preprocessorRuleMetrics =
+              new PreprocessorRuleMetrics(null, null, null);
+          preprocessor
+              .forSpan()
+              .addFilter(
+                  new SpanBlockFilter(SERVICE_TAG_KEY, "^test.*", null, preprocessorRuleMetrics));
+          return preprocessor;
+        };
+
+    String spanLine =
+        "\"invalid.metric\" \"source\"=\"localdev\" "
+            + "\"spanId\"=\"4217104a-690d-4927-baff-d9aa779414c2\" "
+            + "\"traceId\"=\"d5355bf7-fc8d-48d1-b761-75b170f396e0\" "
+            + "\"application\"=\"app\" \"service\"=\"test\" "
+            + startTime
+            + " 100";
+    String spanLogsLine =
+        "{"
+            + "\"customer\":\"dummy\","
+            + "\"traceId\":\"d5355bf7-fc8d-48d1-b761-75b170f396e0\","
+            + "\"spanId\":\"4217104a-690d-4927-baff-d9aa779414c2\","
+            + "\"logs\":[{\"timestamp\":"
+            + startTime
+            + ",\"fields\":{\"error"
+            + ".kind\":\"exception\", \"event\":\"error\"}}],"
+            + "\"span\":\"\\\"invalid.metric\\\" \\\"source\\\"=\\\"localdev\\\" "
+            + "\\\"spanId\\\"=\\\"4217104a-690d-4927-baff-d9aa779414c2\\\" "
+            + "\\\"traceId\\\"=\\\"d5355bf7-fc8d-48d1-b761-75b170f396e0\\\" "
+            + "\\\"application\\\"=\\\"app\\\" \\\"service\\\"=\\\"test\\\" "
+            + startTime
+            + " 100\""
+            + "}";
+    SpanLogs spanLogs =
+        SpanLogs.newBuilder()
+            .setSpan(spanLine)
+            .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0")
+            .setSpanId("4217104a-690d-4927-baff-d9aa779414c2")
+            .setCustomer("dummy")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setFields(
+                            new HashMap() {
+                              {
+                                put("error.kind", "exception");
+                                put("event", "error");
+                              }
+                            })
+                        .setTimestamp(startTime)
+                        .build()))
+            .build();
+    mockTraceSpanLogsHandler.block(spanLogs);
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+    handleSpanLogs(
+        spanLogsLine,
+        spanLogsDocoder,
+        spanDecoder,
+        mockTraceSpanLogsHandler,
+        preprocessorSupplier,
+        null,
+        span -> true);
+    verify(mockTraceSpanLogsHandler);
+  }
+
+  @Test
+  public void testSpanLogsReport() {
+    String spanLogsLine =
+        "{"
+            + "\"customer\":\"dummy\","
+            + "\"traceId\":\"d5355bf7-fc8d-48d1-b761-75b170f396e0\","
+            + "\"spanId\":\"4217104a-690d-4927-baff-d9aa779414c2\","
+            + "\"logs\":[{\"timestamp\":"
+            + startTime
+            + ",\"fields\":{\"error"
+            + ".kind\":\"exception\", \"event\":\"error\"}}],"
+            + "\"span\":\"\\\"valid.metric\\\" \\\"source\\\"=\\\"localdev\\\" "
+            + "\\\"spanId\\\"=\\\"4217104a-690d-4927-baff-d9aa779414c2\\\" "
+            + "\\\"traceId\\\"=\\\"d5355bf7-fc8d-48d1-b761-75b170f396e0\\\" "
+            + "\\\"application\\\"=\\\"app\\\" \\\"service\\\"=\\\"test\\\" "
+            + startTime
+            + " 100\""
+            + "}";
+    SpanLogs spanLogs =
+        SpanLogs.newBuilder()
+            .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0")
+            .setSpanId("4217104a-690d-4927-baff-d9aa779414c2")
+            .setCustomer("dummy")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setFields(
+                            new HashMap() {
+                              {
+                                put("error.kind", "exception");
+                                put("event", "error");
+                              }
+                            })
+                        .setTimestamp(startTime)
+                        .build()))
+            .build();
+    mockTraceSpanLogsHandler.report(spanLogs);
+    expectLastCall();
+
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+    handleSpanLogs(
+        spanLogsLine,
+        spanLogsDocoder,
+        spanDecoder,
+        mockTraceSpanLogsHandler,
+        null,
+        null,
+        span -> true);
+    verify(mockTraceSpanLogsHandler);
+  }
+
+  @Test
+  public void testAddSpanLineWithPolicy() {
+    Span span =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0")
+            .setSpanId("4217104a-690d-4927-baff-d9aa779414c2")
+            .setName("spanName")
+            .setStartMillis(0L)
+            .setDuration(0L)
+            .setAnnotations(Collections.singletonList(new Annotation("_sampledByPolicy", "test")))
+            .build();
+    SpanLogs spanLogs =
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0")
+            .setSpanId("4217104a-690d-4927-baff-d9aa779414c2")
+            .setLogs(Collections.singletonList(SpanLog.newBuilder().setTimestamp(0L).build()))
+            .build();
+
+    addSpanLine(span, spanLogs);
+
+    assertEquals("_sampledByPolicy=test", spanLogs.getSpan());
+  }
+
+  @Test
+  public void testAddSpanLineWithoutPolicy() {
+    Span span =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0")
+            .setSpanId("4217104a-690d-4927-baff-d9aa779414c2")
+            .setName("spanName")
+            .setStartMillis(0L)
+            .setDuration(0L)
+            .build();
+    SpanLogs spanLogs =
+        SpanLogs.newBuilder()
+            .setCustomer("dummy")
+            .setTraceId("d5355bf7-fc8d-48d1-b761-75b170f396e0")
+            .setSpanId("4217104a-690d-4927-baff-d9aa779414c2")
+            .setLogs(Collections.singletonList(SpanLog.newBuilder().setTimestamp(0L).build()))
+            .build();
+
+    addSpanLine(span, spanLogs);
+
+    assertEquals("_sampledByPolicy=NONE", spanLogs.getSpan());
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandlerTest.java b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandlerTest.java
new file mode 100644
index 000000000..cf0fe7672
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/listeners/tracing/ZipkinPortUnificationHandlerTest.java
@@ -0,0 +1,922 @@
+package com.wavefront.agent.listeners.tracing;
+
+import static com.wavefront.agent.TestUtils.verifyWithTimeout;
+import static com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.HEART_BEAT_METRIC;
+import static com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY;
+import static com.wavefront.sdk.common.Constants.SHARD_TAG_KEY;
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.wavefront.agent.channel.NoopHealthCheckManager;
+import com.wavefront.agent.handlers.MockReportableEntityHandlerFactory;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.api.agent.preprocessor.PreprocessorRuleMetrics;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.preprocessor.SpanReplaceRegexTransformer;
+import com.wavefront.agent.sampler.SpanSampler;
+import com.wavefront.sdk.common.WavefrontSender;
+import com.wavefront.sdk.entities.tracing.sampling.DurationSampler;
+import com.wavefront.sdk.entities.tracing.sampling.RateSampler;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.http.DefaultFullHttpRequest;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpVersion;
+import java.util.HashMap;
+import java.util.List;
+import java.util.function.Supplier;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+import wavefront.report.SpanLog;
+import wavefront.report.SpanLogs;
+import zipkin2.Endpoint;
+import zipkin2.codec.SpanBytesEncoder;
+
+public class ZipkinPortUnificationHandlerTest {
+  private static final String DEFAULT_SOURCE = "zipkin";
+  private ReportableEntityHandler mockTraceHandler =
+      MockReportableEntityHandlerFactory.getMockTraceHandler();
+  private ReportableEntityHandler mockTraceSpanLogsHandler =
+      MockReportableEntityHandlerFactory.getMockTraceSpanLogsHandler();
+  private WavefrontSender mockWavefrontSender = EasyMock.createMock(WavefrontSender.class);
+  private long startTime = System.currentTimeMillis();
+
+  // Derived RED metrics related.
+  private final String PREPROCESSED_APPLICATION_TAG_VALUE = "preprocessedApplication";
+  private final String PREPROCESSED_SERVICE_TAG_VALUE = "preprocessedService";
+  private final String PREPROCESSED_CLUSTER_TAG_VALUE = "preprocessedCluster";
+  private final String PREPROCESSED_SHARD_TAG_VALUE = "preprocessedShard";
+  private final String PREPROCESSED_SOURCE_VALUE = "preprocessedSource";
+
+  /**
+   * Test for derived metrics emitted from Zipkin trace listeners. Derived metrics should report tag
+   * values post applying preprocessing rules to the span.
+   */
+  @Test
+  public void testZipkinPreprocessedDerivedMetrics() throws Exception {
+    Supplier preprocessorSupplier =
+        () -> {
+          ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
+          PreprocessorRuleMetrics preprocessorRuleMetrics =
+              new PreprocessorRuleMetrics(null, null, null);
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      APPLICATION_TAG_KEY,
+                      "^Zipkin.*",
+                      PREPROCESSED_APPLICATION_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      SERVICE_TAG_KEY,
+                      "^test.*",
+                      PREPROCESSED_SERVICE_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      "sourceName",
+                      "^zipkin.*",
+                      PREPROCESSED_SOURCE_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      CLUSTER_TAG_KEY,
+                      "^none.*",
+                      PREPROCESSED_CLUSTER_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          preprocessor
+              .forSpan()
+              .addTransformer(
+                  new SpanReplaceRegexTransformer(
+                      SHARD_TAG_KEY,
+                      "^none.*",
+                      PREPROCESSED_SHARD_TAG_VALUE,
+                      null,
+                      null,
+                      false,
+                      x -> true,
+                      preprocessorRuleMetrics));
+          return preprocessor;
+        };
+
+    ZipkinPortUnificationHandler handler =
+        new ZipkinPortUnificationHandler(
+            "9411",
+            new NoopHealthCheckManager(),
+            mockTraceHandler,
+            mockTraceSpanLogsHandler,
+            mockWavefrontSender,
+            () -> false,
+            () -> false,
+            preprocessorSupplier,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    Endpoint localEndpoint1 =
+        Endpoint.newBuilder().serviceName("testService").ip("10.0.0.1").build();
+    zipkin2.Span spanServer1 =
+        zipkin2.Span.newBuilder()
+            .traceId("2822889fe47043bd")
+            .id("2822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getservice")
+            .timestamp(startTime * 1000)
+            .duration(1234 * 1000)
+            .localEndpoint(localEndpoint1)
+            .build();
+
+    List zipkinSpanList = ImmutableList.of(spanServer1);
+
+    // Reset mock
+    reset(mockTraceHandler, mockWavefrontSender);
+
+    // Set Expectation
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(1234)
+            .setName("getservice")
+            .setSource(PREPROCESSED_SOURCE_VALUE)
+            .setSpanId("00000000-0000-0000-2822-889fe47043bd")
+            .setTraceId("00000000-0000-0000-2822-889fe47043bd")
+            .
+            // Note: Order of annotations list matters for this unit test.
+            setAnnotations(
+                ImmutableList.of(
+                    new Annotation("zipkinSpanId", "2822889fe47043bd"),
+                    new Annotation("zipkinTraceId", "2822889fe47043bd"),
+                    new Annotation("span.kind", "server"),
+                    new Annotation("service", PREPROCESSED_SERVICE_TAG_VALUE),
+                    new Annotation("application", PREPROCESSED_APPLICATION_TAG_VALUE),
+                    new Annotation("cluster", PREPROCESSED_CLUSTER_TAG_VALUE),
+                    new Annotation("shard", PREPROCESSED_SHARD_TAG_VALUE),
+                    new Annotation("ipv4", "10.0.0.1")))
+            .build());
+    expectLastCall();
+
+    Capture> tagsCapture = newCapture();
+
+    mockWavefrontSender.sendMetric(
+        eq(HEART_BEAT_METRIC),
+        eq(1.0),
+        anyLong(),
+        eq(PREPROCESSED_SOURCE_VALUE),
+        EasyMock.capture(tagsCapture));
+    expectLastCall().anyTimes();
+    replay(mockTraceHandler, mockWavefrontSender);
+
+    ChannelHandlerContext mockCtx = createNiceMock(ChannelHandlerContext.class);
+    doMockLifecycle(mockCtx);
+
+    ByteBuf content = Unpooled.copiedBuffer(SpanBytesEncoder.JSON_V2.encodeList(zipkinSpanList));
+    FullHttpRequest httpRequest =
+        new DefaultFullHttpRequest(
+            HttpVersion.HTTP_1_1,
+            HttpMethod.POST,
+            "http://localhost:9411/api/v2/spans",
+            content,
+            true);
+    handler.handleHttpMessage(mockCtx, httpRequest);
+    handler.run();
+
+    verifyWithTimeout(500, mockTraceHandler, mockWavefrontSender);
+    HashMap tagsReturned = tagsCapture.getValue();
+    assertEquals(PREPROCESSED_APPLICATION_TAG_VALUE, tagsReturned.get(APPLICATION_TAG_KEY));
+    assertEquals(PREPROCESSED_SERVICE_TAG_VALUE, tagsReturned.get(SERVICE_TAG_KEY));
+    assertEquals(PREPROCESSED_CLUSTER_TAG_VALUE, tagsReturned.get(CLUSTER_TAG_KEY));
+    assertEquals(PREPROCESSED_SHARD_TAG_VALUE, tagsReturned.get(SHARD_TAG_KEY));
+  }
+
+  @Test
+  public void testZipkinHandler() throws Exception {
+    ZipkinPortUnificationHandler handler =
+        new ZipkinPortUnificationHandler(
+            "9411",
+            new NoopHealthCheckManager(),
+            mockTraceHandler,
+            mockTraceSpanLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            "ProxyLevelAppTag",
+            null);
+
+    Endpoint localEndpoint1 = Endpoint.newBuilder().serviceName("frontend").ip("10.0.0.1").build();
+    zipkin2.Span spanServer1 =
+        zipkin2.Span.newBuilder()
+            .traceId("2822889fe47043bd")
+            .id("2822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getservice")
+            .timestamp(startTime * 1000)
+            .duration(1234 * 1000)
+            .localEndpoint(localEndpoint1)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h1c://localhost:8881/")
+            .putTag("http.status_code", "200")
+            .build();
+
+    Endpoint localEndpoint2 = Endpoint.newBuilder().serviceName("backend").ip("10.0.0.1").build();
+    zipkin2.Span spanServer2 =
+        zipkin2.Span.newBuilder()
+            .traceId("2822889fe47043bd")
+            .id("d6ab73f8a3930ae8")
+            .parentId("2822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getbackendservice")
+            .timestamp(startTime * 1000)
+            .duration(2234 * 1000)
+            .localEndpoint(localEndpoint2)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h2c://localhost:9000/api")
+            .putTag("http.status_code", "200")
+            .putTag("component", "jersey-server")
+            .putTag("application", "SpanLevelAppTag")
+            .addAnnotation(startTime * 1000, "start processing")
+            .build();
+
+    zipkin2.Span spanServer3 =
+        zipkin2.Span.newBuilder()
+            .traceId("2822889fe47043bd")
+            .id("d6ab73f8a3930ae8")
+            .kind(zipkin2.Span.Kind.CLIENT)
+            .name("getbackendservice2")
+            .timestamp(startTime * 1000)
+            .duration(2234 * 1000)
+            .localEndpoint(localEndpoint2)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h2c://localhost:9000/api")
+            .putTag("http.status_code", "200")
+            .putTag("component", "jersey-server")
+            .putTag("application", "SpanLevelAppTag")
+            .putTag("emptry.tag", "")
+            .addAnnotation(startTime * 1000, "start processing")
+            .build();
+
+    List zipkinSpanList = ImmutableList.of(spanServer1, spanServer2, spanServer3);
+
+    // Validate all codecs i.e. JSON_V1, JSON_V2, THRIFT and PROTO3.
+    for (SpanBytesEncoder encoder : SpanBytesEncoder.values()) {
+      ByteBuf content = Unpooled.copiedBuffer(encoder.encodeList(zipkinSpanList));
+      // take care of mocks.
+      doMockLifecycle(mockTraceHandler, mockTraceSpanLogsHandler);
+      ChannelHandlerContext mockCtx = createNiceMock(ChannelHandlerContext.class);
+      doMockLifecycle(mockCtx);
+      FullHttpRequest httpRequest =
+          new DefaultFullHttpRequest(
+              HttpVersion.HTTP_1_1,
+              HttpMethod.POST,
+              "http://localhost:9411/api/v1/spans",
+              content,
+              true);
+      handler.handleHttpMessage(mockCtx, httpRequest);
+      verify(mockTraceHandler, mockTraceSpanLogsHandler);
+    }
+  }
+
+  private void doMockLifecycle(ChannelHandlerContext mockCtx) {
+    reset(mockCtx);
+    EasyMock.expect(mockCtx.write(EasyMock.isA(FullHttpResponse.class))).andReturn(null);
+    EasyMock.replay(mockCtx);
+  }
+
+  private void doMockLifecycle(
+      ReportableEntityHandler mockTraceHandler,
+      ReportableEntityHandler mockTraceSpanLogsHandler) {
+    // Reset mock
+    reset(mockTraceHandler, mockTraceSpanLogsHandler);
+
+    // Set Expectation
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(1234)
+            .setName("getservice")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-2822-889fe47043bd")
+            .setTraceId("00000000-0000-0000-2822-889fe47043bd")
+            .
+            // Note: Order of annotations list matters for this unit test.
+            setAnnotations(
+                ImmutableList.of(
+                    new Annotation("zipkinSpanId", "2822889fe47043bd"),
+                    new Annotation("zipkinTraceId", "2822889fe47043bd"),
+                    new Annotation("span.kind", "server"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("http.method", "GET"),
+                    new Annotation("http.status_code", "200"),
+                    new Annotation("http.url", "none+h1c://localhost:8881/"),
+                    new Annotation("application", "ProxyLevelAppTag"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("ipv4", "10.0.0.1")))
+            .build());
+    expectLastCall();
+
+    Span span1 =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(2234)
+            .setName("getbackendservice")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-d6ab-73f8a3930ae8")
+            .setTraceId("00000000-0000-0000-2822-889fe47043bd")
+            .
+            // Note: Order of annotations list matters for this unit test.
+            setAnnotations(
+                ImmutableList.of(
+                    new Annotation("zipkinSpanId", "d6ab73f8a3930ae8"),
+                    new Annotation("zipkinTraceId", "2822889fe47043bd"),
+                    new Annotation("parent", "00000000-0000-0000-2822-889fe47043bd"),
+                    new Annotation("span.kind", "server"),
+                    new Annotation("_spanSecondaryId", "server"),
+                    new Annotation("service", "backend"),
+                    new Annotation("component", "jersey-server"),
+                    new Annotation("http.method", "GET"),
+                    new Annotation("http.status_code", "200"),
+                    new Annotation("http.url", "none+h2c://localhost:9000/api"),
+                    new Annotation("application", "SpanLevelAppTag"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("ipv4", "10.0.0.1"),
+                    new Annotation("_spanLogs", "true")))
+            .build();
+    mockTraceHandler.report(span1);
+    expectLastCall();
+
+    Span span2 =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(2234)
+            .setName("getbackendservice2")
+            .setSource(DEFAULT_SOURCE)
+            .setTraceId("00000000-0000-0000-2822-889fe47043bd")
+            .setSpanId("00000000-0000-0000-d6ab-73f8a3930ae8")
+            .
+            // Note: Order of annotations list matters for this unit test.
+            setAnnotations(
+                ImmutableList.of(
+                    new Annotation("zipkinSpanId", "d6ab73f8a3930ae8"),
+                    new Annotation("zipkinTraceId", "2822889fe47043bd"),
+                    new Annotation("span.kind", "client"),
+                    new Annotation("_spanSecondaryId", "client"),
+                    new Annotation("service", "backend"),
+                    new Annotation("component", "jersey-server"),
+                    new Annotation("http.method", "GET"),
+                    new Annotation("http.status_code", "200"),
+                    new Annotation("http.url", "none+h2c://localhost:9000/api"),
+                    new Annotation("application", "SpanLevelAppTag"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("ipv4", "10.0.0.1"),
+                    new Annotation("_spanLogs", "true")))
+            .build();
+    mockTraceHandler.report(span2);
+    expectLastCall();
+
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("default")
+            .setTraceId("00000000-0000-0000-2822-889fe47043bd")
+            .setSpanId("00000000-0000-0000-d6ab-73f8a3930ae8")
+            .setSpanSecondaryId("server")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(startTime * 1000)
+                        .setFields(ImmutableMap.of("annotation", "start processing"))
+                        .build()))
+            .build());
+    expectLastCall();
+
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("default")
+            .setTraceId("00000000-0000-0000-2822-889fe47043bd")
+            .setSpanId("00000000-0000-0000-d6ab-73f8a3930ae8")
+            .setSpanSecondaryId("client")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(startTime * 1000)
+                        .setFields(ImmutableMap.of("annotation", "start processing"))
+                        .build()))
+            .build());
+    expectLastCall();
+
+    // Replay
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+  }
+
+  @Test
+  public void testZipkinDurationSampler() throws Exception {
+    ZipkinPortUnificationHandler handler =
+        new ZipkinPortUnificationHandler(
+            "9411",
+            new NoopHealthCheckManager(),
+            mockTraceHandler,
+            mockTraceSpanLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new DurationSampler(5), () -> null),
+            null,
+            null);
+
+    Endpoint localEndpoint1 = Endpoint.newBuilder().serviceName("frontend").ip("10.0.0.1").build();
+    zipkin2.Span spanServer1 =
+        zipkin2.Span.newBuilder()
+            .traceId("2822889fe47043bd")
+            .id("2822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getservice")
+            .timestamp(startTime * 1000)
+            .duration(4 * 1000)
+            .localEndpoint(localEndpoint1)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h1c://localhost:8881/")
+            .putTag("http.status_code", "200")
+            .addAnnotation(startTime * 1000, "start processing")
+            .build();
+
+    zipkin2.Span spanServer2 =
+        zipkin2.Span.newBuilder()
+            .traceId("3822889fe47043bd")
+            .id("3822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getservice")
+            .timestamp(startTime * 1000)
+            .duration(9 * 1000)
+            .localEndpoint(localEndpoint1)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h1c://localhost:8881/")
+            .putTag("http.status_code", "200")
+            .addAnnotation(startTime * 1000, "start processing")
+            .build();
+
+    List zipkinSpanList = ImmutableList.of(spanServer1, spanServer2);
+
+    SpanBytesEncoder encoder = SpanBytesEncoder.values()[1];
+    ByteBuf content = Unpooled.copiedBuffer(encoder.encodeList(zipkinSpanList));
+    // take care of mocks.
+    // Reset mock
+    reset(mockTraceHandler, mockTraceSpanLogsHandler);
+
+    // Set Expectation
+    Span expectedSpan2 =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("getservice")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-3822-889fe47043bd")
+            .setTraceId("00000000-0000-0000-3822-889fe47043bd")
+            .
+            // Note: Order of annotations list matters for this unit test.
+            setAnnotations(
+                ImmutableList.of(
+                    new Annotation("zipkinSpanId", "3822889fe47043bd"),
+                    new Annotation("zipkinTraceId", "3822889fe47043bd"),
+                    new Annotation("span.kind", "server"),
+                    new Annotation("_spanSecondaryId", "server"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("http.method", "GET"),
+                    new Annotation("http.status_code", "200"),
+                    new Annotation("http.url", "none+h1c://localhost:8881/"),
+                    new Annotation("application", "Zipkin"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("ipv4", "10.0.0.1"),
+                    new Annotation("_spanLogs", "true")))
+            .build();
+    mockTraceHandler.report(expectedSpan2);
+    expectLastCall();
+
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("default")
+            .setTraceId("00000000-0000-0000-3822-889fe47043bd")
+            .setSpanId("00000000-0000-0000-3822-889fe47043bd")
+            .setSpanSecondaryId("server")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(startTime * 1000)
+                        .setFields(ImmutableMap.of("annotation", "start processing"))
+                        .build()))
+            .build());
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+
+    ChannelHandlerContext mockCtx = createNiceMock(ChannelHandlerContext.class);
+    doMockLifecycle(mockCtx);
+    FullHttpRequest httpRequest =
+        new DefaultFullHttpRequest(
+            HttpVersion.HTTP_1_1,
+            HttpMethod.POST,
+            "http://localhost:9411/api/v1/spans",
+            content,
+            true);
+    handler.handleHttpMessage(mockCtx, httpRequest);
+    verify(mockTraceHandler, mockTraceSpanLogsHandler);
+  }
+
+  @Test
+  public void testZipkinSamplerSync() throws Exception {
+    ZipkinPortUnificationHandler handler =
+        new ZipkinPortUnificationHandler(
+            "9411",
+            new NoopHealthCheckManager(),
+            mockTraceHandler,
+            mockTraceSpanLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new DurationSampler(5), () -> null),
+            null,
+            null);
+
+    Endpoint localEndpoint1 = Endpoint.newBuilder().serviceName("frontend").ip("10.0.0.1").build();
+
+    zipkin2.Span span =
+        zipkin2.Span.newBuilder()
+            .traceId("3822889fe47043bd")
+            .id("3822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getservice")
+            .timestamp(startTime * 1000)
+            .duration(9 * 1000)
+            .localEndpoint(localEndpoint1)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h1c://localhost:8881/")
+            .putTag("http.status_code", "200")
+            .addAnnotation(startTime * 1000, "start processing")
+            .build();
+
+    List zipkinSpanList = ImmutableList.of(span);
+
+    SpanBytesEncoder encoder = SpanBytesEncoder.values()[1];
+    ByteBuf content = Unpooled.copiedBuffer(encoder.encodeList(zipkinSpanList));
+
+    reset(mockTraceHandler, mockTraceSpanLogsHandler);
+
+    Span expectedSpan =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("getservice")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-3822-889fe47043bd")
+            .setTraceId("00000000-0000-0000-3822-889fe47043bd")
+            .
+            // Note: Order of annotations list matters for this unit test.
+            setAnnotations(
+                ImmutableList.of(
+                    new Annotation("zipkinSpanId", "3822889fe47043bd"),
+                    new Annotation("zipkinTraceId", "3822889fe47043bd"),
+                    new Annotation("span.kind", "server"),
+                    new Annotation("_spanSecondaryId", "server"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("http.method", "GET"),
+                    new Annotation("http.status_code", "200"),
+                    new Annotation("http.url", "none+h1c://localhost:8881/"),
+                    new Annotation("application", "Zipkin"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("ipv4", "10.0.0.1"),
+                    new Annotation("_spanLogs", "true")))
+            .build();
+    mockTraceHandler.report(expectedSpan);
+    expectLastCall();
+    Capture capture = newCapture();
+    mockTraceSpanLogsHandler.report(capture(capture));
+    expectLastCall();
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+
+    ChannelHandlerContext mockCtx = createNiceMock(ChannelHandlerContext.class);
+    doMockLifecycle(mockCtx);
+    FullHttpRequest httpRequest =
+        new DefaultFullHttpRequest(
+            HttpVersion.HTTP_1_1,
+            HttpMethod.POST,
+            "http://localhost:9411/api/v1/spans",
+            content,
+            true);
+    handler.handleHttpMessage(mockCtx, httpRequest);
+    assertEquals("_sampledByPolicy=NONE", capture.getValue().getSpan());
+    verify(mockTraceHandler, mockTraceSpanLogsHandler);
+  }
+
+  @Test
+  public void testZipkinDebugOverride() throws Exception {
+    ZipkinPortUnificationHandler handler =
+        new ZipkinPortUnificationHandler(
+            "9411",
+            new NoopHealthCheckManager(),
+            mockTraceHandler,
+            mockTraceSpanLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new DurationSampler(10), () -> null),
+            null,
+            null);
+
+    // take care of mocks.
+    // Reset mock
+    reset(mockTraceHandler, mockTraceSpanLogsHandler);
+
+    // Set Expectation
+    Span expectedSpan2 =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("getservice")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-3822-889fe47043bd")
+            .setTraceId("00000000-0000-0000-3822-889fe47043bd")
+            .
+            // Note: Order of annotations list matters for this unit test.
+            setAnnotations(
+                ImmutableList.of(
+                    new Annotation("zipkinSpanId", "3822889fe47043bd"),
+                    new Annotation("zipkinTraceId", "3822889fe47043bd"),
+                    new Annotation("span.kind", "server"),
+                    new Annotation("_spanSecondaryId", "server"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("http.method", "GET"),
+                    new Annotation("http.status_code", "200"),
+                    new Annotation("http.url", "none+h1c://localhost:8881/"),
+                    new Annotation("application", "Zipkin"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("debug", "true"),
+                    new Annotation("ipv4", "10.0.0.1"),
+                    new Annotation("_spanLogs", "true")))
+            .build();
+    mockTraceHandler.report(expectedSpan2);
+    expectLastCall();
+
+    mockTraceSpanLogsHandler.report(
+        SpanLogs.newBuilder()
+            .setCustomer("default")
+            .setTraceId("00000000-0000-0000-3822-889fe47043bd")
+            .setSpanId("00000000-0000-0000-3822-889fe47043bd")
+            .setSpanSecondaryId("server")
+            .setSpan("_sampledByPolicy=NONE")
+            .setLogs(
+                ImmutableList.of(
+                    SpanLog.newBuilder()
+                        .setTimestamp(startTime * 1000)
+                        .setFields(ImmutableMap.of("annotation", "start processing"))
+                        .build()))
+            .build());
+    expectLastCall();
+
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(6)
+            .setName("getservice")
+            .setSource(DEFAULT_SOURCE)
+            .setSpanId("00000000-0000-0000-5822-889fe47043bd")
+            .setTraceId("00000000-0000-0000-5822-889fe47043bd")
+            .
+            // Note: Order of annotations list matters for this unit test.
+            setAnnotations(
+                ImmutableList.of(
+                    new Annotation("zipkinSpanId", "5822889fe47043bd"),
+                    new Annotation("zipkinTraceId", "5822889fe47043bd"),
+                    new Annotation("span.kind", "server"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("debug", "debug-id-4"),
+                    new Annotation("http.method", "GET"),
+                    new Annotation("http.status_code", "200"),
+                    new Annotation("http.url", "none+h1c://localhost:8881/"),
+                    new Annotation("application", "Zipkin"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("debug", "true"),
+                    new Annotation("ipv4", "10.0.0.1")))
+            .build());
+    expectLastCall();
+
+    Endpoint localEndpoint1 = Endpoint.newBuilder().serviceName("frontend").ip("10.0.0.1").build();
+    zipkin2.Span spanServer1 =
+        zipkin2.Span.newBuilder()
+            .traceId("2822889fe47043bd")
+            .id("2822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getservice")
+            .timestamp(startTime * 1000)
+            .duration(8 * 1000)
+            .localEndpoint(localEndpoint1)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h1c://localhost:8881/")
+            .putTag("http.status_code", "200")
+            .addAnnotation(startTime * 1000, "start processing")
+            .build();
+
+    zipkin2.Span spanServer2 =
+        zipkin2.Span.newBuilder()
+            .traceId("3822889fe47043bd")
+            .id("3822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getservice")
+            .timestamp(startTime * 1000)
+            .duration(9 * 1000)
+            .localEndpoint(localEndpoint1)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h1c://localhost:8881/")
+            .putTag("http.status_code", "200")
+            .debug(true)
+            .addAnnotation(startTime * 1000, "start processing")
+            .build();
+
+    zipkin2.Span spanServer3 =
+        zipkin2.Span.newBuilder()
+            .traceId("4822889fe47043bd")
+            .id("4822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getservice")
+            .timestamp(startTime * 1000)
+            .duration(7 * 1000)
+            .localEndpoint(localEndpoint1)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h1c://localhost:8881/")
+            .putTag("http.status_code", "200")
+            .putTag("debug", "debug-id-1")
+            .addAnnotation(startTime * 1000, "start processing")
+            .build();
+
+    zipkin2.Span spanServer4 =
+        zipkin2.Span.newBuilder()
+            .traceId("5822889fe47043bd")
+            .id("5822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getservice")
+            .timestamp(startTime * 1000)
+            .duration(6 * 1000)
+            .localEndpoint(localEndpoint1)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h1c://localhost:8881/")
+            .putTag("http.status_code", "200")
+            .putTag("debug", "debug-id-4")
+            .debug(true)
+            .build();
+
+    List zipkinSpanList =
+        ImmutableList.of(spanServer1, spanServer2, spanServer3, spanServer4);
+
+    SpanBytesEncoder encoder = SpanBytesEncoder.values()[1];
+    ByteBuf content = Unpooled.copiedBuffer(encoder.encodeList(zipkinSpanList));
+
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+
+    ChannelHandlerContext mockCtx = createNiceMock(ChannelHandlerContext.class);
+    doMockLifecycle(mockCtx);
+    FullHttpRequest httpRequest =
+        new DefaultFullHttpRequest(
+            HttpVersion.HTTP_1_1,
+            HttpMethod.POST,
+            "http://localhost:9411/api/v1/spans",
+            content,
+            true);
+    handler.handleHttpMessage(mockCtx, httpRequest);
+    verify(mockTraceHandler, mockTraceSpanLogsHandler);
+  }
+
+  @Test
+  public void testZipkinCustomSource() throws Exception {
+    ZipkinPortUnificationHandler handler =
+        new ZipkinPortUnificationHandler(
+            "9411",
+            new NoopHealthCheckManager(),
+            mockTraceHandler,
+            mockTraceSpanLogsHandler,
+            null,
+            () -> false,
+            () -> false,
+            null,
+            new SpanSampler(new RateSampler(1.0D), () -> null),
+            null,
+            null);
+
+    // take care of mocks.
+    // Reset mock
+    reset(mockTraceHandler, mockTraceSpanLogsHandler);
+
+    // Set Expectation
+    mockTraceHandler.report(
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(9)
+            .setName("getservice")
+            .setSource("customZipkinSource")
+            .setSpanId("00000000-0000-0000-2822-889fe47043bd")
+            .setTraceId("00000000-0000-0000-2822-889fe47043bd")
+            .
+            // Note: Order of annotations list matters for this unit test.
+            setAnnotations(
+                ImmutableList.of(
+                    new Annotation("zipkinSpanId", "2822889fe47043bd"),
+                    new Annotation("zipkinTraceId", "2822889fe47043bd"),
+                    new Annotation("span.kind", "server"),
+                    new Annotation("service", "frontend"),
+                    new Annotation("http.method", "GET"),
+                    new Annotation("http.status_code", "200"),
+                    new Annotation("http.url", "none+h1c://localhost:8881/"),
+                    new Annotation("application", "Zipkin"),
+                    new Annotation("cluster", "none"),
+                    new Annotation("shard", "none"),
+                    new Annotation("ipv4", "10.0.0.1")))
+            .build());
+    expectLastCall();
+
+    Endpoint localEndpoint1 = Endpoint.newBuilder().serviceName("frontend").ip("10.0.0.1").build();
+    zipkin2.Span spanServer1 =
+        zipkin2.Span.newBuilder()
+            .traceId("2822889fe47043bd")
+            .id("2822889fe47043bd")
+            .kind(zipkin2.Span.Kind.SERVER)
+            .name("getservice")
+            .timestamp(startTime * 1000)
+            .duration(9 * 1000)
+            .localEndpoint(localEndpoint1)
+            .putTag("http.method", "GET")
+            .putTag("http.url", "none+h1c://localhost:8881/")
+            .putTag("http.status_code", "200")
+            .putTag("source", "customZipkinSource")
+            .build();
+
+    List zipkinSpanList = ImmutableList.of(spanServer1);
+
+    SpanBytesEncoder encoder = SpanBytesEncoder.values()[1];
+    ByteBuf content = Unpooled.copiedBuffer(encoder.encodeList(zipkinSpanList));
+
+    replay(mockTraceHandler, mockTraceSpanLogsHandler);
+
+    ChannelHandlerContext mockCtx = createNiceMock(ChannelHandlerContext.class);
+    doMockLifecycle(mockCtx);
+    FullHttpRequest httpRequest =
+        new DefaultFullHttpRequest(
+            HttpVersion.HTTP_1_1,
+            HttpMethod.POST,
+            "http://localhost:9411/api/v1/spans",
+            content,
+            true);
+    handler.handleHttpMessage(mockCtx, httpRequest);
+    verify(mockTraceHandler, mockTraceSpanLogsHandler);
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/logsharvesting/LogsIngesterTest.java b/proxy/src/test/java/com/wavefront/agent/logsharvesting/LogsIngesterTest.java
index 3eee697c6..1f17a8bc1 100644
--- a/proxy/src/test/java/com/wavefront/agent/logsharvesting/LogsIngesterTest.java
+++ b/proxy/src/test/java/com/wavefront/agent/logsharvesting/LogsIngesterTest.java
@@ -1,114 +1,145 @@
 package com.wavefront.agent.logsharvesting;
 
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThan;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactoryBuilder;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-import com.wavefront.agent.PointHandler;
 import com.wavefront.agent.PointMatchers;
+import com.wavefront.agent.auth.TokenAuthenticatorBuilder;
+import com.wavefront.agent.channel.NoopHealthCheckManager;
 import com.wavefront.agent.config.ConfigurationException;
 import com.wavefront.agent.config.LogsIngestionConfig;
 import com.wavefront.agent.config.MetricMatcher;
-
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.agent.handlers.ReportableEntityHandler;
+import com.wavefront.agent.handlers.ReportableEntityHandlerFactory;
+import com.wavefront.agent.listeners.RawLogsIngesterPortUnificationHandler;
 import com.wavefront.common.MetricConstants;
-import org.easymock.Capture;
-import org.easymock.CaptureType;
-import org.easymock.EasyMock;
-import org.junit.After;
-import org.junit.Test;
-import org.logstash.beats.Message;
-
+import com.wavefront.data.ReportableEntityType;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.thekraken.grok.api.exception.GrokException;
 import java.io.File;
 import java.io.IOException;
-import java.net.SocketAddress;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
-
-import javax.annotation.Nullable;
-
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelHandlerContext;
-import oi.thekraken.grok.api.exception.GrokException;
+import org.easymock.Capture;
+import org.easymock.CaptureType;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Test;
+import org.logstash.beats.Message;
+import org.yaml.snakeyaml.LoaderOptions;
 import wavefront.report.Histogram;
 import wavefront.report.ReportPoint;
 
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
-import static org.easymock.EasyMock.verify;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
-
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
+/** @author Mori Bellamy (mori@wavefront.com) */
 public class LogsIngesterTest {
+  private final AtomicLong now;
+  private final AtomicLong nanos;
+  private final ObjectMapper objectMapper;
   private LogsIngestionConfig logsIngestionConfig;
   private LogsIngester logsIngesterUnderTest;
   private FilebeatIngester filebeatIngesterUnderTest;
-  private RawLogsIngester rawLogsIngesterUnderTest;
-  private PointHandler mockPointHandler;
-  private AtomicLong now = new AtomicLong(System.currentTimeMillis());  // 6:30PM california time Oct 13 2016
-  private ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
+  private RawLogsIngesterPortUnificationHandler rawLogsIngesterUnderTest;
+  private ReportableEntityHandlerFactory mockFactory;
+  private ReportableEntityHandler mockPointHandler;
+  private ReportableEntityHandler mockHistogramHandler;
+
+  public LogsIngesterTest() {
+    this.now = new AtomicLong((System.currentTimeMillis() / 60000) * 60000);
+    this.nanos = new AtomicLong(System.nanoTime());
+    YAMLFactoryBuilder factory = new YAMLFactoryBuilder(new YAMLFactory());
+    this.objectMapper = new ObjectMapper(factory.loaderOptions(new LoaderOptions()).build());
+  }
 
   private LogsIngestionConfig parseConfigFile(String configPath) throws IOException {
-    File configFile = new File(LogsIngesterTest.class.getClassLoader().getResource(configPath).getPath());
+    File configFile =
+        new File(LogsIngesterTest.class.getClassLoader().getResource(configPath).getPath());
     return objectMapper.readValue(configFile, LogsIngestionConfig.class);
   }
 
-  private void setup(String configPath) throws IOException, GrokException, ConfigurationException {
-    logsIngestionConfig = parseConfigFile(configPath);
+  private void setup(LogsIngestionConfig config)
+      throws IOException, GrokException, ConfigurationException {
+    logsIngestionConfig = config;
     logsIngestionConfig.aggregationIntervalSeconds = 10000; // HACK: Never call flush automatically.
     logsIngestionConfig.verifyAndInit();
-    mockPointHandler = createMock(PointHandler.class);
-    logsIngesterUnderTest = new LogsIngester(mockPointHandler, () -> logsIngestionConfig, null, now::get);
+    mockPointHandler = createMock(ReportableEntityHandler.class);
+    mockHistogramHandler = createMock(ReportableEntityHandler.class);
+    mockFactory = createMock(ReportableEntityHandlerFactory.class);
+    expect(
+            (ReportableEntityHandler)
+                mockFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, "logs-ingester")))
+        .andReturn(mockPointHandler)
+        .anyTimes();
+    expect(
+            (ReportableEntityHandler)
+                mockFactory.getHandler(
+                    HandlerKey.of(ReportableEntityType.HISTOGRAM, "logs-ingester")))
+        .andReturn(mockHistogramHandler)
+        .anyTimes();
+    replay(mockFactory);
+    logsIngesterUnderTest =
+        new LogsIngester(mockFactory, () -> logsIngestionConfig, null, now::get, nanos::get);
     logsIngesterUnderTest.start();
     filebeatIngesterUnderTest = new FilebeatIngester(logsIngesterUnderTest, now::get);
-    rawLogsIngesterUnderTest = new RawLogsIngester(logsIngesterUnderTest, -1, now::get);
-  }
-
-  private void receiveFilebeatLog(String log) {
-    Map data = Maps.newHashMap();
-    data.put("message", log);
-    data.put("beat", Maps.newHashMap());
-    data.put("@timestamp", "2016-10-13T20:43:45.172Z");
-    filebeatIngesterUnderTest.onNewMessage(null, new Message(0, data));
+    rawLogsIngesterUnderTest =
+        new RawLogsIngesterPortUnificationHandler(
+            "12345",
+            logsIngesterUnderTest,
+            x -> "testHost",
+            TokenAuthenticatorBuilder.create().build(),
+            new NoopHealthCheckManager(),
+            null);
   }
 
   private void receiveRawLog(String log) {
     ChannelHandlerContext ctx = EasyMock.createMock(ChannelHandlerContext.class);
     Channel channel = EasyMock.createMock(Channel.class);
     EasyMock.expect(ctx.channel()).andReturn(channel);
-    // Hack: Returning a mock SocketAddress simply causes the fallback to be used in getHostOrDefault.
-    EasyMock.expect(channel.remoteAddress()).andReturn(EasyMock.createMock(SocketAddress.class));
+    InetSocketAddress addr = InetSocketAddress.createUnresolved("testHost", 1234);
+    EasyMock.expect(channel.remoteAddress()).andReturn(addr);
     EasyMock.replay(ctx, channel);
-    rawLogsIngesterUnderTest.ingestLog(ctx, log);
+    rawLogsIngesterUnderTest.processLine(ctx, log, null);
     EasyMock.verify(ctx, channel);
   }
 
   private void receiveLog(String log) {
-    LogsMessage logsMessage = new LogsMessage() {
-      @Override
-      public String getLogLine() {
-        return log;
-      }
-
-      @Override
-      public String hostOrDefault(String fallbackHost) {
-        return "testHost";
-      }
-    };
+    LogsMessage logsMessage =
+        new LogsMessage() {
+          @Override
+          public String getLogLine() {
+            return log;
+          }
+
+          @Override
+          public String hostOrDefault(String fallbackHost) {
+            return "testHost";
+          }
+        };
     logsIngesterUnderTest.ingestLog(logsMessage);
   }
 
@@ -126,15 +157,26 @@ private List getPoints(int numPoints, String... logLines) throws Ex
     return getPoints(numPoints, 0, this::receiveLog, logLines);
   }
 
-  private List getPoints(int numPoints, int lagPerLogLine, Consumer consumer, String... logLines)
+  private List getPoints(
+      int numPoints, int lagPerLogLine, Consumer consumer, String... logLines)
+      throws Exception {
+    return getPoints(mockPointHandler, numPoints, lagPerLogLine, consumer, logLines);
+  }
+
+  private List getPoints(
+      ReportableEntityHandler handler,
+      int numPoints,
+      int lagPerLogLine,
+      Consumer consumer,
+      String... logLines)
       throws Exception {
     Capture reportPointCapture = Capture.newInstance(CaptureType.ALL);
-    reset(mockPointHandler);
+    reset(handler);
     if (numPoints > 0) {
-      mockPointHandler.reportPoint(EasyMock.capture(reportPointCapture), EasyMock.notNull(String.class));
+      handler.report(EasyMock.capture(reportPointCapture));
       expectLastCall().times(numPoints);
     }
-    replay(mockPointHandler);
+    replay(handler);
     for (String line : logLines) {
       consumer.accept(line);
       tick(lagPerLogLine);
@@ -144,276 +186,533 @@ private List getPoints(int numPoints, int lagPerLogLine, Consumer logsIngestionConfig, "myPrefix", now::get);
+    setup(parseConfigFile("test.yml"));
+    logsIngesterUnderTest =
+        new LogsIngester(mockFactory, () -> logsIngestionConfig, "myPrefix", now::get, nanos::get);
     assertThat(
         getPoints(1, "plainCounter"),
-        contains(PointMatchers.matches(1L, MetricConstants.DELTA_PREFIX + "myPrefix" +
-                        ".plainCounter", ImmutableMap.of())));
+        contains(
+            PointMatchers.matches(
+                1L,
+                MetricConstants.DELTA_PREFIX + "myPrefix" + ".plainCounter",
+                ImmutableMap.of())));
+  }
+
+  @Test
+  public void testFilebeatIngesterDefaultHostname() throws Exception {
+    setup(parseConfigFile("test.yml"));
+    assertThat(
+        getPoints(
+            1,
+            0,
+            log -> {
+              Map data = Maps.newHashMap();
+              data.put("message", log);
+              data.put("beat", Maps.newHashMap());
+              data.put("@timestamp", "2016-10-13T20:43:45.172Z");
+              filebeatIngesterUnderTest.onNewMessage(null, new Message(0, data));
+            },
+            "plainCounter"),
+        contains(
+            PointMatchers.matches(
+                1L,
+                MetricConstants.DELTA_PREFIX + "plainCounter",
+                "parsed-logs",
+                ImmutableMap.of())));
   }
 
   @Test
-  public void testFilebeatIngester() throws Exception {
-    setup("test.yml");
+  public void testFilebeatIngesterOverrideHostname() throws Exception {
+    setup(parseConfigFile("test.yml"));
     assertThat(
-        getPoints(1, 0, this::receiveFilebeatLog, "plainCounter"),
-        contains(PointMatchers.matches(1L, MetricConstants.DELTA_PREFIX + "plainCounter",
+        getPoints(
+            1,
+            0,
+            log -> {
+              Map data = Maps.newHashMap();
+              data.put("message", log);
+              data.put("beat", new HashMap<>(ImmutableMap.of("hostname", "overrideHostname")));
+              data.put("@timestamp", "2016-10-13T20:43:45.172Z");
+              filebeatIngesterUnderTest.onNewMessage(null, new Message(0, data));
+            },
+            "plainCounter"),
+        contains(
+            PointMatchers.matches(
+                1L,
+                MetricConstants.DELTA_PREFIX + "plainCounter",
+                "overrideHostname",
+                ImmutableMap.of())));
+  }
+
+  @Test
+  public void testFilebeat7Ingester() throws Exception {
+    setup(parseConfigFile("test.yml"));
+    assertThat(
+        getPoints(
+            1,
+            0,
+            log -> {
+              Map data = Maps.newHashMap();
+              data.put("message", log);
+              data.put("host", ImmutableMap.of("name", "filebeat7hostname"));
+              data.put("@timestamp", "2016-10-13T20:43:45.172Z");
+              filebeatIngesterUnderTest.onNewMessage(null, new Message(0, data));
+            },
+            "plainCounter"),
+        contains(
+            PointMatchers.matches(
+                1L,
+                MetricConstants.DELTA_PREFIX + "plainCounter",
+                "filebeat7hostname",
+                ImmutableMap.of())));
+  }
+
+  @Test
+  public void testFilebeat7IngesterAlternativeHostname() throws Exception {
+    setup(parseConfigFile("test.yml"));
+    assertThat(
+        getPoints(
+            1,
+            0,
+            log -> {
+              Map data = Maps.newHashMap();
+              data.put("message", log);
+              data.put("agent", ImmutableMap.of("hostname", "filebeat7althost"));
+              data.put("@timestamp", "2016-10-13T20:43:45.172Z");
+              filebeatIngesterUnderTest.onNewMessage(null, new Message(0, data));
+            },
+            "plainCounter"),
+        contains(
+            PointMatchers.matches(
+                1L,
+                MetricConstants.DELTA_PREFIX + "plainCounter",
+                "filebeat7althost",
                 ImmutableMap.of())));
   }
 
   @Test
   public void testRawLogsIngester() throws Exception {
-    setup("test.yml");
+    setup(parseConfigFile("test.yml"));
     assertThat(
         getPoints(1, 0, this::receiveRawLog, "plainCounter"),
-        contains(PointMatchers.matches(1L, MetricConstants.DELTA_PREFIX + "plainCounter",
-                ImmutableMap.of())));
+        contains(
+            PointMatchers.matches(
+                1L, MetricConstants.DELTA_PREFIX + "plainCounter", ImmutableMap.of())));
+  }
+
+  @Test
+  public void testEmptyTagValuesIgnored() throws Exception {
+    setup(parseConfigFile("test.yml"));
+    assertThat(
+        getPoints(
+            1,
+            0,
+            this::receiveRawLog,
+            "pingSSO|2020-03-13 09:11:30,490|OAuth| RLin123456| "
+                + "10.0.0.1 | | pa_wam| OAuth20| sso2-prod| AS| success| SecurID| | 249"),
+        contains(
+            PointMatchers.matches(
+                249.0,
+                "pingSSO",
+                ImmutableMap.builder()
+                    .put("sso_host", "sso2-prod")
+                    .put("protocol", "OAuth20")
+                    .put("role", "AS")
+                    .put("subject", "RLin123456")
+                    .put("ip", "10.0.0.1")
+                    .put("connectionid", "pa_wam")
+                    .put("adapterid", "SecurID")
+                    .put("event", "OAuth")
+                    .put("status", "success")
+                    .build())));
   }
 
   @Test(expected = ConfigurationException.class)
   public void testGaugeWithoutValue() throws Exception {
-    setup("badGauge.yml");
+    setup(parseConfigFile("badGauge.yml"));
   }
 
   @Test(expected = ConfigurationException.class)
   public void testTagsNonParallelArrays() throws Exception {
-    setup("badTags.yml");
+    setup(parseConfigFile("badTags.yml"));
   }
 
   @Test
   public void testHotloadedConfigClearsOldMetrics() throws Exception {
-    setup("test.yml");
+    setup(parseConfigFile("test.yml"));
     assertThat(
         getPoints(1, "plainCounter"),
-        contains(PointMatchers.matches(1L, MetricConstants.DELTA_PREFIX + "plainCounter",
-                ImmutableMap.of())));
+        contains(
+            PointMatchers.matches(
+                1L, MetricConstants.DELTA_PREFIX + "plainCounter", ImmutableMap.of())));
     // once the counter is reported, it is reset because now it is treated as delta counter.
     // hence we check that plainCounter has value 1L below.
     assertThat(
         getPoints(2, "plainCounter", "counterWithValue 42"),
         containsInAnyOrder(
             ImmutableList.of(
-                PointMatchers.matches(42L, MetricConstants.DELTA_PREFIX + "counterWithValue",
-                        ImmutableMap.of()),
-                PointMatchers.matches(1L, MetricConstants.DELTA_PREFIX + "plainCounter",
-                        ImmutableMap.of()))));
+                PointMatchers.matches(
+                    42L, MetricConstants.DELTA_PREFIX + "counterWithValue", ImmutableMap.of()),
+                PointMatchers.matches(
+                    1L, MetricConstants.DELTA_PREFIX + "plainCounter", ImmutableMap.of()))));
     List counters = Lists.newCopyOnWriteArrayList(logsIngestionConfig.counters);
     int oldSize = counters.size();
     counters.removeIf((metricMatcher -> metricMatcher.getPattern().equals("plainCounter")));
     assertThat(counters, hasSize(oldSize - 1));
-    // Get a new config file because the SUT has a reference to the old one, and we'll be monkey patching
+    // Get a new config file because the SUT has a reference to the old one, and we'll be monkey
+    // patching
     // this one.
     logsIngestionConfig = parseConfigFile("test.yml");
     logsIngestionConfig.verifyAndInit();
     logsIngestionConfig.counters = counters;
     logsIngesterUnderTest.logsIngestionConfigManager.forceConfigReload();
     // once the counter is reported, it is reset because now it is treated as delta counter.
-    // hence we check that counterWithValue has value 0L below.
+    // since zero values are filtered out, no new values are expected.
+    assertThat(getPoints(0, "plainCounter"), emptyIterable());
+  }
+
+  @Test
+  public void testEvictedDeltaMetricReportingAgain() throws Exception {
+    setup(parseConfigFile("test.yml"));
     assertThat(
-        getPoints(1, "plainCounter"),
-        contains(PointMatchers.matches(0L, MetricConstants.DELTA_PREFIX + "counterWithValue",
-                ImmutableMap.of())));
+        getPoints(1, "plainCounter", "plainCounter", "plainCounter", "plainCounter"),
+        contains(
+            PointMatchers.matches(
+                4L, MetricConstants.DELTA_PREFIX + "plainCounter", ImmutableMap.of())));
+    nanos.addAndGet(1_800_000L * 1000L * 1000L);
+    assertThat(
+        getPoints(1, "plainCounter", "plainCounter", "plainCounter"),
+        contains(
+            PointMatchers.matches(
+                3L, MetricConstants.DELTA_PREFIX + "plainCounter", ImmutableMap.of())));
+    nanos.addAndGet(3_601_000L * 1000L * 1000L);
+    assertThat(
+        getPoints(1, "plainCounter", "plainCounter"),
+        contains(
+            PointMatchers.matches(
+                2L, MetricConstants.DELTA_PREFIX + "plainCounter", ImmutableMap.of())));
+  }
+
+  @Test
+  public void testEvictedMetricReportingAgain() throws Exception {
+    setup(parseConfigFile("test-non-delta.yml"));
+    assertThat(
+        getPoints(1, "plainCounter", "plainCounter", "plainCounter", "plainCounter"),
+        contains(PointMatchers.matches(4L, "plainCounter", ImmutableMap.of())));
+    nanos.addAndGet(1_800_000L * 1000L * 1000L);
+    assertThat(
+        getPoints(1, "plainCounter", "plainCounter", "plainCounter"),
+        contains(PointMatchers.matches(7L, "plainCounter", ImmutableMap.of())));
+    nanos.addAndGet(3_601_000L * 1000L * 1000L);
+    assertThat(
+        getPoints(1, "plainCounter", "plainCounter"),
+        contains(PointMatchers.matches(2L, "plainCounter", ImmutableMap.of())));
   }
 
   @Test
   public void testMetricsAggregation() throws Exception {
-    setup("test.yml");
+    setup(parseConfigFile("test.yml"));
     assertThat(
-        getPoints(6,
-            "plainCounter", "noMatch 42.123 bar", "plainCounter",
+        getPoints(
+            6,
+            "plainCounter",
+            "noMatch 42.123 bar",
+            "plainCounter",
             "gauges 42",
-            "counterWithValue 2", "counterWithValue 3",
-            "dynamicCounter foo 1 done", "dynamicCounter foo 2 done", "dynamicCounter baz 1 done"),
+            "counterWithValue 2",
+            "counterWithValue 3",
+            "dynamicCounter foo 1 done",
+            "dynamicCounter foo 2 done",
+            "dynamicCounter baz 1 done"),
         containsInAnyOrder(
             ImmutableList.of(
-                PointMatchers.matches(2L, MetricConstants.DELTA_PREFIX + "plainCounter",
-                        ImmutableMap.of()),
-                PointMatchers.matches(5L, MetricConstants.DELTA_PREFIX + "counterWithValue",
-                        ImmutableMap.of()),
-                PointMatchers.matches(1L, MetricConstants.DELTA_PREFIX + "dynamic_foo_1",
-                        ImmutableMap.of()),
-                PointMatchers.matches(1L, MetricConstants.DELTA_PREFIX + "dynamic_foo_2",
-                        ImmutableMap.of()),
-                PointMatchers.matches(1L, MetricConstants.DELTA_PREFIX + "dynamic_baz_1",
-                        ImmutableMap.of()),
-                PointMatchers.matches(42.0, "myGauge", ImmutableMap.of())))
-    );
+                PointMatchers.matches(
+                    2L, MetricConstants.DELTA_PREFIX + "plainCounter", ImmutableMap.of()),
+                PointMatchers.matches(
+                    5L, MetricConstants.DELTA_PREFIX + "counterWithValue", ImmutableMap.of()),
+                PointMatchers.matches(
+                    1L, MetricConstants.DELTA_PREFIX + "dynamic_foo_1", ImmutableMap.of()),
+                PointMatchers.matches(
+                    1L, MetricConstants.DELTA_PREFIX + "dynamic_foo_2", ImmutableMap.of()),
+                PointMatchers.matches(
+                    1L, MetricConstants.DELTA_PREFIX + "dynamic_baz_1", ImmutableMap.of()),
+                PointMatchers.matches(42.0, "myGauge", ImmutableMap.of()))));
   }
 
-  /**
-   * This test is not required, because delta counters have different naming convention than gauges
+  @Test
+  public void testMetricsAggregationNonDeltaCounters() throws Exception {
+    LogsIngestionConfig config = parseConfigFile("test.yml");
+    config.useDeltaCounters = false;
+    setup(config);
+    assertThat(
+        getPoints(
+            6,
+            "plainCounter",
+            "noMatch 42.123 bar",
+            "plainCounter",
+            "gauges 42",
+            "counterWithValue 2",
+            "counterWithValue 3",
+            "dynamicCounter foo 1 done",
+            "dynamicCounter foo 2 done",
+            "dynamicCounter baz 1 done"),
+        containsInAnyOrder(
+            ImmutableList.of(
+                PointMatchers.matches(2L, "plainCounter", ImmutableMap.of()),
+                PointMatchers.matches(5L, "counterWithValue", ImmutableMap.of()),
+                PointMatchers.matches(1L, "dynamic_foo_1", ImmutableMap.of()),
+                PointMatchers.matches(1L, "dynamic_foo_2", ImmutableMap.of()),
+                PointMatchers.matches(1L, "dynamic_baz_1", ImmutableMap.of()),
+                PointMatchers.matches(42.0, "myGauge", ImmutableMap.of()))));
+  }
 
-  @Test(expected = ClassCastException.class)
-  public void testDuplicateMetric() throws Exception {
-    setup("dupe.yml");
-    assertThat(getPoints(2, "plainCounter", "plainGauge 42"), notNullValue());
+  @Test
+  public void testExtractHostname() throws Exception {
+    setup(parseConfigFile("test.yml"));
+    assertThat(
+        getPoints(
+            3,
+            "operation foo on host web001 took 2 seconds",
+            "operation foo on host web001 took 2 seconds",
+            "operation foo on host web002 took 3 seconds",
+            "operation bar on host web001 took 4 seconds"),
+        containsInAnyOrder(
+            ImmutableList.of(
+                PointMatchers.matches(
+                    4L,
+                    MetricConstants.DELTA_PREFIX + "Host.foo.totalSeconds",
+                    "web001.acme.corp",
+                    ImmutableMap.of("static", "value")),
+                PointMatchers.matches(
+                    3L,
+                    MetricConstants.DELTA_PREFIX + "Host.foo.totalSeconds",
+                    "web002.acme.corp",
+                    ImmutableMap.of("static", "value")),
+                PointMatchers.matches(
+                    4L,
+                    MetricConstants.DELTA_PREFIX + "Host.bar.totalSeconds",
+                    "web001.acme.corp",
+                    ImmutableMap.of("static", "value")))));
   }
-   */
 
+  /**
+   * This test is not required, because delta counters have different naming convention than
+   * gauges @Test(expected = ClassCastException.class) public void testDuplicateMetric() throws
+   * Exception { setup(parseConfigFile("dupe.yml")); assertThat(getPoints(2, "plainCounter",
+   * "plainGauge 42"), notNullValue()); }
+   */
   @Test
   public void testDynamicLabels() throws Exception {
-    setup("test.yml");
+    setup(parseConfigFile("test.yml"));
     assertThat(
-        getPoints(3,
+        getPoints(
+            3,
             "operation foo took 2 seconds in DC=wavefront AZ=2a",
             "operation foo took 2 seconds in DC=wavefront AZ=2a",
             "operation foo took 3 seconds in DC=wavefront AZ=2b",
             "operation bar took 4 seconds in DC=wavefront AZ=2a"),
         containsInAnyOrder(
             ImmutableList.of(
-                PointMatchers.matches(4L, MetricConstants.DELTA_PREFIX + "foo.totalSeconds",
-                        ImmutableMap.of("theDC", "wavefront", "theAZ", "2a")),
-                PointMatchers.matches(3L, MetricConstants.DELTA_PREFIX + "foo.totalSeconds",
-                        ImmutableMap.of("theDC", "wavefront", "theAZ", "2b")),
-                PointMatchers.matches(4L, MetricConstants.DELTA_PREFIX + "bar.totalSeconds",
-                        ImmutableMap.of("theDC", "wavefront", "theAZ", "2a"))
-            )
-        ));
+                PointMatchers.matches(
+                    4L,
+                    MetricConstants.DELTA_PREFIX + "foo.totalSeconds",
+                    ImmutableMap.of("theDC", "wavefront", "theAZ", "2a")),
+                PointMatchers.matches(
+                    3L,
+                    MetricConstants.DELTA_PREFIX + "foo.totalSeconds",
+                    ImmutableMap.of("theDC", "wavefront", "theAZ", "2b")),
+                PointMatchers.matches(
+                    4L,
+                    MetricConstants.DELTA_PREFIX + "bar.totalSeconds",
+                    ImmutableMap.of("theDC", "wavefront", "theAZ", "2a")))));
   }
 
   @Test
   public void testDynamicTagValues() throws Exception {
-    setup("test.yml");
+    setup(parseConfigFile("test.yml"));
     assertThat(
-        getPoints(3,
+        getPoints(
+            3,
             "operation TagValue foo took 2 seconds in DC=wavefront AZ=2a",
             "operation TagValue foo took 2 seconds in DC=wavefront AZ=2a",
             "operation TagValue foo took 3 seconds in DC=wavefront AZ=2b",
             "operation TagValue bar took 4 seconds in DC=wavefront AZ=2a"),
         containsInAnyOrder(
             ImmutableList.of(
-                PointMatchers.matches(4L, MetricConstants.DELTA_PREFIX +
-                                "TagValue.foo.totalSeconds",
-                    ImmutableMap.of("theDC", "wavefront", "theAZ", "az-2a", "static", "value", "noMatch", "aa%{q}bb")),
-                PointMatchers.matches(3L, MetricConstants.DELTA_PREFIX +
-                                "TagValue.foo.totalSeconds",
-                    ImmutableMap.of("theDC", "wavefront", "theAZ", "az-2b", "static", "value", "noMatch", "aa%{q}bb")),
-                PointMatchers.matches(4L, MetricConstants.DELTA_PREFIX +
-                                "TagValue.bar.totalSeconds",
-                    ImmutableMap.of("theDC", "wavefront", "theAZ", "az-2a", "static", "value", "noMatch", "aa%{q}bb"))
-            )
-        ));
+                PointMatchers.matches(
+                    4L,
+                    MetricConstants.DELTA_PREFIX + "TagValue.foo.totalSeconds",
+                    ImmutableMap.of(
+                        "theDC",
+                        "wavefront",
+                        "theAZ",
+                        "az-2a",
+                        "static",
+                        "value",
+                        "noMatch",
+                        "aa%{q}bb")),
+                PointMatchers.matches(
+                    3L,
+                    MetricConstants.DELTA_PREFIX + "TagValue.foo.totalSeconds",
+                    ImmutableMap.of(
+                        "theDC",
+                        "wavefront",
+                        "theAZ",
+                        "az-2b",
+                        "static",
+                        "value",
+                        "noMatch",
+                        "aa%{q}bb")),
+                PointMatchers.matches(
+                    4L,
+                    MetricConstants.DELTA_PREFIX + "TagValue.bar.totalSeconds",
+                    ImmutableMap.of(
+                        "theDC",
+                        "wavefront",
+                        "theAZ",
+                        "az-2a",
+                        "static",
+                        "value",
+                        "noMatch",
+                        "aa%{q}bb")))));
   }
 
   @Test
   public void testAdditionalPatterns() throws Exception {
-    setup("test.yml");
+    setup(parseConfigFile("test.yml"));
     assertThat(
         getPoints(1, "foo and 42"),
-        contains(PointMatchers.matches(42L, MetricConstants.DELTA_PREFIX +
-                "customPatternCounter", ImmutableMap.of())));
+        contains(
+            PointMatchers.matches(
+                42L, MetricConstants.DELTA_PREFIX + "customPatternCounter", ImmutableMap.of())));
   }
 
   @Test
   public void testParseValueFromCombinedApacheLog() throws Exception {
-    setup("test.yml");
+    setup(parseConfigFile("test.yml"));
     assertThat(
-        getPoints(3,
-            "52.34.54.96 - - [11/Oct/2016:06:35:45 +0000] \"GET /api/alert/summary HTTP/1.0\" " +
-                "200 632 \"https://dev-2b.corp.wavefront.com/chart\" " +
-                "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) " +
-                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36\""
-        ),
+        getPoints(
+            3,
+            "52.34.54.96 - - [11/Oct/2016:06:35:45 +0000] \"GET /api/alert/summary HTTP/1.0\" "
+                + "200 632 \"https://dev-2b.corp.wavefront.com/chart\" "
+                + "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) "
+                + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36\""),
         containsInAnyOrder(
             ImmutableList.of(
-                PointMatchers.matches(632L, MetricConstants.DELTA_PREFIX + "apacheBytes",
-                        ImmutableMap.of()),
-                PointMatchers.matches(632L, MetricConstants.DELTA_PREFIX + "apacheBytes2",
-                        ImmutableMap.of()),
-                PointMatchers.matches(200.0, "apacheStatus", ImmutableMap.of())
-            )
-        ));
+                PointMatchers.matches(
+                    632L, MetricConstants.DELTA_PREFIX + "apacheBytes", ImmutableMap.of()),
+                PointMatchers.matches(
+                    632L, MetricConstants.DELTA_PREFIX + "apacheBytes2", ImmutableMap.of()),
+                PointMatchers.matches(200.0, "apacheStatus", ImmutableMap.of()))));
   }
 
   @Test
   public void testIncrementCounterWithImplied1() throws Exception {
-    setup("test.yml");
+    setup(parseConfigFile("test.yml"));
     assertThat(
         getPoints(1, "plainCounter"),
-        contains(PointMatchers.matches(1L, MetricConstants.DELTA_PREFIX + "plainCounter",
-                ImmutableMap.of())));
-    // once the counter has been reported, the counter is reset because it is now treated as delta
+        contains(
+            PointMatchers.matches(
+                1L, MetricConstants.DELTA_PREFIX + "plainCounter", ImmutableMap.of())));
+    // once the counter has been reported, the counter is reset because it is now treated as
+    // delta
     // counter. Hence we check that plainCounter has value 1 below.
     assertThat(
         getPoints(1, "plainCounter"),
-        contains(PointMatchers.matches(1L, MetricConstants.DELTA_PREFIX + "plainCounter",
-                ImmutableMap.of())));
+        contains(
+            PointMatchers.matches(
+                1L, MetricConstants.DELTA_PREFIX + "plainCounter", ImmutableMap.of())));
   }
 
   @Test
   public void testHistogram() throws Exception {
-    setup("test.yml");
+    setup(parseConfigFile("test.yml"));
     String[] lines = new String[100];
     for (int i = 1; i < 101; i++) {
       lines[i - 1] = "histo " + i;
     }
+    List points = getPoints(9, 2000, this::receiveLog, lines);
+    tick(60000);
     assertThat(
-        getPoints(11, lines),
-        containsInAnyOrder(ImmutableList.of(
-            PointMatchers.almostMatches(100.0, "myHisto.count", ImmutableMap.of()),
-            PointMatchers.almostMatches(1.0, "myHisto.min", ImmutableMap.of()),
-            PointMatchers.almostMatches(100.0, "myHisto.max", ImmutableMap.of()),
-            PointMatchers.almostMatches(50.5, "myHisto.mean", ImmutableMap.of()),
-            PointMatchers.almostMatches(50.5, "myHisto.median", ImmutableMap.of()),
-            PointMatchers.almostMatches(75.25, "myHisto.p75", ImmutableMap.of()),
-            PointMatchers.almostMatches(95.05, "myHisto.p95", ImmutableMap.of()),
-            PointMatchers.almostMatches(99.01, "myHisto.p99", ImmutableMap.of()),
-            PointMatchers.almostMatches(99.901, "myHisto.p999", ImmutableMap.of()),
-            PointMatchers.matches(Double.NaN, "myHisto.sum", ImmutableMap.of()),
-            PointMatchers.matches(Double.NaN, "myHisto.stddev", ImmutableMap.of())
-        ))
-    );
+        points,
+        containsInAnyOrder(
+            ImmutableList.of(
+                PointMatchers.almostMatches(100.0, "myHisto.count", ImmutableMap.of()),
+                PointMatchers.almostMatches(1.0, "myHisto.min", ImmutableMap.of()),
+                PointMatchers.almostMatches(100.0, "myHisto.max", ImmutableMap.of()),
+                PointMatchers.almostMatches(50.5, "myHisto.mean", ImmutableMap.of()),
+                PointMatchers.almostMatches(50.5, "myHisto.median", ImmutableMap.of()),
+                PointMatchers.almostMatches(75.5, "myHisto.p75", ImmutableMap.of()),
+                PointMatchers.almostMatches(95.5, "myHisto.p95", ImmutableMap.of()),
+                PointMatchers.almostMatches(99.5, "myHisto.p99", ImmutableMap.of()),
+                PointMatchers.almostMatches(100, "myHisto.p999", ImmutableMap.of()))));
   }
 
   @Test
   public void testProxyLogLine() throws Exception {
-    setup("test.yml");
+    setup(parseConfigFile("test.yml"));
     assertThat(
         getPoints(1, "WARNING: [2878] (SUMMARY): points attempted: 859432; blocked: 0"),
-        contains(PointMatchers.matches(859432.0, "wavefrontPointsSent.2878", ImmutableMap.of()))
-    );
+        contains(PointMatchers.matches(859432.0, "wavefrontPointsSent.2878", ImmutableMap.of())));
   }
 
   @Test
   public void testWavefrontHistogram() throws Exception {
-    setup("histos.yml");
-    String[] lines = new String[100];
-    for (int i = 1; i < 101; i++) {
-      lines[i - 1] = "histo " + i;
+    setup(parseConfigFile("histos.yml"));
+    List logs = new ArrayList<>();
+    logs.add("histo 100");
+    logs.add("histo 100");
+    logs.add("histo 100");
+    for (int i = 0; i < 10; i++) logs.add("histo 1");
+    for (int i = 0; i < 1000; i++) {
+      logs.add("histo 75");
+    }
+    for (int i = 0; i < 100; i++) {
+      logs.add("histo 90");
+    }
+    for (int i = 0; i < 10; i++) {
+      logs.add("histo 99");
+    }
+    for (int i = 0; i < 10000; i++) {
+      logs.add("histo 50");
     }
-    ReportPoint reportPoint = getPoints(1, lines).get(0);
+
+    ReportPoint reportPoint =
+        getPoints(mockHistogramHandler, 1, 0, this::receiveLog, logs.toArray(new String[0])).get(0);
     assertThat(reportPoint.getValue(), instanceOf(Histogram.class));
     Histogram wavefrontHistogram = (Histogram) reportPoint.getValue();
-    assertThat(wavefrontHistogram.getBins(), hasSize(1));
-    assertThat(wavefrontHistogram.getBins(), contains(50.5));
-    assertThat(wavefrontHistogram.getCounts(), hasSize(1));
-    assertThat(wavefrontHistogram.getCounts(), contains(100));
+    assertThat(wavefrontHistogram.getCounts().stream().reduce(Integer::sum).get(), equalTo(11123));
+    assertThat(wavefrontHistogram.getBins().size(), lessThan(110));
+    assertThat(wavefrontHistogram.getBins().get(0), equalTo(1.0));
+    assertThat(
+        wavefrontHistogram.getBins().get(wavefrontHistogram.getBins().size() - 1), equalTo(100.0));
   }
 
   @Test
   public void testWavefrontHistogramMultipleCentroids() throws Exception {
-    setup("histos.yml");
-    String[] lines = new String[60];
-    for (int i = 1; i < 61; i++) {
-      lines[i - 1] = "histo " + i;
+    setup(parseConfigFile("histos.yml"));
+    String[] lines = new String[240];
+    for (int i = 0; i < 240; i++) {
+      lines[i] = "histo " + (i + 1);
     }
-    ReportPoint reportPoint = getPoints(1, 1000, this::receiveLog, lines).get(0);
-    assertThat(reportPoint.getValue(), instanceOf(Histogram.class));
-    Histogram wavefrontHistogram = (Histogram) reportPoint.getValue();
-    assertThat(wavefrontHistogram.getBins(), hasSize(2));
-    assertThat(wavefrontHistogram.getCounts(), hasSize(2));
-    assertThat(wavefrontHistogram.getCounts().stream().reduce(Integer::sum).get(), equalTo(60));
+    List reportPoints =
+        getPoints(mockHistogramHandler, 2, 500, this::receiveLog, lines);
+    assertThat(reportPoints.size(), equalTo(2));
+    assertThat(
+        reportPoints,
+        containsInAnyOrder(
+            PointMatchers.histogramMatches(120, 7260.0),
+            PointMatchers.histogramMatches(120, 21660.0)));
   }
 
   @Test(expected = ConfigurationException.class)
   public void testBadName() throws Exception {
-    setup("badName.yml");
+    setup(parseConfigFile("badName.yml"));
   }
 }
diff --git a/proxy/src/test/java/com/wavefront/agent/preprocessor/AgentConfigurationTest.java b/proxy/src/test/java/com/wavefront/agent/preprocessor/AgentConfigurationTest.java
deleted file mode 100644
index f4c47f797..000000000
--- a/proxy/src/test/java/com/wavefront/agent/preprocessor/AgentConfigurationTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.InputStream;
-
-import static org.junit.Assert.fail;
-
-public class AgentConfigurationTest {
-
-  @Test
-  public void testLoadInvalidRules() {
-    AgentPreprocessorConfiguration config = new AgentPreprocessorConfiguration();
-    try {
-      InputStream stream = PreprocessorRulesTest.class.getResourceAsStream("preprocessor_rules_invalid.yaml");
-      config.loadFromStream(stream);
-      fail("Invalid rules did not cause an exception");
-    } catch (RuntimeException ex) {
-      Assert.assertEquals(0, config.totalValidRules);
-      Assert.assertEquals(111, config.totalInvalidRules);
-    }
-  }
-
-  @Test
-  public void testLoadValidRules() {
-    AgentPreprocessorConfiguration config = new AgentPreprocessorConfiguration();
-    InputStream stream = PreprocessorRulesTest.class.getResourceAsStream("preprocessor_rules.yaml");
-    config.loadFromStream(stream);
-    Assert.assertEquals(0, config.totalInvalidRules);
-    Assert.assertEquals(34, config.totalValidRules);
-  }
-}
diff --git a/proxy/src/test/java/com/wavefront/agent/preprocessor/InteractivePreprocessorTesterTest.java b/proxy/src/test/java/com/wavefront/agent/preprocessor/InteractivePreprocessorTesterTest.java
new file mode 100644
index 000000000..023532ded
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/preprocessor/InteractivePreprocessorTesterTest.java
@@ -0,0 +1,112 @@
+package com.wavefront.agent.preprocessor;
+
+import com.wavefront.agent.formatter.DataFormat;
+import com.wavefront.api.agent.preprocessor.ReportableEntityPreprocessor;
+import com.wavefront.api.agent.preprocessor.Preprocessor;
+import com.wavefront.data.ReportableEntityType;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.util.Collections;
+import java.util.function.Supplier;
+
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.*;
+
+public class InteractivePreprocessorTesterTest {
+
+    private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+    private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+    private final PrintStream originalOut = System.out;
+    private final PrintStream originalErr = System.err;
+    private final InputStream originalIn = System.in;
+
+    @Before
+    public void setUp() {
+        System.setOut(new PrintStream(outContent));
+        System.setErr(new PrintStream(errContent));
+        // System.in will be set per test method
+    }
+
+    @After
+    public void tearDown() {
+        System.setOut(originalOut);
+        System.setErr(originalErr);
+        System.setIn(originalIn);
+    }
+
+    private void setSystemIn(String input) {
+        System.setIn(new ByteArrayInputStream(input.getBytes()));
+    }
+
+    private String getOutput() {
+        return outContent.toString().trim();
+    }
+
+    @Test
+    public void testInteractiveTest_TraceEntity_ReportsSpan() {
+        setSystemIn("test-span\n");
+        Supplier mockPreprocessorSupplier = EasyMock.mock(Supplier.class);
+        ReportableEntityPreprocessor mockPreprocessor = EasyMock.mock(ReportableEntityPreprocessor.class);
+        expect(mockPreprocessorSupplier.get()).andReturn(mockPreprocessor).anyTimes();
+
+        // Preprocessor does not block or modify for this test, so no specific expectations on its methods
+        expect(mockPreprocessor.forSpan()).andReturn(new Preprocessor() {}).anyTimes();
+        expect(mockPreprocessor.forPointLine()).andReturn(new Preprocessor<>()).anyTimes();
+
+        replay(mockPreprocessorSupplier, mockPreprocessor);
+
+        InteractivePreprocessorTester tester = new InteractivePreprocessorTester(
+                mockPreprocessorSupplier,
+                ReportableEntityType.TRACE,
+                "2878",
+                Collections.emptyList()
+        );
+
+        boolean hasNext = tester.interactiveTest();
+
+        assertFalse(hasNext);
+        // Verify System.out contains the serialized span. The actual spanId/traceId are random, so use regex.
+        String expectedOutputRegex = "Rejected: test-span";
+        assertTrue("Output should contain serialized span matching regex: " + getOutput(), getOutput().matches(expectedOutputRegex));
+
+        verify(mockPreprocessorSupplier, mockPreprocessor);
+    }
+
+    @Test
+    public void testInteractiveTest_PointEntity_ReportsPoint() {
+        setSystemIn("some.metric 10.0 source=test\n");
+        Supplier mockPreprocessorSupplier = EasyMock.mock(Supplier.class);
+        ReportableEntityPreprocessor mockPreprocessor = EasyMock.mock(ReportableEntityPreprocessor.class);
+        expect(mockPreprocessorSupplier.get()).andReturn(mockPreprocessor).anyTimes();
+        expect(mockPreprocessor.forReportPoint()).andReturn(new Preprocessor() {}).anyTimes();
+        expect(mockPreprocessor.forPointLine()).andReturn(new Preprocessor<>()).anyTimes();
+
+        replay(mockPreprocessorSupplier, mockPreprocessor);
+
+        InteractivePreprocessorTester tester = new InteractivePreprocessorTester(
+                mockPreprocessorSupplier,
+                ReportableEntityType.POINT,
+                "2878",
+                Collections.emptyList()
+        );
+
+        boolean hasNext = tester.interactiveTest();
+
+        assertFalse("Should indicate more input, as 'some.metric...' was followed by a newline", hasNext);
+
+        String actualOutput = outContent.toString().trim();
+        assertTrue("Output should contain serialized report point", actualOutput.startsWith("\"some.metric\" 10.0"));
+        assertTrue("Output should contain serialized report point", actualOutput.endsWith("source=\"test\""));
+
+        verify(mockPreprocessorSupplier, mockPreprocessor);
+    }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/preprocessor/PreprocessorRulesTest.java b/proxy/src/test/java/com/wavefront/agent/preprocessor/PreprocessorRulesTest.java
deleted file mode 100644
index 438a413b8..000000000
--- a/proxy/src/test/java/com/wavefront/agent/preprocessor/PreprocessorRulesTest.java
+++ /dev/null
@@ -1,430 +0,0 @@
-package com.wavefront.agent.preprocessor;
-
-import com.google.common.collect.Lists;
-
-import com.wavefront.ingester.GraphiteDecoder;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import static org.junit.Assert.*;
-
-import wavefront.report.ReportPoint;
-
-public class PreprocessorRulesTest {
-
-  private static AgentPreprocessorConfiguration config;
-  private final static List emptyCustomSourceTags = Collections.emptyList();
-  private final GraphiteDecoder decoder = new GraphiteDecoder(emptyCustomSourceTags);
-  private final PreprocessorRuleMetrics metrics = new PreprocessorRuleMetrics(null, null);
-
-  @BeforeClass
-  public static void setup() throws IOException {
-    InputStream stream = PreprocessorRulesTest.class.getResourceAsStream("preprocessor_rules.yaml");
-    config = new AgentPreprocessorConfiguration();
-    config.loadFromStream(stream);
-  }
-
-  @Test
-  public void testPointInRangeCorrectForTimeRanges() throws NoSuchMethodException, InvocationTargetException,
-      IllegalAccessException {
-
-    long millisPerYear = 31536000000L;
-    long millisPerDay = 86400000L;
-    long millisPerHour = 3600000L;
-
-    AnnotatedPredicate pointInRange1year = new ReportPointTimestampInRangeFilter(8760, 24);
-
-    // not in range if over a year ago
-    ReportPoint rp = new ReportPoint("some metric", System.currentTimeMillis() - millisPerYear, 10L, "host", "table",
-        new HashMap<>());
-    Assert.assertFalse(pointInRange1year.apply(rp));
-
-    rp.setTimestamp(System.currentTimeMillis() - millisPerYear - 1);
-    Assert.assertFalse(pointInRange1year.apply(rp));
-
-    // in range if within a year ago
-    rp.setTimestamp(System.currentTimeMillis() - (millisPerYear / 2));
-    Assert.assertTrue(pointInRange1year.apply(rp));
-
-    // in range for right now
-    rp.setTimestamp(System.currentTimeMillis());
-    Assert.assertTrue(pointInRange1year.apply(rp));
-
-    // in range if within a day in the future
-    rp.setTimestamp(System.currentTimeMillis() + millisPerDay - 1);
-    Assert.assertTrue(pointInRange1year.apply(rp));
-
-    // out of range for over a day in the future
-    rp.setTimestamp(System.currentTimeMillis() + (millisPerDay * 2));
-    Assert.assertFalse(pointInRange1year.apply(rp));
-
-    // now test with 1 day limit
-    AnnotatedPredicate pointInRange1day = new ReportPointTimestampInRangeFilter(24, 24);
-
-    rp.setTimestamp(System.currentTimeMillis() - millisPerDay - 1);
-    Assert.assertFalse(pointInRange1day.apply(rp));
-
-    // in range if within 1 day ago
-    rp.setTimestamp(System.currentTimeMillis() - (millisPerDay / 2));
-    Assert.assertTrue(pointInRange1day.apply(rp));
-
-    // in range for right now
-    rp.setTimestamp(System.currentTimeMillis());
-    Assert.assertTrue(pointInRange1day.apply(rp));
-
-    // assert for future range within 12 hours
-    AnnotatedPredicate pointInRange12hours = new ReportPointTimestampInRangeFilter(12, 12);
-
-    rp.setTimestamp(System.currentTimeMillis() + (millisPerHour * 10));
-    Assert.assertTrue(pointInRange12hours.apply(rp));
-
-    rp.setTimestamp(System.currentTimeMillis() - (millisPerHour * 10));
-    Assert.assertTrue(pointInRange12hours.apply(rp));
-
-    rp.setTimestamp(System.currentTimeMillis() + (millisPerHour * 20));
-    Assert.assertFalse(pointInRange12hours.apply(rp));
-
-    rp.setTimestamp(System.currentTimeMillis() - (millisPerHour * 20));
-    Assert.assertFalse(pointInRange12hours.apply(rp));
-
-    AnnotatedPredicate pointInRange10Days = new ReportPointTimestampInRangeFilter(240, 240);
-
-    rp.setTimestamp(System.currentTimeMillis() + (millisPerDay * 9));
-    Assert.assertTrue(pointInRange10Days.apply(rp));
-
-    rp.setTimestamp(System.currentTimeMillis() - (millisPerDay * 9));
-    Assert.assertTrue(pointInRange10Days.apply(rp));
-
-    rp.setTimestamp(System.currentTimeMillis() + (millisPerDay * 20));
-    Assert.assertFalse(pointInRange10Days.apply(rp));
-
-    rp.setTimestamp(System.currentTimeMillis() - (millisPerDay * 20));
-    Assert.assertFalse(pointInRange10Days.apply(rp));
-
-  }
-
-  @Test(expected = NullPointerException.class)
-  public void testLineReplaceRegexNullMatchThrows() {
-    // try to create a regex replace rule with a null match pattern
-    PointLineReplaceRegexTransformer invalidRule = new PointLineReplaceRegexTransformer(null, "foo", null, null, metrics);
-  }
-
-  @Test(expected = IllegalArgumentException.class)
-  public void testLineReplaceRegexBlankMatchThrows() {
-    // try to create a regex replace rule with a blank match pattern
-    PointLineReplaceRegexTransformer invalidRule = new PointLineReplaceRegexTransformer("", "foo", null, null, metrics);
-  }
-
-  @Test(expected = NullPointerException.class)
-  public void testLineWhitelistRegexNullMatchThrows() {
-    // try to create a whitelist rule with a null match pattern
-    PointLineWhitelistRegexFilter invalidRule = new PointLineWhitelistRegexFilter(null, metrics);
-  }
-
-  @Test(expected = NullPointerException.class)
-  public void testLineBlacklistRegexNullMatchThrows() {
-    // try to create a blacklist rule with a null match pattern
-    PointLineBlacklistRegexFilter invalidRule = new PointLineBlacklistRegexFilter(null, metrics);
-  }
-
-  @Test(expected = NullPointerException.class)
-  public void testPointBlacklistRegexNullScopeThrows() {
-    // try to create a blacklist rule with a null scope
-    ReportPointBlacklistRegexFilter invalidRule = new ReportPointBlacklistRegexFilter(null, "foo", metrics);
-  }
-
-  @Test(expected = NullPointerException.class)
-  public void testPointBlacklistRegexNullMatchThrows() {
-    // try to create a blacklist rule with a null pattern
-    ReportPointBlacklistRegexFilter invalidRule = new ReportPointBlacklistRegexFilter("foo", null, metrics);
-  }
-
-  @Test(expected = NullPointerException.class)
-  public void testPointWhitelistRegexNullScopeThrows() {
-    // try to create a whitelist rule with a null scope
-    ReportPointWhitelistRegexFilter invalidRule = new ReportPointWhitelistRegexFilter(null, "foo", metrics);
-  }
-
-  @Test(expected = NullPointerException.class)
-  public void testPointWhitelistRegexNullMatchThrows() {
-    // try to create a blacklist rule with a null pattern
-    ReportPointWhitelistRegexFilter invalidRule = new ReportPointWhitelistRegexFilter("foo", null, metrics);
-  }
-
-  @Test
-  public void testPointLineRules() {
-    String testPoint1 = "collectd.cpu.loadavg.1m 7 1459527231 source=hostname foo=bar boo=baz";
-    String testPoint2 = "collectd.#cpu#.&loadavg^.1m 7 1459527231 source=source$hostname foo=bar boo=baz";
-    String testPoint3 = "collectd.cpu.loadavg.1m;foo=bar;boo=baz;tag=extra 7 1459527231 source=hostname";
-
-    PointLineReplaceRegexTransformer rule1 = new PointLineReplaceRegexTransformer("(boo)=baz", "$1=qux", null, null, metrics);
-    PointLineReplaceRegexTransformer rule2 = new PointLineReplaceRegexTransformer("[#&\\$\\^]", "", null, null, metrics);
-    PointLineBlacklistRegexFilter rule3 = new PointLineBlacklistRegexFilter(".*source=source.*", metrics);
-    PointLineWhitelistRegexFilter rule4 = new PointLineWhitelistRegexFilter(".*source=source.*", metrics);
-    PointLineReplaceRegexTransformer rule5 = new PointLineReplaceRegexTransformer("cpu", "gpu", ".*hostname.*", null, metrics);
-    PointLineReplaceRegexTransformer rule6 = new PointLineReplaceRegexTransformer("cpu", "gpu", ".*nomatch.*", null, metrics);
-    PointLineReplaceRegexTransformer rule7 = new PointLineReplaceRegexTransformer("([^;]*);([^; ]*)([ ;].*)", "$1$3 $2", null, 2, metrics);
-
-    String expectedPoint1 = "collectd.cpu.loadavg.1m 7 1459527231 source=hostname foo=bar boo=qux";
-    String expectedPoint2 = "collectd.cpu.loadavg.1m 7 1459527231 source=sourcehostname foo=bar boo=baz";
-    String expectedPoint5 = "collectd.gpu.loadavg.1m 7 1459527231 source=hostname foo=bar boo=baz";
-    String expectedPoint7 = "collectd.cpu.loadavg.1m;tag=extra 7 1459527231 source=hostname foo=bar boo=baz";
-
-    assertEquals(expectedPoint1, rule1.apply(testPoint1));
-    assertEquals(expectedPoint2, rule2.apply(testPoint2));
-    assertTrue(rule3.apply(testPoint1));
-    assertFalse(rule3.apply(testPoint2));
-    assertFalse(rule4.apply(testPoint1));
-    assertTrue(rule4.apply(testPoint2));
-    assertEquals(expectedPoint5, rule5.apply(testPoint1));
-    assertEquals(testPoint1, rule6.apply(testPoint1));
-    assertEquals(expectedPoint7, rule7.apply(testPoint3));
-  }
-
-  @Test
-  public void testReportPointRules() {
-    String pointLine = "\"Some Metric\" 10.0 1469751813 source=\"Host\" \"boo\"=\"Baz\" \"foo\"=\"bar\"";
-    ReportPoint point = parsePointLine(pointLine);
-
-    // lowercase a point tag value with no match - shouldn't change anything
-    new ReportPointForceLowercaseTransformer("boo", "nomatch.*", metrics).apply(point);
-    assertEquals(pointLine, referencePointToStringImpl(point));
-
-    // lowercase a point tag value - shouldn't affect metric name or source
-    new ReportPointForceLowercaseTransformer("boo", null, metrics).apply(point);
-    String expectedPoint1a = "\"Some Metric\" 10.0 1469751813 source=\"Host\" \"boo\"=\"baz\" \"foo\"=\"bar\"";
-    assertEquals(expectedPoint1a, referencePointToStringImpl(point));
-
-    // lowercase a metric name - shouldn't affect remaining source
-    new ReportPointForceLowercaseTransformer("metricName", null, metrics).apply(point);
-    String expectedPoint1b = "\"some metric\" 10.0 1469751813 source=\"Host\" \"boo\"=\"baz\" \"foo\"=\"bar\"";
-    assertEquals(expectedPoint1b, referencePointToStringImpl(point));
-
-    // lowercase source
-    new ReportPointForceLowercaseTransformer("sourceName", null, metrics).apply(point);
-    assertEquals(pointLine.toLowerCase(), referencePointToStringImpl(point));
-
-    // try to remove a point tag when value doesn't match the regex - shouldn't change
-    new ReportPointDropTagTransformer("foo", "bar(never|match)", metrics).apply(point);
-    assertEquals(pointLine.toLowerCase(), referencePointToStringImpl(point));
-
-    // try to remove a point tag when value does match the regex - should work
-    new ReportPointDropTagTransformer("foo", "ba.", metrics).apply(point);
-    String expectedPoint1 = "\"some metric\" 10.0 1469751813 source=\"host\" \"boo\"=\"baz\"";
-    assertEquals(expectedPoint1, referencePointToStringImpl(point));
-
-    // try to remove a point tag without a regex specified - should work
-    new ReportPointDropTagTransformer("boo", null, metrics).apply(point);
-    String expectedPoint2 = "\"some metric\" 10.0 1469751813 source=\"host\"";
-    assertEquals(expectedPoint2, referencePointToStringImpl(point));
-
-    // add a point tag back
-    new ReportPointAddTagTransformer("boo", "baz", metrics).apply(point);
-    String expectedPoint3 = "\"some metric\" 10.0 1469751813 source=\"host\" \"boo\"=\"baz\"";
-    assertEquals(expectedPoint3, referencePointToStringImpl(point));
-
-    // try to add a duplicate point tag - shouldn't change
-    new ReportPointAddTagIfNotExistsTransformer("boo", "bar", metrics).apply(point);
-    assertEquals(expectedPoint3, referencePointToStringImpl(point));
-
-    // add another point tag back - should work this time
-    new ReportPointAddTagIfNotExistsTransformer("foo", "bar", metrics).apply(point);
-    assertEquals(pointLine.toLowerCase(), referencePointToStringImpl(point));
-
-    // rename a point tag - should work
-    new ReportPointRenameTagTransformer("foo", "qux", null, metrics).apply(point);
-    String expectedPoint4 = "\"some metric\" 10.0 1469751813 source=\"host\" \"boo\"=\"baz\" \"qux\"=\"bar\"";
-    assertEquals(expectedPoint4, referencePointToStringImpl(point));
-
-    // rename a point tag matching the regex - should work
-    new ReportPointRenameTagTransformer("boo", "foo", "b[a-z]z", metrics).apply(point);
-    String expectedPoint5 = "\"some metric\" 10.0 1469751813 source=\"host\" \"foo\"=\"baz\" \"qux\"=\"bar\"";
-    assertEquals(expectedPoint5, referencePointToStringImpl(point));
-
-    // try to rename a point tag that doesn't match the regex - shouldn't change
-    new ReportPointRenameTagTransformer("foo", "boo", "wat", metrics).apply(point);
-    assertEquals(expectedPoint5, referencePointToStringImpl(point));
-
-    // add null metrics prefix - shouldn't change
-    new ReportPointAddPrefixTransformer(null).apply(point);
-    assertEquals(expectedPoint5, referencePointToStringImpl(point));
-
-    // add blank metrics prefix - shouldn't change
-    new ReportPointAddPrefixTransformer("").apply(point);
-    assertEquals(expectedPoint5, referencePointToStringImpl(point));
-
-    // add metrics prefix - should work
-    new ReportPointAddPrefixTransformer("prefix").apply(point);
-    String expectedPoint6 = "\"prefix.some metric\" 10.0 1469751813 source=\"host\" \"foo\"=\"baz\" \"qux\"=\"bar\"";
-    assertEquals(expectedPoint6, referencePointToStringImpl(point));
-
-    // replace regex in metric name, no matches - shouldn't change
-    new ReportPointReplaceRegexTransformer("metricName", "Z", "", null, null, metrics).apply(point);
-    assertEquals(expectedPoint6, referencePointToStringImpl(point));
-
-    // replace regex in metric name - shouldn't affect anything else
-    new ReportPointReplaceRegexTransformer("metricName", "o", "0", null, null, metrics).apply(point);
-    String expectedPoint7 = "\"prefix.s0me metric\" 10.0 1469751813 source=\"host\" \"foo\"=\"baz\" \"qux\"=\"bar\"";
-    assertEquals(expectedPoint7, referencePointToStringImpl(point));
-
-    // replace regex in source name - shouldn't affect anything else
-    new ReportPointReplaceRegexTransformer("sourceName", "o", "0", null, null, metrics).apply(point);
-    String expectedPoint8 = "\"prefix.s0me metric\" 10.0 1469751813 source=\"h0st\" \"foo\"=\"baz\" \"qux\"=\"bar\"";
-    assertEquals(expectedPoint8, referencePointToStringImpl(point));
-
-    // replace regex in a point tag value - shouldn't affect anything else
-    new ReportPointReplaceRegexTransformer("foo", "b", "z", null, null, metrics).apply(point);
-    String expectedPoint9 = "\"prefix.s0me metric\" 10.0 1469751813 source=\"h0st\" \"foo\"=\"zaz\" \"qux\"=\"bar\"";
-    assertEquals(expectedPoint9, referencePointToStringImpl(point));
-
-    // replace regex in a point tag value with matching groups
-    new ReportPointReplaceRegexTransformer("qux", "([a-c][a-c]).", "$1z", null, null, metrics).apply(point);
-    String expectedPoint10 = "\"prefix.s0me metric\" 10.0 1469751813 source=\"h0st\" \"foo\"=\"zaz\" \"qux\"=\"baz\"";
-    assertEquals(expectedPoint10, referencePointToStringImpl(point));
-
-    // replace regex in a point tag value with placeholders
-    // try to substitute sourceName, a point tag value and a non-existent point tag
-    new ReportPointReplaceRegexTransformer("qux", "az", "{{foo}}-{{no_match}}-g{{sourceName}}", null, null, metrics)
-        .apply(point);
-    String expectedPoint11 =
-        "\"prefix.s0me metric\" 10.0 1469751813 source=\"h0st\" \"foo\"=\"zaz\" \"qux\"=\"bzaz-{{no_match}}-gh0st\"";
-    assertEquals(expectedPoint11, referencePointToStringImpl(point));
-
-  }
-
-  @Test
-  public void testAgentPreprocessorForPointLine() {
-
-    // test point line transformers
-    String testPoint1 = "collectd.#cpu#.&load$avg^.1m 7 1459527231 source=source$hostname foo=bar boo=baz";
-    String expectedPoint1 = "collectd._cpu_._load_avg^.1m 7 1459527231 source=source_hostname foo=bar boo=baz";
-    assertEquals(expectedPoint1, config.forPort("2878").forPointLine().transform(testPoint1));
-
-    // test filters
-    String testPoint2 = "collectd.cpu.loadavg.1m 7 1459527231 source=hostname foo=bar boo=baz";
-    assertTrue(config.forPort("2878").forPointLine().filter(testPoint2));
-
-    String testPoint3 = "collectd.cpu.loadavg.1m 7 1459527231 source=hostname bar=foo boo=baz";
-    assertFalse(config.forPort("2878").forPointLine().filter(testPoint3));
-  }
-
-  @Test
-  public void testAgentPreprocessorForReportPoint() {
-    ReportPoint testPoint1 = parsePointLine("collectd.cpu.loadavg.1m 7 1459527231 source=hostname foo=bar boo=baz");
-    assertTrue(config.forPort("2878").forReportPoint().filter(testPoint1));
-
-    ReportPoint testPoint2 = parsePointLine("foo.collectd.cpu.loadavg.1m 7 1459527231 source=hostname foo=bar boo=baz");
-    assertFalse(config.forPort("2878").forReportPoint().filter(testPoint2));
-
-    ReportPoint testPoint3 = parsePointLine("collectd.cpu.loadavg.1m 7 1459527231 source=hostname foo=west123 boo=baz");
-    assertFalse(config.forPort("2878").forReportPoint().filter(testPoint3));
-
-    ReportPoint testPoint4 = parsePointLine("collectd.cpu.loadavg.1m 7 1459527231 source=bar123 foo=bar boo=baz");
-    assertFalse(config.forPort("2878").forReportPoint().filter(testPoint4));
-
-    // in this test we are confirming that the rule sets for different ports are in fact different
-    // on port 2878 we add "newtagkey=1", on port 4242 we don't
-    ReportPoint testPoint1a = parsePointLine("collectd.cpu.loadavg.1m 7 1459527231 source=hostname foo=bar boo=baz");
-    config.forPort("2878").forReportPoint().transform(testPoint1);
-    config.forPort("4242").forReportPoint().transform(testPoint1a);
-    String expectedPoint1 = "\"collectd.cpu.loadavg.1m\" 7.0 1459527231 " +
-        "source=\"hostname\" \"baz\"=\"bar\" \"boo\"=\"baz\" \"newtagkey\"=\"1\"";
-    String expectedPoint1a = "\"collectd.cpu.loadavg.1m\" 7.0 1459527231 " +
-        "source=\"hostname\" \"baz\"=\"bar\" \"boo\"=\"baz\"";
-    assertEquals(expectedPoint1, referencePointToStringImpl(testPoint1));
-    assertEquals(expectedPoint1a, referencePointToStringImpl(testPoint1a));
-
-    // in this test the following should happen:
-    // - rename foo tag to baz
-    // - "metrictest." prefix gets dropped from the metric name
-    // - replace dashes with dots in bar tag
-    String expectedPoint5 = "\"metric\" 7.0 1459527231 source=\"src\" " +
-        "\"bar\"=\"baz.baz.baz\" \"baz\"=\"bar\" \"datacenter\"=\"az1\" \"newtagkey\"=\"1\" \"qux\"=\"123z\"";
-    assertEquals(expectedPoint5, applyAllTransformers(
-        "metrictest.metric 7 1459527231 source=src foo=bar datacenter=az1 bar=baz-baz-baz qux=123z", "2878"));
-
-    // in this test the following should happen:
-    // - rename tag foo to baz
-    // - add new tag newtagkey=1
-    // - drop dc1 tag
-    // - drop datacenter tag as it matches az[4-6]
-    // - rename qux tag to numericTag
-    String expectedPoint6 = "\"some.metric\" 7.0 1459527231 source=\"hostname\" " +
-        "\"baz\"=\"bar\" \"newtagkey\"=\"1\" \"numericTag\"=\"12345\" \"prefix\"=\"some\"";
-    assertEquals(expectedPoint6, applyAllTransformers(
-        "some.metric 7 1459527231 source=hostname foo=bar dc1=baz datacenter=az4 qux=12345", "2878"));
-
-    // in this test the following should happen:
-    // - fromMetric point tag extracted
-    // - "node2" removed from the metric name
-    // - fromSource point tag extracted
-    // - fromTag point tag extracted
-    String expectedPoint7 = "\"node0.node1.testExtractTag.node4\" 7.0 1459527231 source=\"host0-host1\" " +
-        "\"fromMetric\"=\"node2\" \"fromSource\"=\"host2\" \"fromTag\"=\"tag0\" " +
-        "\"testExtractTag\"=\"tag0.tag1.tag2.tag3.tag4\"";
-    assertEquals(expectedPoint7, applyAllTransformers(
-        "node0.node1.node2.testExtractTag.node4 7.0 1459527231 source=host0-host1-host2 " +
-            "testExtractTag=tag0.tag1.tag2.tag3.tag4", "1234"));
-
-  }
-
-  @Test
-  public void testAllFilters() {
-    assertTrue(applyAllFilters("valid.metric.loadavg.1m 7 1459527231 source=h.prod.corp foo=bar boo=baz", "1111"));
-    assertTrue(applyAllFilters("valid.metric.loadavg.1m 7 1459527231 source=h.prod.corp foo=b_r boo=baz", "1111"));
-    assertTrue(applyAllFilters("valid.metric.loadavg.1m 7 1459527231 source=h.prod.corp foo=b_r boo=baz", "1111"));
-    assertFalse(applyAllFilters("invalid.metric.loadavg.1m 7 1459527231 source=h.prod.corp foo=bar boo=baz", "1111"));
-    assertFalse(applyAllFilters("valid.metric.loadavg.1m 7 1459527231 source=h.prod.corp foo=bar baz=boo", "1111"));
-    assertFalse(applyAllFilters("valid.metric.loadavg.1m 7 1459527231 source=h.dev.corp foo=bar boo=baz", "1111"));
-    assertFalse(applyAllFilters("valid.metric.loadavg.1m 7 1459527231 source=h.prod.corp foo=bar boo=stop", "1111"));
-    assertFalse(applyAllFilters("loadavg.1m 7 1459527231 source=h.prod.corp foo=bar boo=baz", "1111"));
-  }
-
-  private boolean applyAllFilters(String pointLine, String strPort) {
-    if (!config.forPort(strPort).forPointLine().filter(pointLine))
-      return false;
-    ReportPoint point = parsePointLine(pointLine);
-    return config.forPort(strPort).forReportPoint().filter(point);
-  }
-
-  private String applyAllTransformers(String pointLine, String strPort) {
-    String transformedPointLine = config.forPort(strPort).forPointLine().transform(pointLine);
-    ReportPoint point = parsePointLine(transformedPointLine);
-    config.forPort(strPort).forReportPoint().transform(point);
-    return referencePointToStringImpl(point);
-  }
-
-  private static String referencePointToStringImpl(ReportPoint point) {
-    String toReturn = String.format("\"%s\" %s %d source=\"%s\"",
-        point.getMetric().replaceAll("\"", "\\\""),
-        point.getValue(),
-        point.getTimestamp() / 1000,
-        point.getHost().replaceAll("\"", "\\\""));
-    for (Map.Entry entry : point.getAnnotations().entrySet()) {
-      toReturn += String.format(" \"%s\"=\"%s\"",
-          entry.getKey().replaceAll("\"", "\\\""),
-          entry.getValue().replaceAll("\"", "\\\""));
-    }
-    return toReturn;
-  }
-
-  private ReportPoint parsePointLine(String pointLine) {
-    List points = Lists.newArrayListWithExpectedSize(1);
-    decoder.decodeReportPoints(pointLine, points, "dummy");
-    ReportPoint point = points.get(0);
-    // convert annotations to TreeMap so the result is deterministic
-    point.setAnnotations(new TreeMap<>(point.getAnnotations()));
-    return point;
-  }
-}
diff --git a/proxy/src/test/java/com/wavefront/agent/preprocessor/ProxyPreprocessorConfigurationTest.java b/proxy/src/test/java/com/wavefront/agent/preprocessor/ProxyPreprocessorConfigurationTest.java
new file mode 100644
index 000000000..ff969c8e0
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/preprocessor/ProxyPreprocessorConfigurationTest.java
@@ -0,0 +1,271 @@
+package com.wavefront.agent.preprocessor;
+
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertNotNull;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.nio.file.Files;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.function.Supplier;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import com.wavefront.agent.ProxyCheckInScheduler;
+import com.wavefront.api.agent.preprocessor.ReportPointAddPrefixTransformer;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Test;
+import wavefront.report.ReportPoint;
+
+public class ProxyPreprocessorConfigurationTest {
+    final String tempRules = "'3000':\n  - rule    : drop-ts-tag\n    action  : dropTag\n    tag   : ts\n    match  : ts.*";
+
+    private static File createTempFile(String content) throws IOException {
+        File tempFile = File.createTempFile("preprocessor", ".yaml");
+        Files.write(tempFile.toPath(), content.getBytes(StandardCharsets.UTF_8));
+        tempFile.deleteOnExit(); // Clean up on JVM exit
+        return tempFile;
+    }
+
+    /**
+     * Test that system rules applied before user rules
+     */
+    @Test
+    public void testPreprocessorRulesOrder() {
+        InputStream stream =
+                ProxyPreprocessorConfigurationTest.class.getResourceAsStream("preprocessor_rules_order_test.yaml");
+        ProxyPreprocessorConfigManager config = new ProxyPreprocessorConfigManager();
+        config.loadFromStream(stream);
+        config
+                .getSystemPreprocessor("2878")
+                .forReportPoint()
+                .addTransformer(new ReportPointAddPrefixTransformer("methodPrefix"));
+        ReportPoint point =
+                new ReportPoint(
+                        "testMetric", System.currentTimeMillis(), 10, "host", "table", new HashMap<>());
+        config.get("2878").get().forReportPoint().transform(point);
+        assertEquals("methodPrefix.testMetric", point.getMetric());
+    }
+
+    @Test
+    public void testMultiPortPreprocessorRules() {
+        // test that preprocessor rules take priority over local rules
+        InputStream stream =
+                ProxyPreprocessorConfigurationTest.class.getResourceAsStream("preprocessor_rules_multiport.yaml");
+        ProxyPreprocessorConfigManager config = new ProxyPreprocessorConfigManager();
+        config.loadFromStream(stream);
+        ReportPoint point =
+                new ReportPoint(
+                        "metric1", System.currentTimeMillis(), 4, "host", "table", new HashMap<>());
+        config.get("2879").get().forReportPoint().transform(point);
+        assertEquals("metric1", point.getMetric());
+        assertEquals(1, point.getAnnotations().size());
+        assertEquals("multiTagVal", point.getAnnotations().get("multiPortTagKey"));
+
+        ReportPoint point1 =
+                new ReportPoint(
+                        "metric2", System.currentTimeMillis(), 4, "host", "table", new HashMap<>());
+        config.get("1111").get().forReportPoint().transform(point1);
+        assertEquals("metric2", point1.getMetric());
+        assertEquals(1, point1.getAnnotations().size());
+        assertEquals("multiTagVal", point1.getAnnotations().get("multiPortTagKey"));
+    }
+
+    @Test
+    public void testEmptyRules() {
+        InputStream stream = new ByteArrayInputStream("".getBytes());
+        ProxyPreprocessorConfigManager config = new ProxyPreprocessorConfigManager();
+        config.loadFromStream(stream);
+    }
+
+    @Test
+    public void testLoadFERulesAndGetProxyConfigRules() {
+        ProxyPreprocessorConfigManager config = new ProxyPreprocessorConfigManager();
+        String testRules = tempRules;
+
+        // Get initial state of the flag and reset
+        boolean initialIsRulesSetInFE = ProxyCheckInScheduler.isRulesSetInFE.get();
+        boolean initialPreprocessorRulesNeedUpdate = ProxyCheckInScheduler.preprocessorRulesNeedUpdate.get();
+        ProxyCheckInScheduler.isRulesSetInFE.set(false); // Ensure FE rules are processed
+
+        config.loadFERules(testRules);
+
+        assertNotNull(config.get("3000").get());
+        assertEquals(testRules, ProxyPreprocessorConfigManager.getProxyConfigRules());
+        assertTrue(ProxyCheckInScheduler.preprocessorRulesNeedUpdate.get());
+
+        // cleanup
+        ProxyCheckInScheduler.isRulesSetInFE.set(initialIsRulesSetInFE);
+        ProxyCheckInScheduler.preprocessorRulesNeedUpdate.set(initialPreprocessorRulesNeedUpdate);
+
+        ProxyPreprocessorConfigManager.clearProxyConfigRules();
+    }
+
+    @Test
+    public void testGetFileRulesReadsFileContent() throws IOException {
+        String fileContent = "key: value\nlist:\n  - item1\n  - item2\n";
+        File tempFile = createTempFile(fileContent);
+        String readContent = ProxyPreprocessorConfigManager.getFileRules(tempFile.getAbsolutePath());
+
+        assertEquals(fileContent, readContent);
+    }
+
+    /**
+     * Test that getFileRules throws a RuntimeException when given a non-existent file path.
+     */
+    @Test(expected = RuntimeException.class)
+    public void testGetFileRulesNonExistentFileThrowsRuntimeException() {
+        ProxyPreprocessorConfigManager.getFileRules("/path/to/nonexistent/file.yaml");
+    }
+
+    /**
+     * Test that getFileRules returns null for empty or null file names.
+     */
+    @Test
+    public void testGetFileRulesEmptyFileNameReturnsNull() {
+        assertNull(ProxyPreprocessorConfigManager.getFileRules(""));
+        assertNull(ProxyPreprocessorConfigManager.getFileRules(null));
+    }
+
+    /**
+     * Test loadFileIfModified skips loading if isRulesSetInFE is true.
+     */
+    @Test
+    public void testLoadFileIfModifiedSkipsIfFERulesSet() throws IOException {
+        File tempFile = createTempFile("tempRules");
+
+        ProxyPreprocessorConfigManager config = EasyMock.partialMockBuilder(ProxyPreprocessorConfigManager.class)
+                .addMockedMethod("loadFile", String.class)
+                .withConstructor().createMock();
+
+        replay(config);
+
+        boolean initialIsRulesSetInFE = ProxyCheckInScheduler.isRulesSetInFE.get();
+        ProxyCheckInScheduler.isRulesSetInFE.set(true);
+
+        config.loadFileIfModified(tempFile.getAbsolutePath());
+
+        // verify loadFile was not called
+        verify(config);
+
+        // cleanup
+        ProxyCheckInScheduler.isRulesSetInFE.set(initialIsRulesSetInFE);
+    }
+
+    /**
+     * Test that loadFileIfModified reloads rules and updates timestamp when file is updated.
+     */
+    @Test
+    public void testLoadFileIfModifiedReloadsAndUpdatesTimestamp() throws IOException, FileNotFoundException {
+        String expectedRules = tempRules;
+        File tempFile = createTempFile(expectedRules); // Create a file with content
+
+        Supplier mockTimeSupplier = EasyMock.mock(Supplier.class);
+
+        expect(mockTimeSupplier.get()).andReturn(1000L).once();
+        expect(mockTimeSupplier.get()).andReturn(tempFile.lastModified() + 100L).anyTimes(); // loadFileIfModified checks updated lastModified
+        replay(mockTimeSupplier);
+
+        ProxyPreprocessorConfigManager config = new ProxyPreprocessorConfigManager(mockTimeSupplier);
+
+        config.loadFileIfModified(tempFile.getAbsolutePath());
+
+        verify(mockTimeSupplier);
+
+        // test loadFile results in updated proxyConfigRules
+        assertEquals(expectedRules, ProxyPreprocessorConfigManager.getProxyConfigRules());
+        assertTrue(ProxyCheckInScheduler.preprocessorRulesNeedUpdate.get());
+
+        // cleanup
+        ProxyCheckInScheduler.preprocessorRulesNeedUpdate.set(false);
+    }
+
+    /**
+     * Test that loadFileIfModified does not reload if the file's last modified timestamp is not updated.
+     */
+    @Test
+    public void testLoadFileIfModifiedDoesNotReloadIfFileNotNewer() throws IOException {
+        File tempFile = createTempFile(tempRules);
+
+        Supplier mockTimeSupplier = EasyMock.mock(Supplier.class);
+        expect(mockTimeSupplier.get()).andReturn(tempFile.lastModified()).once();
+        // mock same timestamp to indicate no time has passed
+        expect(mockTimeSupplier.get()).andReturn(tempFile.lastModified()).anyTimes();
+        replay(mockTimeSupplier);
+
+        ProxyPreprocessorConfigManager config = new ProxyPreprocessorConfigManager(mockTimeSupplier);
+        config.loadFileIfModified(tempFile.getAbsolutePath());
+
+        // Verify mocks (will fail if loadFile or getFileRules were unexpectedly called)
+        verify(mockTimeSupplier);
+
+        // Assert that proxyConfigRules was not updated
+        assertNull(ProxyPreprocessorConfigManager.getProxyConfigRules());
+    }
+
+    @Test
+    public void testLoadFileIfModifiedHandlesExceptionInLoadFile() throws IOException, FileNotFoundException {
+        File tempFile = createTempFile("rules that will cause exception");
+
+        Supplier mockTimeSupplier = EasyMock.mock(Supplier.class);
+        expect(mockTimeSupplier.get()).andReturn(1000L).once(); // For constructor
+        expect(mockTimeSupplier.get()).andReturn(tempFile.lastModified() + 100L).anyTimes(); // Simulate newer file
+        replay(mockTimeSupplier);
+
+        ProxyPreprocessorConfigManager config = EasyMock.partialMockBuilder(ProxyPreprocessorConfigManager.class)
+                .addMockedMethod("loadFile", String.class)
+                .withConstructor(mockTimeSupplier)
+                .createMock();
+
+        config.loadFile(tempFile.getAbsolutePath());
+        expectLastCall().andThrow(new FileNotFoundException("Simulated error")).once();
+
+        replay(config);
+
+        config.loadFileIfModified(tempFile.getAbsolutePath());
+
+        verify(mockTimeSupplier, config);
+    }
+
+    /**
+     * Test setUpConfigFileMonitoring schedules a TimerTask that periodically calls loadFileIfModified.
+     */
+    @Test
+    public void testSetUpConfigFileMonitoring() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        ProxyPreprocessorConfigManager config = EasyMock.partialMockBuilder(ProxyPreprocessorConfigManager.class)
+            .addMockedMethod("loadFileIfModified", String.class)
+            .withConstructor()
+            .createMock();
+
+        String dummyFileName = "test.yaml";
+        int checkInterval = 100; // 100ms
+
+        config.loadFileIfModified(dummyFileName);
+        // countdown once loadFileIfModified is called by timer to mock time passing
+        expectLastCall().andAnswer(() -> {
+            latch.countDown();
+            return null;
+        }).atLeastOnce();
+
+        replay(config);
+
+        config.setUpConfigFileMonitoring(dummyFileName, checkInterval);
+
+        assertTrue("loadFileIfModified was not called within the timeout.", latch.await(500, TimeUnit.MILLISECONDS));
+        verify(config);
+    }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/queueing/ConcurrentShardedQueueFileTest.java b/proxy/src/test/java/com/wavefront/agent/queueing/ConcurrentShardedQueueFileTest.java
new file mode 100644
index 000000000..ebf7a55a3
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/queueing/ConcurrentShardedQueueFileTest.java
@@ -0,0 +1,141 @@
+package com.wavefront.agent.queueing;
+
+import static com.wavefront.agent.queueing.ConcurrentShardedQueueFile.incrementFileName;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import com.squareup.tape2.QueueFile;
+import com.wavefront.common.Pair;
+import java.io.File;
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Queue;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.junit.Test;
+
+/** @author vasily@wavefront.com */
+public class ConcurrentShardedQueueFileTest {
+  private static final Random RANDOM = new Random();
+
+  @Test
+  public void nextFileNameTest() {
+    assertEquals("points.2878.1_0000", incrementFileName("points.2878.1", ".spool"));
+    assertEquals("points.2878.1.spool_0000", incrementFileName("points.2878.1.spool", ".spool"));
+    assertEquals(
+        "points.2878.1.spool_0001", incrementFileName("points.2878.1.spool_0000", ".spool"));
+    assertEquals(
+        "points.2878.1.spool_0002", incrementFileName("points.2878.1.spool_0001", ".spool"));
+    assertEquals(
+        "points.2878.1.spool_000a", incrementFileName("points.2878.1.spool_0009", ".spool"));
+    assertEquals(
+        "points.2878.1.spool_0010", incrementFileName("points.2878.1.spool_000f", ".spool"));
+    assertEquals(
+        "points.2878.1.spool_0100", incrementFileName("points.2878.1.spool_00ff", ".spool"));
+    assertEquals(
+        "points.2878.1.spool_ffff", incrementFileName("points.2878.1.spool_fffe", ".spool"));
+    assertEquals(
+        "points.2878.1.spool_0000", incrementFileName("points.2878.1.spool_ffff", ".spool"));
+  }
+
+  @Test
+  public void testConcurrency() throws Exception {
+    File file = new File(File.createTempFile("proxyConcurrencyTest", null).getPath() + ".spool");
+    ConcurrentShardedQueueFile queueFile =
+        new ConcurrentShardedQueueFile(
+            file.getCanonicalPath(),
+            ".spool",
+            1024 * 1024,
+            s -> new TapeQueueFile(new QueueFile.Builder(new File(s)).build()));
+    Queue> taskCheatSheet = new ArrayDeque<>();
+    System.out.println(queueFile.shards.size());
+    AtomicLong tasksGenerated = new AtomicLong();
+    AtomicLong nanosAdd = new AtomicLong();
+    AtomicLong nanosGet = new AtomicLong();
+    while (queueFile.shards.size() < 4) {
+      byte[] task = randomTask();
+      queueFile.add(task);
+      taskCheatSheet.add(Pair.of(task.length, task[0]));
+      tasksGenerated.incrementAndGet();
+    }
+    AtomicBoolean done = new AtomicBoolean(false);
+    AtomicBoolean fail = new AtomicBoolean(false);
+    Runnable addTask =
+        () -> {
+          int delay = 0;
+          while (!done.get() && !fail.get()) {
+            try {
+              byte[] task = randomTask();
+              long start = System.nanoTime();
+              queueFile.add(task);
+              nanosAdd.addAndGet(System.nanoTime() - start);
+              taskCheatSheet.add(Pair.of(task.length, task[0]));
+              tasksGenerated.incrementAndGet();
+              Thread.sleep(delay / 1000);
+              delay++;
+            } catch (Exception e) {
+              e.printStackTrace();
+              fail.set(true);
+            }
+          }
+        };
+    Runnable getTask =
+        () -> {
+          int delay = 2000;
+          while (!taskCheatSheet.isEmpty() && !fail.get()) {
+            try {
+              long start = System.nanoTime();
+              Pair taskData = taskCheatSheet.remove();
+              byte[] task = queueFile.peek();
+              queueFile.remove();
+              nanosGet.addAndGet(System.nanoTime() - start);
+              if (taskData._1 != task.length) {
+                System.out.println(
+                    "Data integrity fail! Expected: "
+                        + taskData._1
+                        + " bytes, got "
+                        + task.length
+                        + " bytes");
+                fail.set(true);
+              }
+              for (byte b : task) {
+                if (taskData._2 != b) {
+                  System.out.println("Data integrity fail! Expected " + taskData._2 + ", got " + b);
+                  fail.set(true);
+                }
+              }
+              Thread.sleep(delay / 500);
+              if (delay > 0) delay--;
+            } catch (Exception e) {
+              e.printStackTrace();
+              fail.set(true);
+            }
+          }
+          done.set(true);
+        };
+    ExecutorService executor = Executors.newFixedThreadPool(2);
+    long start = System.nanoTime();
+    Future addFuture = executor.submit(addTask);
+    Future getFuture = executor.submit(getTask);
+    addFuture.get();
+    getFuture.get();
+    assertFalse(fail.get());
+    System.out.println("Tasks generated: " + tasksGenerated.get());
+    System.out.println("Real time (ms) = " + (System.nanoTime() - start) / 1_000_000);
+    System.out.println("Add + remove time (ms) = " + (nanosGet.get() + nanosAdd.get()) / 1_000_000);
+    System.out.println("Add time (ms) = " + nanosAdd.get() / 1_000_000);
+    System.out.println("Remove time (ms) = " + nanosGet.get() / 1_000_000);
+  }
+
+  private byte[] randomTask() {
+    int size = RANDOM.nextInt(32 * 1024) + 1;
+    byte[] result = new byte[size];
+    RANDOM.nextBytes(result);
+    Arrays.fill(result, result[0]);
+    return result;
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/queueing/InMemorySubmissionQueueTest.java b/proxy/src/test/java/com/wavefront/agent/queueing/InMemorySubmissionQueueTest.java
new file mode 100644
index 000000000..a0bf0358d
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/queueing/InMemorySubmissionQueueTest.java
@@ -0,0 +1,152 @@
+package com.wavefront.agent.queueing;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.data.DefaultEntityPropertiesForTesting;
+import com.wavefront.agent.data.EventDataSubmissionTask;
+import com.wavefront.agent.data.LineDelimitedDataSubmissionTask;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.agent.data.SourceTagSubmissionTask;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.dto.Event;
+import com.wavefront.dto.SourceTag;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import wavefront.report.ReportEvent;
+import wavefront.report.ReportSourceTag;
+import wavefront.report.SourceOperationType;
+import wavefront.report.SourceTagAction;
+
+/** @author mike@wavefront.com */
+@RunWith(Parameterized.class)
+public class InMemorySubmissionQueueTest> {
+  private final T expectedTask;
+  private final AtomicLong time = new AtomicLong(77777);
+
+  public InMemorySubmissionQueueTest(TaskConverter.CompressionType compressionType, T task) {
+    this.expectedTask = task;
+    System.out.println(task.getClass().getSimpleName() + " compression type: " + compressionType);
+  }
+
+  @Parameterized.Parameters
+  public static Collection scenarios() {
+    Collection scenarios = new ArrayList<>();
+    for (TaskConverter.CompressionType type : TaskConverter.CompressionType.values()) {
+      RetryTaskConverter converter =
+          new RetryTaskConverter<>("2878", type);
+      LineDelimitedDataSubmissionTask task =
+          converter.fromBytes(
+              "WF\u0001\u0001{\"__CLASS\":\"com.wavefront.agent.data.LineDelimitedDataSubmissionTask\",\"enqueuedTimeMillis\":77777,\"attempts\":0,\"serverErrors\":0,\"handle\":\"2878\",\"entityType\":\"POINT\",\"format\":\"wavefront\",\"payload\":[\"java.util.ArrayList\",[\"item1\",\"item2\",\"item3\"]],\"enqueuedMillis\":77777}"
+                  .getBytes());
+      scenarios.add(new Object[] {type, task});
+    }
+    for (TaskConverter.CompressionType type : TaskConverter.CompressionType.values()) {
+      RetryTaskConverter converter =
+          new RetryTaskConverter<>("2878", type);
+      EventDataSubmissionTask task =
+          converter.fromBytes(
+              "WF\u0001\u0001{\"__CLASS\":\"com.wavefront.agent.data.EventDataSubmissionTask\",\"enqueuedTimeMillis\":77777,\"attempts\":0,\"serverErrors\":0,\"handle\":\"2878\",\"entityType\":\"EVENT\",\"events\":[\"java.util.ArrayList\",[{\"name\":\"Event name for testing\",\"startTime\":77777000,\"endTime\":77777001,\"annotations\":[\"java.util.HashMap\",{\"severity\":\"INFO\"}],\"dimensions\":[\"java.util.HashMap\",{\"multi\":[\"java.util.ArrayList\",[\"bar\",\"baz\"]]}],\"hosts\":[\"java.util.ArrayList\",[\"host1\",\"host2\"]],\"tags\":[\"java.util.ArrayList\",[\"tag1\"]]}]],\"enqueuedMillis\":77777}"
+                  .getBytes());
+      scenarios.add(new Object[] {type, task});
+    }
+    for (TaskConverter.CompressionType type : TaskConverter.CompressionType.values()) {
+      RetryTaskConverter converter =
+          new RetryTaskConverter<>("2878", type);
+      SourceTagSubmissionTask task =
+          converter.fromBytes(
+              "WF\u0001\u0001{\"__CLASS\":\"com.wavefront.agent.data.SourceTagSubmissionTask\",\"enqueuedTimeMillis\":77777,\"attempts\":0,\"serverErrors\":0,\"handle\":\"2878\",\"entityType\":\"SOURCE_TAG\",\"limitRetries\":true,\"sourceTag\":{\"operation\":\"SOURCE_TAG\",\"action\":\"SAVE\",\"source\":\"testSource\",\"annotations\":[\"java.util.ArrayList\",[\"newtag1\",\"newtag2\"]]},\"enqueuedMillis\":77777}\n"
+                  .getBytes());
+      scenarios.add(new Object[] {type, task});
+    }
+    return scenarios;
+  }
+
+  @Test
+  public void testTaskRead() {
+    TaskQueue queue = new InMemorySubmissionQueue<>();
+    UUID proxyId = UUID.randomUUID();
+    DataSubmissionTask> task = null;
+    if (this.expectedTask instanceof LineDelimitedDataSubmissionTask) {
+      task =
+          new LineDelimitedDataSubmissionTask(
+              null,
+              proxyId,
+              new DefaultEntityPropertiesForTesting(),
+              queue,
+              "wavefront",
+              ReportableEntityType.POINT,
+              "2878",
+              ImmutableList.of("item1", "item2", "item3"),
+              time::get);
+    } else if (this.expectedTask instanceof EventDataSubmissionTask) {
+      task =
+          new EventDataSubmissionTask(
+              null,
+              proxyId,
+              new DefaultEntityPropertiesForTesting(),
+              queue,
+              "2878",
+              ImmutableList.of(
+                  new Event(
+                      ReportEvent.newBuilder()
+                          .setStartTime(time.get() * 1000)
+                          .setEndTime(time.get() * 1000 + 1)
+                          .setName("Event name for testing")
+                          .setHosts(ImmutableList.of("host1", "host2"))
+                          .setDimensions(ImmutableMap.of("multi", ImmutableList.of("bar", "baz")))
+                          .setAnnotations(ImmutableMap.of("severity", "INFO"))
+                          .setTags(ImmutableList.of("tag1"))
+                          .build())),
+              time::get);
+    } else if (this.expectedTask instanceof SourceTagSubmissionTask) {
+      task =
+          new SourceTagSubmissionTask(
+              null,
+              new DefaultEntityPropertiesForTesting(),
+              queue,
+              "2878",
+              new SourceTag(
+                  ReportSourceTag.newBuilder()
+                      .setOperation(SourceOperationType.SOURCE_TAG)
+                      .setAction(SourceTagAction.SAVE)
+                      .setSource("testSource")
+                      .setAnnotations(ImmutableList.of("newtag1", "newtag2"))
+                      .build()),
+              time::get);
+    }
+    assertNotNull(task);
+    task.enqueue(QueueingReason.RETRY);
+
+    if (this.expectedTask instanceof LineDelimitedDataSubmissionTask) {
+      LineDelimitedDataSubmissionTask readTask = (LineDelimitedDataSubmissionTask) queue.peek();
+      assertNotNull(readTask);
+      assertEquals(((LineDelimitedDataSubmissionTask) task).payload(), readTask.payload());
+      assertEquals(
+          ((LineDelimitedDataSubmissionTask) this.expectedTask).payload(), readTask.payload());
+      assertEquals(77777, readTask.getEnqueuedMillis());
+    }
+    if (this.expectedTask instanceof EventDataSubmissionTask) {
+      EventDataSubmissionTask readTask = (EventDataSubmissionTask) queue.peek();
+      assertNotNull(readTask);
+      assertEquals(((EventDataSubmissionTask) task).payload(), readTask.payload());
+      assertEquals(((EventDataSubmissionTask) this.expectedTask).payload(), readTask.payload());
+      assertEquals(77777, readTask.getEnqueuedMillis());
+    }
+    if (this.expectedTask instanceof SourceTagSubmissionTask) {
+      SourceTagSubmissionTask readTask = (SourceTagSubmissionTask) queue.peek();
+      assertNotNull(readTask);
+      assertEquals(((SourceTagSubmissionTask) task).payload(), readTask.payload());
+      assertEquals(((SourceTagSubmissionTask) this.expectedTask).payload(), readTask.payload());
+      assertEquals(77777, readTask.getEnqueuedMillis());
+    }
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/queueing/InstrumentedTaskQueueDelegateTest.java b/proxy/src/test/java/com/wavefront/agent/queueing/InstrumentedTaskQueueDelegateTest.java
new file mode 100644
index 000000000..caeee4bb0
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/queueing/InstrumentedTaskQueueDelegateTest.java
@@ -0,0 +1,146 @@
+package com.wavefront.agent.queueing;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.squareup.tape2.QueueFile;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.data.DefaultEntityPropertiesForTesting;
+import com.wavefront.agent.data.EventDataSubmissionTask;
+import com.wavefront.agent.data.LineDelimitedDataSubmissionTask;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.agent.data.SourceTagSubmissionTask;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.dto.Event;
+import com.wavefront.dto.SourceTag;
+import java.io.File;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+import org.junit.Test;
+import wavefront.report.ReportEvent;
+import wavefront.report.ReportSourceTag;
+import wavefront.report.SourceOperationType;
+import wavefront.report.SourceTagAction;
+
+/**
+ * Tests object serialization.
+ *
+ * @author vasily@wavefront.com
+ */
+public class InstrumentedTaskQueueDelegateTest {
+
+  @Test
+  public void testLineDelimitedTask() throws Exception {
+    AtomicLong time = new AtomicLong(77777);
+    for (RetryTaskConverter.CompressionType type : RetryTaskConverter.CompressionType.values()) {
+      System.out.println("LineDelimited task, compression type: " + type);
+      File file = new File(File.createTempFile("proxyTestConverter", null).getPath() + ".queue");
+      file.deleteOnExit();
+      TaskQueue queue = getTaskQueue(file, type);
+      queue.clear();
+      UUID proxyId = UUID.randomUUID();
+      LineDelimitedDataSubmissionTask task =
+          new LineDelimitedDataSubmissionTask(
+              null,
+              proxyId,
+              new DefaultEntityPropertiesForTesting(),
+              queue,
+              "wavefront",
+              ReportableEntityType.POINT,
+              "2878",
+              ImmutableList.of("item1", "item2", "item3"),
+              time::get);
+      task.enqueue(QueueingReason.RETRY);
+      queue.close();
+      TaskQueue readQueue = getTaskQueue(file, type);
+      LineDelimitedDataSubmissionTask readTask = readQueue.peek();
+      assertEquals(task.payload(), readTask.payload());
+      assertEquals(77777, readTask.getEnqueuedMillis());
+    }
+  }
+
+  @Test
+  public void testSourceTagTask() throws Exception {
+    for (RetryTaskConverter.CompressionType type : RetryTaskConverter.CompressionType.values()) {
+      System.out.println("SourceTag task, compression type: " + type);
+      File file = new File(File.createTempFile("proxyTestConverter", null).getPath() + ".queue");
+      file.deleteOnExit();
+      TaskQueue queue = getTaskQueue(file, type);
+      queue.clear();
+      SourceTagSubmissionTask task =
+          new SourceTagSubmissionTask(
+              null,
+              new DefaultEntityPropertiesForTesting(),
+              queue,
+              "2878",
+              new SourceTag(
+                  ReportSourceTag.newBuilder()
+                      .setOperation(SourceOperationType.SOURCE_TAG)
+                      .setAction(SourceTagAction.SAVE)
+                      .setSource("testSource")
+                      .setAnnotations(ImmutableList.of("newtag1", "newtag2"))
+                      .build()),
+              () -> 77777L);
+      task.enqueue(QueueingReason.RETRY);
+      queue.close();
+      TaskQueue readQueue = getTaskQueue(file, type);
+      SourceTagSubmissionTask readTask = readQueue.peek();
+      assertEquals(task.payload(), readTask.payload());
+      assertEquals(77777, readTask.getEnqueuedMillis());
+    }
+  }
+
+  @Test
+  public void testEventTask() throws Exception {
+    AtomicLong time = new AtomicLong(77777);
+    for (RetryTaskConverter.CompressionType type : RetryTaskConverter.CompressionType.values()) {
+      System.out.println("Event task, compression type: " + type);
+      File file = new File(File.createTempFile("proxyTestConverter", null).getPath() + ".queue");
+      file.deleteOnExit();
+      TaskQueue queue = getTaskQueue(file, type);
+      queue.clear();
+      UUID proxyId = UUID.randomUUID();
+      EventDataSubmissionTask task =
+          new EventDataSubmissionTask(
+              null,
+              proxyId,
+              new DefaultEntityPropertiesForTesting(),
+              queue,
+              "2878",
+              ImmutableList.of(
+                  new Event(
+                      ReportEvent.newBuilder()
+                          .setStartTime(time.get() * 1000)
+                          .setEndTime(time.get() * 1000 + 1)
+                          .setName("Event name for testing")
+                          .setHosts(ImmutableList.of("host1", "host2"))
+                          .setDimensions(ImmutableMap.of("multi", ImmutableList.of("bar", "baz")))
+                          .setAnnotations(ImmutableMap.of("severity", "INFO"))
+                          .setTags(ImmutableList.of("tag1"))
+                          .build())),
+              time::get);
+      task.enqueue(QueueingReason.RETRY);
+      queue.close();
+      TaskQueue readQueue = getTaskQueue(file, type);
+      EventDataSubmissionTask readTask = readQueue.peek();
+      assertEquals(task.payload(), readTask.payload());
+      assertEquals(77777, readTask.getEnqueuedMillis());
+    }
+  }
+
+  private > TaskQueue getTaskQueue(
+      File file, RetryTaskConverter.CompressionType compressionType) throws Exception {
+    return new InstrumentedTaskQueueDelegate<>(
+        new FileBasedTaskQueue<>(
+            new ConcurrentShardedQueueFile(
+                file.getCanonicalPath(),
+                ".spool",
+                16 * 1024,
+                s -> new TapeQueueFile(new QueueFile.Builder(new File(s)).build())),
+            new RetryTaskConverter("2878", compressionType)),
+        null,
+        null,
+        null);
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/queueing/QueueExporterTest.java b/proxy/src/test/java/com/wavefront/agent/queueing/QueueExporterTest.java
new file mode 100644
index 000000000..8953e5dd9
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/queueing/QueueExporterTest.java
@@ -0,0 +1,248 @@
+package com.wavefront.agent.queueing;
+
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Files;
+import com.wavefront.agent.data.DefaultEntityPropertiesFactoryForTesting;
+import com.wavefront.agent.data.DefaultEntityPropertiesForTesting;
+import com.wavefront.agent.data.EntityPropertiesFactory;
+import com.wavefront.agent.data.EventDataSubmissionTask;
+import com.wavefront.agent.data.LineDelimitedDataSubmissionTask;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.agent.data.SourceTagSubmissionTask;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.dto.Event;
+import com.wavefront.dto.SourceTag;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import wavefront.report.ReportEvent;
+import wavefront.report.ReportSourceTag;
+import wavefront.report.SourceOperationType;
+import wavefront.report.SourceTagAction;
+
+/** @author vasily@wavefront.com */
+public class QueueExporterTest {
+
+  @Test
+  public void testQueueExporter() throws Exception {
+    File file = new File(File.createTempFile("proxyTestConverter", null).getPath() + ".queue");
+    file.deleteOnExit();
+    String bufferFile = file.getAbsolutePath();
+    TaskQueueFactory taskQueueFactory = new TaskQueueFactoryImpl(bufferFile, false, false, 128);
+    EntityPropertiesFactory entityPropFactory = new DefaultEntityPropertiesFactoryForTesting();
+    QueueExporter qe =
+        new QueueExporter(
+            bufferFile, "2878", bufferFile + "-output", false, taskQueueFactory, entityPropFactory);
+    BufferedWriter mockedWriter = EasyMock.createMock(BufferedWriter.class);
+    reset(mockedWriter);
+    HandlerKey key = HandlerKey.of(ReportableEntityType.POINT, "2878");
+    TaskQueue queue = taskQueueFactory.getTaskQueue(key, 0);
+    queue.clear();
+    UUID proxyId = UUID.randomUUID();
+    LineDelimitedDataSubmissionTask task =
+        new LineDelimitedDataSubmissionTask(
+            null,
+            proxyId,
+            new DefaultEntityPropertiesForTesting(),
+            queue,
+            "wavefront",
+            ReportableEntityType.POINT,
+            "2878",
+            ImmutableList.of("item1", "item2", "item3"),
+            () -> 12345L);
+    task.enqueue(QueueingReason.RETRY);
+    LineDelimitedDataSubmissionTask task2 =
+        new LineDelimitedDataSubmissionTask(
+            null,
+            proxyId,
+            new DefaultEntityPropertiesForTesting(),
+            queue,
+            "wavefront",
+            ReportableEntityType.POINT,
+            "2878",
+            ImmutableList.of("item4", "item5"),
+            () -> 12345L);
+    task2.enqueue(QueueingReason.RETRY);
+    mockedWriter.write("item1");
+    mockedWriter.newLine();
+    mockedWriter.write("item2");
+    mockedWriter.newLine();
+    mockedWriter.write("item3");
+    mockedWriter.newLine();
+    mockedWriter.write("item4");
+    mockedWriter.newLine();
+    mockedWriter.write("item5");
+    mockedWriter.newLine();
+
+    TaskQueue queue2 =
+        taskQueueFactory.getTaskQueue(HandlerKey.of(ReportableEntityType.EVENT, "2888"), 0);
+    queue2.clear();
+    EventDataSubmissionTask eventTask =
+        new EventDataSubmissionTask(
+            null,
+            proxyId,
+            new DefaultEntityPropertiesForTesting(),
+            queue2,
+            "2888",
+            ImmutableList.of(
+                new Event(
+                    ReportEvent.newBuilder()
+                        .setStartTime(123456789L * 1000)
+                        .setEndTime(123456789L * 1000 + 1)
+                        .setName("Event name for testing")
+                        .setHosts(ImmutableList.of("host1", "host2"))
+                        .setDimensions(ImmutableMap.of("multi", ImmutableList.of("bar", "baz")))
+                        .setAnnotations(ImmutableMap.of("severity", "INFO"))
+                        .setTags(ImmutableList.of("tag1"))
+                        .build()),
+                new Event(
+                    ReportEvent.newBuilder()
+                        .setStartTime(123456789L * 1000)
+                        .setEndTime(123456789L * 1000 + 1)
+                        .setName("Event name for testing")
+                        .setHosts(ImmutableList.of("host1", "host2"))
+                        .setAnnotations(ImmutableMap.of("severity", "INFO"))
+                        .build())),
+            () -> 12345L);
+    eventTask.enqueue(QueueingReason.RETRY);
+    mockedWriter.write(
+        "@Event 123456789000 123456789001 \"Event name for testing\" "
+            + "\"host\"=\"host1\" \"host\"=\"host2\" \"severity\"=\"INFO\" \"multi\"=\"bar\" "
+            + "\"multi\"=\"baz\" \"tag\"=\"tag1\"");
+    mockedWriter.newLine();
+    mockedWriter.write(
+        "@Event 123456789000 123456789001 \"Event name for testing\" "
+            + "\"host\"=\"host1\" \"host\"=\"host2\" \"severity\"=\"INFO\"");
+    mockedWriter.newLine();
+
+    TaskQueue queue3 =
+        taskQueueFactory.getTaskQueue(HandlerKey.of(ReportableEntityType.SOURCE_TAG, "2898"), 0);
+    queue3.clear();
+    SourceTagSubmissionTask sourceTagTask =
+        new SourceTagSubmissionTask(
+            null,
+            new DefaultEntityPropertiesForTesting(),
+            queue3,
+            "2898",
+            new SourceTag(
+                ReportSourceTag.newBuilder()
+                    .setOperation(SourceOperationType.SOURCE_TAG)
+                    .setAction(SourceTagAction.SAVE)
+                    .setSource("testSource")
+                    .setAnnotations(ImmutableList.of("newtag1", "newtag2"))
+                    .build()),
+            () -> 12345L);
+    sourceTagTask.enqueue(QueueingReason.RETRY);
+    mockedWriter.write("@SourceTag action=save source=\"testSource\" \"newtag1\" \"newtag2\"");
+    mockedWriter.newLine();
+
+    expectLastCall().once();
+    replay(mockedWriter);
+
+    assertEquals(2, queue.size());
+    qe.processQueue(queue, mockedWriter);
+    assertEquals(0, queue.size());
+
+    assertEquals(1, queue2.size());
+    qe.processQueue(queue2, mockedWriter);
+    assertEquals(0, queue2.size());
+
+    assertEquals(1, queue3.size());
+    qe.processQueue(queue3, mockedWriter);
+    assertEquals(0, queue3.size());
+
+    verify(mockedWriter);
+
+    List files =
+        ConcurrentShardedQueueFile.listFiles(bufferFile, ".spool").stream()
+            .map(x -> x.replace(bufferFile + ".", ""))
+            .collect(Collectors.toList());
+    assertEquals(3, files.size());
+    assertTrue(files.contains("points.2878.0.spool_0000"));
+    assertTrue(files.contains("events.2888.0.spool_0000"));
+    assertTrue(files.contains("sourceTags.2898.0.spool_0000"));
+
+    HandlerKey k1 = HandlerKey.of(ReportableEntityType.POINT, "2878");
+    HandlerKey k2 = HandlerKey.of(ReportableEntityType.EVENT, "2888");
+    HandlerKey k3 = HandlerKey.of(ReportableEntityType.SOURCE_TAG, "2898");
+    files = ConcurrentShardedQueueFile.listFiles(bufferFile, ".spool");
+    Set hk = QueueExporter.getValidHandlerKeys(files, "all");
+    assertEquals(3, hk.size());
+    assertTrue(hk.contains(k1));
+    assertTrue(hk.contains(k2));
+    assertTrue(hk.contains(k3));
+
+    hk = QueueExporter.getValidHandlerKeys(files, "2878, 2898");
+    assertEquals(2, hk.size());
+    assertTrue(hk.contains(k1));
+    assertTrue(hk.contains(k3));
+
+    hk = QueueExporter.getValidHandlerKeys(files, "2888");
+    assertEquals(1, hk.size());
+    assertTrue(hk.contains(k2));
+  }
+
+  @Test
+  public void testQueueExporterWithRetainData() throws Exception {
+    File file = new File(File.createTempFile("proxyTestConverter", null).getPath() + ".queue");
+    file.deleteOnExit();
+    String bufferFile = file.getAbsolutePath();
+    TaskQueueFactory taskQueueFactory = new TaskQueueFactoryImpl(bufferFile, false, false, 128);
+    EntityPropertiesFactory entityPropFactory = new DefaultEntityPropertiesFactoryForTesting();
+    QueueExporter qe =
+        new QueueExporter(
+            bufferFile, "2878", bufferFile + "-output", true, taskQueueFactory, entityPropFactory);
+    BufferedWriter mockedWriter = EasyMock.createMock(BufferedWriter.class);
+    reset(mockedWriter);
+    HandlerKey key = HandlerKey.of(ReportableEntityType.POINT, "2878");
+    TaskQueue queue = taskQueueFactory.getTaskQueue(key, 0);
+    queue.clear();
+    UUID proxyId = UUID.randomUUID();
+    LineDelimitedDataSubmissionTask task =
+        new LineDelimitedDataSubmissionTask(
+            null,
+            proxyId,
+            new DefaultEntityPropertiesForTesting(),
+            queue,
+            "wavefront",
+            ReportableEntityType.POINT,
+            "2878",
+            ImmutableList.of("item1", "item2", "item3"),
+            () -> 12345L);
+    task.enqueue(QueueingReason.RETRY);
+    LineDelimitedDataSubmissionTask task2 =
+        new LineDelimitedDataSubmissionTask(
+            null,
+            proxyId,
+            new DefaultEntityPropertiesForTesting(),
+            queue,
+            "wavefront",
+            ReportableEntityType.POINT,
+            "2878",
+            ImmutableList.of("item4", "item5"),
+            () -> 12345L);
+    task2.enqueue(QueueingReason.RETRY);
+
+    qe.export();
+    File outputTextFile = new File(file.getAbsolutePath() + "-output.points.2878.0.txt");
+    assertEquals(
+        ImmutableList.of("item1", "item2", "item3", "item4", "item5"),
+        Files.asCharSource(outputTextFile, Charsets.UTF_8).readLines());
+    assertEquals(2, taskQueueFactory.getTaskQueue(key, 0).size());
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/queueing/RetryTaskConverterTest.java b/proxy/src/test/java/com/wavefront/agent/queueing/RetryTaskConverterTest.java
new file mode 100644
index 000000000..a58acc795
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/queueing/RetryTaskConverterTest.java
@@ -0,0 +1,36 @@
+package com.wavefront.agent.queueing;
+
+import static org.junit.Assert.assertNull;
+
+import com.google.common.collect.ImmutableList;
+import com.wavefront.agent.data.DefaultEntityPropertiesForTesting;
+import com.wavefront.agent.data.LineDelimitedDataSubmissionTask;
+import com.wavefront.data.ReportableEntityType;
+import java.util.UUID;
+import org.junit.Test;
+
+public class RetryTaskConverterTest {
+
+  @Test
+  public void testTaskSerialize() {
+    UUID proxyId = UUID.randomUUID();
+    LineDelimitedDataSubmissionTask task =
+        new LineDelimitedDataSubmissionTask(
+            null,
+            proxyId,
+            new DefaultEntityPropertiesForTesting(),
+            null,
+            "wavefront",
+            ReportableEntityType.POINT,
+            "2878",
+            ImmutableList.of("item1", "item2", "item3"),
+            () -> 12345L);
+    RetryTaskConverter converter =
+        new RetryTaskConverter<>("2878", RetryTaskConverter.CompressionType.NONE);
+
+    assertNull(converter.fromBytes(new byte[] {0, 0, 0}));
+    assertNull(converter.fromBytes(new byte[] {'W', 'F', 0}));
+    assertNull(converter.fromBytes(new byte[] {'W', 'F', 1}));
+    assertNull(converter.fromBytes(new byte[] {'W', 'F', 1, 0}));
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/queueing/SQSQueueFactoryImplTest.java b/proxy/src/test/java/com/wavefront/agent/queueing/SQSQueueFactoryImplTest.java
new file mode 100644
index 000000000..24d3c5975
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/queueing/SQSQueueFactoryImplTest.java
@@ -0,0 +1,39 @@
+package com.wavefront.agent.queueing;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertTrue;
+
+import com.wavefront.agent.ProxyConfig;
+import com.wavefront.agent.handlers.HandlerKey;
+import com.wavefront.data.ReportableEntityType;
+import org.junit.Test;
+
+/** @author mike@wavefront.com */
+public class SQSQueueFactoryImplTest {
+  @Test
+  public void testQueueTemplate() {
+    // we have to have all three
+    assertTrue(SQSQueueFactoryImpl.isValidSQSTemplate("{{id}}{{entity}}{{port}}"));
+    assertTrue(SQSQueueFactoryImpl.isValidSQSTemplate(new ProxyConfig().getSqsQueueNameTemplate()));
+
+    // Typo or missing one (or all) of the three keys in some fashion
+    assertFalse(SQSQueueFactoryImpl.isValidSQSTemplate("{{id}{{entity}}{{port}}"));
+    assertFalse(SQSQueueFactoryImpl.isValidSQSTemplate("{{id}}{entity}}{{port}}"));
+    assertFalse(SQSQueueFactoryImpl.isValidSQSTemplate("{{id}}{{entity}}{port}}"));
+    assertFalse(SQSQueueFactoryImpl.isValidSQSTemplate(""));
+    assertFalse(SQSQueueFactoryImpl.isValidSQSTemplate("{{id}}"));
+    assertFalse(SQSQueueFactoryImpl.isValidSQSTemplate("{{entity}}"));
+    assertFalse(SQSQueueFactoryImpl.isValidSQSTemplate("{{port}}"));
+  }
+
+  @Test
+  public void testQueueNameGeneration() {
+    SQSQueueFactoryImpl queueFactory =
+        new SQSQueueFactoryImpl(
+            new ProxyConfig().getSqsQueueNameTemplate(), "us-west-2", "myid", false);
+    assertEquals(
+        "wf-proxy-myid-points-2878",
+        queueFactory.getQueueName(HandlerKey.of(ReportableEntityType.POINT, "2878")));
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/queueing/SQSSubmissionQueueTest.java b/proxy/src/test/java/com/wavefront/agent/queueing/SQSSubmissionQueueTest.java
new file mode 100644
index 000000000..d722d3752
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/queueing/SQSSubmissionQueueTest.java
@@ -0,0 +1,154 @@
+package com.wavefront.agent.queueing;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.junit.Assert.assertEquals;
+
+import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.model.Message;
+import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
+import com.amazonaws.services.sqs.model.ReceiveMessageResult;
+import com.amazonaws.services.sqs.model.SendMessageRequest;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.wavefront.agent.data.DataSubmissionTask;
+import com.wavefront.agent.data.DefaultEntityPropertiesForTesting;
+import com.wavefront.agent.data.EventDataSubmissionTask;
+import com.wavefront.agent.data.LineDelimitedDataSubmissionTask;
+import com.wavefront.agent.data.QueueingReason;
+import com.wavefront.data.ReportableEntityType;
+import com.wavefront.dto.Event;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import wavefront.report.ReportEvent;
+
+/** @author mike@wavefront.com */
+@RunWith(Parameterized.class)
+public class SQSSubmissionQueueTest> {
+  private final String queueUrl = "https://amazonsqs.some.queue";
+  private AmazonSQS client = createMock(AmazonSQS.class);
+  private final T expectedTask;
+  private final RetryTaskConverter converter;
+  private final AtomicLong time = new AtomicLong(77777);
+
+  public SQSSubmissionQueueTest(
+      TaskConverter.CompressionType compressionType, RetryTaskConverter converter, T task) {
+    this.converter = converter;
+    this.expectedTask = task;
+    System.out.println(task.getClass().getSimpleName() + " compression type: " + compressionType);
+  }
+
+  @Parameterized.Parameters
+  public static Collection scenarios() {
+    Collection scenarios = new ArrayList<>();
+    for (TaskConverter.CompressionType type : TaskConverter.CompressionType.values()) {
+      RetryTaskConverter converter =
+          new RetryTaskConverter<>("2878", type);
+      LineDelimitedDataSubmissionTask task =
+          converter.fromBytes(
+              "WF\u0001\u0001{\"__CLASS\":\"com.wavefront.agent.data.LineDelimitedDataSubmissionTask\",\"enqueuedTimeMillis\":77777,\"attempts\":0,\"serverErrors\":0,\"handle\":\"2878\",\"entityType\":\"POINT\",\"format\":\"wavefront\",\"payload\":[\"java.util.ArrayList\",[\"item1\",\"item2\",\"item3\"]],\"enqueuedMillis\":77777}"
+                  .getBytes());
+      scenarios.add(new Object[] {type, converter, task});
+    }
+    for (TaskConverter.CompressionType type : TaskConverter.CompressionType.values()) {
+      RetryTaskConverter converter =
+          new RetryTaskConverter<>("2878", type);
+      EventDataSubmissionTask task =
+          converter.fromBytes(
+              "WF\u0001\u0001{\"__CLASS\":\"com.wavefront.agent.data.EventDataSubmissionTask\",\"enqueuedTimeMillis\":77777,\"attempts\":0,\"serverErrors\":0,\"handle\":\"2878\",\"entityType\":\"EVENT\",\"events\":[\"java.util.ArrayList\",[{\"name\":\"Event name for testing\",\"startTime\":77777000,\"endTime\":77777001,\"annotations\":[\"java.util.HashMap\",{\"severity\":\"INFO\"}],\"dimensions\":[\"java.util.HashMap\",{\"multi\":[\"java.util.ArrayList\",[\"bar\",\"baz\"]]}],\"hosts\":[\"java.util.ArrayList\",[\"host1\",\"host2\"]],\"tags\":[\"java.util.ArrayList\",[\"tag1\"]]}]],\"enqueuedMillis\":77777}"
+                  .getBytes());
+      scenarios.add(new Object[] {type, converter, task});
+    }
+    return scenarios;
+  }
+
+  @Test
+  public void testTaskRead() throws IOException {
+    SQSSubmissionQueue queue = getTaskQueue();
+    UUID proxyId = UUID.randomUUID();
+    DataSubmissionTask> task;
+    if (this.expectedTask instanceof LineDelimitedDataSubmissionTask) {
+      task =
+          new LineDelimitedDataSubmissionTask(
+              null,
+              proxyId,
+              new DefaultEntityPropertiesForTesting(),
+              queue,
+              "wavefront",
+              ReportableEntityType.POINT,
+              "2878",
+              ImmutableList.of("item1", "item2", "item3"),
+              time::get);
+    } else if (this.expectedTask instanceof EventDataSubmissionTask) {
+      task =
+          new EventDataSubmissionTask(
+              null,
+              proxyId,
+              new DefaultEntityPropertiesForTesting(),
+              queue,
+              "2878",
+              ImmutableList.of(
+                  new Event(
+                      ReportEvent.newBuilder()
+                          .setStartTime(time.get() * 1000)
+                          .setEndTime(time.get() * 1000 + 1)
+                          .setName("Event name for testing")
+                          .setHosts(ImmutableList.of("host1", "host2"))
+                          .setDimensions(ImmutableMap.of("multi", ImmutableList.of("bar", "baz")))
+                          .setAnnotations(ImmutableMap.of("severity", "INFO"))
+                          .setTags(ImmutableList.of("tag1"))
+                          .build())),
+              time::get);
+    } else {
+      task = null;
+    }
+    expect(
+            client.sendMessage(
+                new SendMessageRequest(
+                    queueUrl, queue.encodeMessageForDelivery(this.expectedTask))))
+        .andReturn(null);
+    replay(client);
+    task.enqueue(QueueingReason.RETRY);
+
+    reset(client);
+    ReceiveMessageRequest msgRequest =
+        new ReceiveMessageRequest()
+            .withMaxNumberOfMessages(1)
+            .withWaitTimeSeconds(1)
+            .withQueueUrl(queueUrl);
+    ReceiveMessageResult msgResult =
+        new ReceiveMessageResult()
+            .withMessages(
+                new Message()
+                    .withBody(queue.encodeMessageForDelivery(task))
+                    .withReceiptHandle("handle1"));
+
+    expect(client.receiveMessage(msgRequest)).andReturn(msgResult);
+    replay(client);
+    if (this.expectedTask instanceof LineDelimitedDataSubmissionTask) {
+      LineDelimitedDataSubmissionTask readTask = (LineDelimitedDataSubmissionTask) queue.peek();
+      assertEquals(((LineDelimitedDataSubmissionTask) task).payload(), readTask.payload());
+      assertEquals(
+          ((LineDelimitedDataSubmissionTask) this.expectedTask).payload(), readTask.payload());
+      assertEquals(77777, readTask.getEnqueuedMillis());
+    }
+    if (this.expectedTask instanceof EventDataSubmissionTask) {
+      EventDataSubmissionTask readTask = (EventDataSubmissionTask) queue.peek();
+      assertEquals(((EventDataSubmissionTask) task).payload(), readTask.payload());
+      assertEquals(((EventDataSubmissionTask) this.expectedTask).payload(), readTask.payload());
+      assertEquals(77777, readTask.getEnqueuedMillis());
+    }
+  }
+
+  private SQSSubmissionQueue getTaskQueue() {
+    return new SQSSubmissionQueue<>(queueUrl, client, converter);
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/sampler/SpanSamplerTest.java b/proxy/src/test/java/com/wavefront/agent/sampler/SpanSamplerTest.java
new file mode 100644
index 000000000..6caa61a14
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/sampler/SpanSamplerTest.java
@@ -0,0 +1,176 @@
+package com.wavefront.agent.sampler;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableList;
+import com.wavefront.api.agent.SpanSamplingPolicy;
+import com.wavefront.data.AnnotationUtils;
+import com.wavefront.sdk.entities.tracing.sampling.DurationSampler;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.MetricName;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import org.junit.Test;
+import wavefront.report.Annotation;
+import wavefront.report.Span;
+
+/** @author Han Zhang (zhanghan@vmware.com) */
+public class SpanSamplerTest {
+  @Test
+  public void testSample() {
+    long startTime = System.currentTimeMillis();
+    String traceId = UUID.randomUUID().toString();
+    Span spanToAllow =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(6)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .build();
+    Span spanToDiscard =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .build();
+    SpanSampler sampler = new SpanSampler(new DurationSampler(5), () -> null);
+    assertTrue(sampler.sample(spanToAllow));
+    assertFalse(sampler.sample(spanToDiscard));
+
+    Counter discarded =
+        Metrics.newCounter(new MetricName("SpanSamplerTest", "testSample", "discarded"));
+    assertTrue(sampler.sample(spanToAllow, discarded));
+    assertEquals(0, discarded.count());
+    assertFalse(sampler.sample(spanToDiscard, discarded));
+    assertEquals(1, discarded.count());
+  }
+
+  @Test
+  public void testAlwaysSampleDebug() {
+    long startTime = System.currentTimeMillis();
+    String traceId = UUID.randomUUID().toString();
+    Span span =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .setAnnotations(ImmutableList.of(new Annotation("debug", "true")))
+            .build();
+    SpanSampler sampler = new SpanSampler(new DurationSampler(5), () -> null);
+    assertTrue(sampler.sample(span));
+  }
+
+  @Test
+  public void testMultipleSpanSamplingPolicies() {
+    long startTime = System.currentTimeMillis();
+    String traceId = UUID.randomUUID().toString();
+    Span spanToAllow =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .build();
+    Span spanToAllowWithDebugTag =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .setAnnotations(ImmutableList.of(new Annotation("debug", "true")))
+            .build();
+    Span spanToDiscard =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4)
+            .setName("testSpanName")
+            .setSource("source")
+            .setSpanId("testspanid")
+            .setTraceId(traceId)
+            .build();
+    List activeSpanSamplingPolicies =
+        ImmutableList.of(
+            new SpanSamplingPolicy("SpanNamePolicy", "{{spanName}}='testSpanName'", 0),
+            new SpanSamplingPolicy("SpanSourcePolicy", "{{sourceName}}='testsource'", 100));
+
+    SpanSampler sampler = new SpanSampler(new DurationSampler(5), () -> activeSpanSamplingPolicies);
+    assertTrue(sampler.sample(spanToAllow));
+    assertEquals(
+        "SpanSourcePolicy",
+        AnnotationUtils.getValue(spanToAllow.getAnnotations(), "_sampledByPolicy"));
+    assertTrue(sampler.sample(spanToAllowWithDebugTag));
+    assertNull(
+        AnnotationUtils.getValue(spanToAllowWithDebugTag.getAnnotations(), "_sampledByPolicy"));
+    assertFalse(sampler.sample(spanToDiscard));
+    assertTrue(spanToDiscard.getAnnotations().isEmpty());
+  }
+
+  @Test
+  public void testSpanSamplingPolicySamplingPercent() {
+    long startTime = System.currentTimeMillis();
+    Span span =
+        Span.newBuilder()
+            .setCustomer("dummy")
+            .setStartMillis(startTime)
+            .setDuration(4)
+            .setName("testSpanName")
+            .setSource("testsource")
+            .setSpanId("testspanid")
+            .setTraceId(UUID.randomUUID().toString())
+            .build();
+    List activeSpanSamplingPolicies = new ArrayList<>();
+    activeSpanSamplingPolicies.add(
+        new SpanSamplingPolicy("SpanNamePolicy", "{{spanName}}='testSpanName'", 50));
+    SpanSampler sampler = new SpanSampler(new DurationSampler(5), () -> activeSpanSamplingPolicies);
+    int sampledSpans = 0;
+    for (int i = 0; i < 1000; i++) {
+      if (sampler.sample(Span.newBuilder(span).setTraceId(UUID.randomUUID().toString()).build())) {
+        sampledSpans++;
+      }
+    }
+    assertTrue(sampledSpans < 1000 && sampledSpans > 0);
+    activeSpanSamplingPolicies.clear();
+    activeSpanSamplingPolicies.add(
+        new SpanSamplingPolicy("SpanNamePolicy", "{{spanName" + "}}='testSpanName'", 100));
+    sampledSpans = 0;
+    for (int i = 0; i < 1000; i++) {
+      if (sampler.sample(Span.newBuilder(span).setTraceId(UUID.randomUUID().toString()).build())) {
+        sampledSpans++;
+      }
+    }
+    assertEquals(1000, sampledSpans);
+    activeSpanSamplingPolicies.clear();
+    activeSpanSamplingPolicies.add(
+        new SpanSamplingPolicy("SpanNamePolicy", "{{spanName" + "}}='testSpanName'", 0));
+    sampledSpans = 0;
+    for (int i = 0; i < 1000; i++) {
+      if (sampler.sample(Span.newBuilder(span).setTraceId(UUID.randomUUID().toString()).build())) {
+        sampledSpans++;
+      }
+    }
+    assertEquals(0, sampledSpans);
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/agent/tls/NaiveTrustManager.java b/proxy/src/test/java/com/wavefront/agent/tls/NaiveTrustManager.java
new file mode 100644
index 000000000..282e265ec
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/agent/tls/NaiveTrustManager.java
@@ -0,0 +1,14 @@
+package com.wavefront.agent.tls;
+
+import java.security.cert.X509Certificate;
+import javax.net.ssl.X509TrustManager;
+
+public class NaiveTrustManager implements X509TrustManager {
+  public void checkClientTrusted(X509Certificate[] cert, String authType) {}
+
+  public void checkServerTrusted(X509Certificate[] cert, String authType) {}
+
+  public X509Certificate[] getAcceptedIssuers() {
+    return null;
+  }
+}
diff --git a/proxy/src/test/java/com/wavefront/common/HistogramUtilsTest.java b/proxy/src/test/java/com/wavefront/common/HistogramUtilsTest.java
new file mode 100644
index 000000000..acdb25438
--- /dev/null
+++ b/proxy/src/test/java/com/wavefront/common/HistogramUtilsTest.java
@@ -0,0 +1,39 @@
+package com.wavefront.common;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import javax.ws.rs.core.Response;
+import org.junit.Test;
+
+/** @author vasily@wavefront.com */
+public class HistogramUtilsTest {
+
+  @Test
+  public void testIsWavefrontResponse() {
+    // normal response
+    assertTrue(
+        Utils.isWavefrontResponse(
+            Response.status(408).entity("{\"code\":408,\"message\":\"some_message\"}").build()));
+    // order does not matter, should be true
+    assertTrue(
+        Utils.isWavefrontResponse(
+            Response.status(407).entity("{\"message\":\"some_message\",\"code\":407}").build()));
+    // extra fields ok
+    assertTrue(
+        Utils.isWavefrontResponse(
+            Response.status(408)
+                .entity("{\"code\":408,\"message\":\"some_message\",\"extra\":0}")
+                .build()));
+    // non well formed JSON: closing curly brace missing, should be false
+    assertFalse(
+        Utils.isWavefrontResponse(
+            Response.status(408).entity("{\"code\":408,\"message\":\"some_message\"").build()));
+    // code is not the same as status code, should be false
+    assertFalse(
+        Utils.isWavefrontResponse(
+            Response.status(407).entity("{\"code\":408,\"message\":\"some_message\"}").build()));
+    // message missing
+    assertFalse(Utils.isWavefrontResponse(Response.status(407).entity("{\"code\":408}").build()));
+  }
+}
diff --git a/proxy/src/test/java/org/logstash/beats/BatchIdentityTest.java b/proxy/src/test/java/org/logstash/beats/BatchIdentityTest.java
new file mode 100644
index 000000000..5f8a8cddb
--- /dev/null
+++ b/proxy/src/test/java/org/logstash/beats/BatchIdentityTest.java
@@ -0,0 +1,165 @@
+package org.logstash.beats;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Collections;
+import java.util.Iterator;
+import javax.annotation.Nonnull;
+import org.junit.Test;
+
+/** @author vasily@wavefront.com. */
+public class BatchIdentityTest {
+
+  @Test
+  public void testEquals() {
+    assertEquals(
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 6, 5, null, null),
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 6, 5, null, null));
+    assertEquals(
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 1, 5, "test.log", 123),
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 1, 5, "test.log", 123));
+    assertNotEquals(
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 1, 5, "test.log", 123),
+        new BatchIdentity("2019-09-16T01:02:03.123Z", 1, 5, "test.log", 123));
+    assertNotEquals(
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 1, 5, "test.log", 123),
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 2, 5, "test.log", 123));
+    assertNotEquals(
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 1, 5, "test.log", 123),
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 1, 4, "test.log", 123));
+    assertNotEquals(
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 1, 5, null, 123),
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 1, 5, "test.log", 123));
+    assertNotEquals(
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 1, 5, null, null),
+        new BatchIdentity("2019-09-17T01:02:03.123Z", 1, 5, null, 123));
+  }
+
+  @Test
+  public void testCreateFromMessage() {
+    Message message =
+        new Message(
+            101,
+            ImmutableMap.builder()
+                .put(
+                    "@metadata",
+                    ImmutableMap.of("beat", "filebeat", "type", "_doc", "version", "7.3.1"))
+                .put("@timestamp", "2019-09-17T01:02:03.123Z")
+                .put("input", ImmutableMap.of("type", "log"))
+                .put("message", "This is a log line #1")
+                .put(
+                    "host",
+                    ImmutableMap.of(
+                        "name",
+                        "host1.acme.corp",
+                        "hostname",
+                        "host1.acme.corp",
+                        "id",
+                        "6DF46E56-37A3-54F8-9541-74EC4DE13483"))
+                .put(
+                    "log",
+                    ImmutableMap.of("offset", 6599, "file", ImmutableMap.of("path", "test.log")))
+                .put(
+                    "agent",
+                    ImmutableMap.of(
+                        "id",
+                        "30ff3498-ae71-41e3-bbcb-4a39352da0fe",
+                        "version",
+                        "7.3.1",
+                        "type",
+                        "filebeat",
+                        "hostname",
+                        "host1.acme.corp",
+                        "ephemeral_id",
+                        "fcb2e75f-859f-4706-9f14-a16dc1965ff1"))
+                .build());
+    Message message2 =
+        new Message(
+            102,
+            ImmutableMap.builder()
+                .put(
+                    "@metadata",
+                    ImmutableMap.of("beat", "filebeat", "type", "_doc", "version", "7.3.1"))
+                .put("@timestamp", "2019-09-17T01:02:04.123Z")
+                .put("input", ImmutableMap.of("type", "log"))
+                .put("message", "This is a log line #2")
+                .put(
+                    "host",
+                    ImmutableMap.of(
+                        "name",
+                        "host1.acme.corp",
+                        "hostname",
+                        "host1.acme.corp",
+                        "id",
+                        "6DF46E56-37A3-54F8-9541-74EC4DE13483"))
+                .put(
+                    "log",
+                    ImmutableMap.of("offset", 6799, "file", ImmutableMap.of("path", "test.log")))
+                .put(
+                    "agent",
+                    ImmutableMap.of(
+                        "id",
+                        "30ff3498-ae71-41e3-bbcb-4a39352da0fe",
+                        "version",
+                        "7.3.1",
+                        "type",
+                        "filebeat",
+                        "hostname",
+                        "host1.acme.corp",
+                        "ephemeral_id",
+                        "fcb2e75f-859f-4706-9f14-a16dc1965ff1"))
+                .build());
+    Batch batch =
+        new Batch() {
+          @Override
+          public byte getProtocol() {
+            return 0;
+          }
+
+          @Override
+          public int getBatchSize() {
+            return 2;
+          }
+
+          @Override
+          public void setBatchSize(int batchSize) {}
+
+          @Override
+          public int getHighestSequence() {
+            return 102;
+          }
+
+          @Override
+          public int size() {
+            return 2;
+          }
+
+          @Override
+          public boolean isEmpty() {
+            return false;
+          }
+
+          @Override
+          public boolean isComplete() {
+            return true;
+          }
+
+          @Override
+          public void release() {}
+
+          @Nonnull
+          @Override
+          public Iterator iterator() {
+            return Collections.emptyIterator();
+          }
+        };
+    message.setBatch(batch);
+    message2.setBatch(batch);
+    String key = BatchIdentity.keyFrom(message);
+    BatchIdentity identity = BatchIdentity.valueFrom(message);
+    assertEquals("30ff3498-ae71-41e3-bbcb-4a39352da0fe", key);
+    assertEquals(new BatchIdentity("2019-09-17T01:02:03.123Z", 102, 2, "test.log", 6599), identity);
+  }
+}
diff --git a/proxy/src/test/resources/com.wavefront.agent/ddTestSystemMetadataOnly.json b/proxy/src/test/resources/com.wavefront.agent/ddTestSystemMetadataOnly.json
new file mode 100644
index 000000000..68b863eb3
--- /dev/null
+++ b/proxy/src/test/resources/com.wavefront.agent/ddTestSystemMetadataOnly.json
@@ -0,0 +1,9 @@
+{
+  "internalHostname": "testHost",
+  "host-tags": {
+    "system": [
+      "app:closedstack",
+      "role:control"
+    ]
+  }
+}
\ No newline at end of file
diff --git a/proxy/src/test/resources/com.wavefront.agent/ddTestTimeseries.json b/proxy/src/test/resources/com.wavefront.agent/ddTestTimeseries.json
index fd268dcda..a0b04a8cf 100644
--- a/proxy/src/test/resources/com.wavefront.agent/ddTestTimeseries.json
+++ b/proxy/src/test/resources/com.wavefront.agent/ddTestTimeseries.json
@@ -22,7 +22,8 @@
           0.0
         ]
       ],
-      "host": "testHost"
+      "host": "testHost",
+      "tags": ["env:prod,app:openstack", "source:Launcher"]
     },
     {
       "type": "gauge",
@@ -36,6 +37,19 @@
       "source_type_name": "System",
       "metric": "system.net.packets_in.count",
       "device": "eth0"
+    },
+    {
+      "metric":"test.metric",
+      "points":[
+        [
+          1531176936,
+          20
+        ]
+      ],
+      "type":"rate",
+      "interval": 20,
+      "host":"testhost",
+      "tags":null
     }
   ]
 }
\ No newline at end of file
diff --git a/proxy/src/test/resources/com/wavefront/agent/preprocessor/log_preprocessor_rules.yaml b/proxy/src/test/resources/com/wavefront/agent/preprocessor/log_preprocessor_rules.yaml
new file mode 100644
index 000000000..3d23d220f
--- /dev/null
+++ b/proxy/src/test/resources/com/wavefront/agent/preprocessor/log_preprocessor_rules.yaml
@@ -0,0 +1,139 @@
+## set of log preprocessor rules for unit tests
+
+'106':
+  - rule          : test-logreplaceregex
+    action        : logReplaceRegex
+    scope         : fooBarBaz
+    search        : aaa
+    replace       : bbb
+
+  - rule          : test-logforcelowercase
+    action        : logForceLowercase
+    scope         : fooBarBaz
+
+  - rule          : test-logaddannotation
+    action        : logAddAnnotation
+    key           : customtag1
+    value         : val1
+
+  - rule          : test-logaddtag
+    action        : logAddTag
+    key           : customtag1
+    value         : val1
+
+  - rule          : test-logaddtagifnotexists
+    action        : logAddTagIfNotExists
+    key           : testExtractTag
+    value         : "Oi! This should never happen!"
+
+  - rule          : test-logaddAnnotationifnotexists
+    action        : logAddAnnotationIfNotExists
+    key           : testAddTag
+    value         : "extra annotation added"
+
+  - rule          : test-logDropTag
+    action        : logDropTag
+    key           : datacenter
+    match         : "az[4-6]"  # remove az4, az5, az6 (leave az1, az2, az3...)
+
+  - rule          : test-logDropAnnotation
+    action        : logDropAnnotation
+    key           : datacenter
+    match         : "az[4-6]"  # remove az4, az5, az6 (leave az1, az2, az3...)
+
+  - rule          : test-logExtractAnnotation
+    action        : logExtractAnnotation
+    key           : fromSource
+    input         : spanName
+    match         : "^.*testExtractTag.*"
+    search        : "^([^\\.]*\\.[^\\.]*\\.)([^\\.]*)\\.(.*)$"
+    replace       : "$2"
+    replaceInput  : "$1$3"
+
+  - rule          : test-logExtractTag
+    action        : logExtractTag
+    key           : fromSource
+    input         : spanName
+    match         : "^.*testExtractTag.*"
+    search        : "^([^\\.]*\\.[^\\.]*\\.)([^\\.]*)\\.(.*)$"
+    replace       : "$2"
+    replaceInput  : "$1$3"
+
+  - rule          : test-logextracttagifnotexists
+    action        : logExtractAnnotationIfNotExists
+    key           : fromSource  # should not work because such tag already exists!
+    input         : testExtractTag
+    search        : "^.*$"
+    replace       : "Oi! This should never happen!"
+
+  - rule          : test-logextracttagifnotexists
+    action        : logExtractTagIfNotExists
+    key           : fromSource  # should not work because such tag already exists!
+    input         : testExtractTag
+    search        : "^.*$"
+    replace       : "Oi! This should never happen!"
+
+  - rule          : test-logrenametag
+    action        : logRenameTag
+    key           : myDevice
+    newkey        : device
+    match         : "^\\d*$"
+
+  - rule          : test-logrenameannotation
+    action        : logRenameAnnotation
+    key           : myDevice
+    newkey        : device
+    match         : "^\\d*$"
+
+  - rule          : test-loglimitlength
+    action        : logLimitLength
+    maxLength     : 1000
+    scope         : message
+    actionSubtype : truncate
+    match         : "^.*"
+
+  - rule          : test-logcount
+    action        : logCount
+    if            : "1=1"
+
+  - rule          : test-logBlacklistRegex
+    action        : logBlacklistRegex
+    match         : "^.*"
+    scope         : message
+    if            : "1=1"
+
+  - rule          : test-logBlock
+    action        : logBlock
+    match         : "^.*"
+    scope         : message
+    if            : "1=1"
+
+  - rule          : test-logWhiteListRegex
+    action        : logWhitelistRegex
+    match         : "^.*"
+    scope         : message
+    if            : "1=1"
+
+  - rule          : test-logAllow
+    action        : logAllow
+    match         : "^.*"
+    scope         : message
+    if            : "1=1"
+
+  - rule: test-logAllowAnnotations
+    action: logAllowAnnotation
+    allow:
+      - key1
+      - key2
+      - foo
+      - application
+      - shard
+
+  - rule: test-logAllowTags
+    action: logAllowTag
+    allow:
+      - key1
+      - key2
+      - foo
+      - application
+      - shard
diff --git a/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules.yaml b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules.yaml
index 746830669..daab068e7 100644
--- a/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules.yaml
+++ b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules.yaml
@@ -1,70 +1,69 @@
 ## set of preprocessor rules for unit tests
 
-# test all blacklist/whitelist combinations
+# test all block/allow combinations
 '1111':
     - rule    : test-filter-1
-      action  : whitelistRegex
+      action  : allow
       scope   : pointLine
       match   : ".*prod.*"
 
     - rule    : test-filter-2
-      action  : whitelistRegex
+      action  : allow
       scope   : sourceName
       match   : ".*prod\\.corp"
 
     - rule    : test-filter-3
-      action  : whitelistRegex
+      action  : allow
       scope   : metricName
       match   : "valid\\.metric.*"
 
     - rule    : test-filter-4
-      action  : whitelistRegex
+      action  : allow
       scope   : foo
       match   : "b.r"
 
     - rule    : test-filter-5
-      action  : blacklistRegex
+      action  : block
       scope   : pointLine
       match   : ".*stop.*"
 
     - rule    : test-filter-6
-      action  : blacklistRegex
+      action  : block
       scope   : sourceName
       match   : ".*dev\\.corp"
 
     - rule    : test-filter-7
-      action  : blacklistRegex
+      action  : block
       scope   : metricName
       match   : "invalid\\.metric.*"
 
     - rule    : test-filter-8
-      action  : blacklistRegex
+      action  : block
       scope   : baz
       match   : ".*"
 
-
 '2878':
     # only allow points that contain "foo" substring anywhere in the point line
-    - rule    : test-whitelist-line
-      action  : whitelistRegex
-      scope   : pointLine
+    - rule    : test-allow-line
+      action  : allow
+      scope   : inputText
       match   : ".*foo=.*"
 
     # block source=bar*
-    - rule    : test-blacklist-sourcename
-      action  : blacklistRegex
+    - rule    : test-block-sourcename
+      action  : block
       scope   : sourceName
       match   : "bar.*"
 
     # block all metric names that start with foo.
-    - rule    : test-blacklist-metrics
-      action  : blacklistRegex
+    - rule    : test-block-metrics
+      action  : block
       scope   : metricName
       match   : "foo\\..*"
 
     # block all points where "datacenter" point tag starts with "west"
-    - rule    : test-blacklist-tag
-      action  : blacklistRegex
+    - rule    : test-block-tag
+      action  : block
       scope   : foo
       match   : "west.*"
 
@@ -106,7 +105,7 @@
     - rule    : example-tag-all-metrics
       action  : addTag
       tag     : newtagkey
-      value   : "1"
+      value   : 1
 
     # remove "dc1" and "dc2" point tags
     - rule    : example-drop-dc12
@@ -126,6 +125,12 @@
       search  : "^(some).*"
       replace : "$1"
 
+    - rule    : test-count
+      action  : count
+
+    - rule    : test-count-2
+      action  : count
+      if      : '$value = 0'
 
 '4242':
 
@@ -217,3 +222,208 @@
       source        : testExtractTag
       search        : "^([^\\.]*)\\..*$"
       replace       : "$1"
+
+# Span Preprocessor rules:
+'30123':
+    - rule          : test-spanAddTag
+      action        : spanAddTag
+      key           : customtag1
+      value         : val1
+
+    - rule          : test-spanAddAnnotation
+      action        : spanAddAnnotation
+      key           : customtag1
+      value         : val1
+
+    - rule          : test-spanDropTag
+      action        : spanDropTag
+      key           : datacenter
+      match         : "az[4-6]"  # remove az4, az5, az6 (leave az1, az2, az3...)
+
+    - rule          : test-spanDropAnnotation
+      action        : spanDropAnnotation
+      key           : datacenter
+      match         : "az[4-6]"  # remove az4, az5, az6 (leave az1, az2, az3...)
+
+    # extract 3rd dot-delimited node from the span name into fromSource tag and remove it from the span name
+    - rule          : test-extracttag-spanAnnotation
+      action        : spanExtractAnnotation
+      key           : fromSource
+      input         : spanName
+      match         : "^.*testExtractTag.*"
+      search        : "^([^\\.]*\\.[^\\.]*\\.)([^\\.]*)\\.(.*)$"
+      replace       : "$2"
+      replaceInput  : "$1$3"
+
+    - rule          : test-extracttag-spanTag
+      action        : spanExtractTag
+      key           : fromSource
+      input         : spanName
+      match         : "^.*testExtractTag.*"
+      search        : "^([^\\.]*\\.[^\\.]*\\.)([^\\.]*)\\.(.*)$"
+      replace       : "$2"
+      replaceInput  : "$1$3"
+
+    - rule          : test-extracttagifnotexists-spanAnnotation
+      action        : spanExtractAnnotationIfNotExists
+      key           : fromSource  # should not work because such tag already exists!
+      input         : testExtractTag
+      search        : "^.*$"
+      replace       : "Oi! This should never happen!"
+
+    - rule          : test-extracttagifnotexists-spanTag
+      action        : spanExtractTagIfNotExists
+      key           : fromSource  # should not work because such tag already exists!
+      input         : testExtractTag
+      search        : "^.*$"
+      replace       : "Oi! This should never happen!"
+
+    # rename a span tag/annotation if its value matches a regex
+    - rule          : test-spanrenametag
+      action        : spanRenameTag
+      key           : myDevice
+      newkey        : device
+      match         : "^\\d*$"
+
+    - rule          : test-spanrenameannotation
+      action        : spanRenameAnnotation
+      key           : myDevice
+      newkey        : device
+      match         : "^\\d*$"
+
+    - rule          : test-spanreplaceregex
+      action        : spanReplaceRegex
+      scope         : fooBarBaz
+      search        : aaa
+      replace       : bbb
+      firstMatchOnly: true
+
+    - rule          : test-spanforcelowecase
+      action        : spanForceLowercase
+      scope         : fooBarBaz
+
+    - rule          : test-spanaddtagifnotexists
+      action        : spanAddAnnotationIfNotExists
+      key           : testExtractTag
+      value         : "Oi! This should never happen!"
+
+    - rule          : test-spanBlock
+      action        : spanBlock
+      scope         : spanName
+      match         : "^badSpan.*$"
+
+    - rule          : test-spanAllow
+      action        : spanAllow
+      scope         : spanName
+      match         : "^spanName.*$"
+
+'30124':
+  - rule: test-spanAllowAnnotations
+    action: spanAllowAnnotation
+    allow:
+      - key1
+      - key2
+      - foo
+      - application
+      - shard
+
+'30125':
+  - rule: test-spanAllowAnnotations
+    action: spanAllowAnnotation
+    allow:
+      application: "[a-zA-Z]{0,10}"
+      key2: "bar[0-9]{1,3}"
+      version: "[0-9\\.]{1,10}"
+
+  - rule          : test-spanCount
+    action        : spanCount
+    if            : '1 = 1'
+
+'30126':
+    - rule : test-v2Predicate
+      action : spanAllow
+      if:
+        all:
+          - any:
+            - all:
+              - equals:
+                  scope: key2
+                  value: "val2"
+              - startsWith:
+                  scope: metricName
+                  value: "foo"
+            - startsWith:
+                scope: metricName
+                value: "trace.deri."
+            - equals:
+                scope: key1
+                value: "val1"
+          - none:
+            - any:
+              - equals:
+                  scope: key1
+                  value: "val1"
+              - startsWith:
+                  scope: metricName
+                  value: "trace.derive."
+            - startsWith:
+                scope: metricName
+                value: "trace.der."
+            - equals:
+                scope: debug
+                value: "true"
+
+
+'30127':
+    - rule : test-v2Predicate
+      action : spanAllow
+      if:
+       equals:
+          scope: debug
+          value: "true"
+
+'2880, 2881':
+  # add a "multiPortTagKey=multiTagVal" point tag to all points
+  - rule    : example-tag-all-metrics
+    action  : addTag
+    tag     : multiPortTagKey
+    value   : multiTagVal
+
+  # the below is valid preprocessor rule, albeit not applicable to a port accepting only points.
+  # add a "multiPortSpanTagKey=multiSpanTagVal" point tag to all spans
+  - rule          : test-spanAddTag
+    action        : spanAddTag
+    key           : multiPortSpanTagKey
+    value         : multiSpanTagVal
+
+'9999':
+  - rule: testFilter
+    action: metricsFilter
+    function: allow
+    names:
+      - "metrics.1"
+      - "/metrics.2.*/"
+      - "/.*.ok$/"
+      - "metrics.ok.*"
+
+'9998':
+  - rule: testFilter
+    action: metricsFilter
+    function: drop
+    names:
+      - "metrics.1"
+      - "/metrics.2.*/"
+      - "/.*.ok$/"
+      - "metrics.ok.*"
+
+'9997':
+  - rule: testFilter
+    action: metricsFilter
+    function: allow
+    opts:
+      cacheSize: 0
+    names:
+      - "metrics.1"
+      - "/metrics.2.*/"
+      - "/.*.ok$/"
+      - "metrics.ok.*"
diff --git a/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_invalid.yaml b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_invalid.yaml
index 12a7fe6f3..2c24d35d3 100644
--- a/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_invalid.yaml
+++ b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_invalid.yaml
@@ -1,773 +1,994 @@
 ## set of preprocessor rules for unit tests - all of these rules should fail
 
 '2878':
-    # completely empty rule
-    -
-
-    # missing rule name
-    - action  : dropTag
-      tag     : dc1
-
-    # empty rule name
-    - rule    :
-      action  : dropTag
-      tag     : dc1
-
-    # rule name contains only invalid characters
-    - rule    : "$%^&*()!@/.,"
-      action  : dropTag
-      tag     : dc1
-
-    # missing action
-    - rule    : test-missing-action
-      tag     : dc1
-
-    # invalid action
-    - rule    : test-invalid-action
-      action  : nonexistentAction
-
-    # invalid argument
-    - rule    : test-invalid-argument
-      action  : dropTag
-      tag     : dc1
-      invalid : argument
-
-    # "scope" cannot be used with actions relevant to tags
-    - rule    : test-inconsistent-action-1
-      scope   : pointLine
-      action  : dropTag
-      tag     : dc1
-
-    - rule    : test-inconsistent-action-2
-      scope   : pointLine
-      action  : addTag
-      tag     : newtagkey
-      value   : "1"
-
-    - rule    : test-inconsistent-action-3
-      scope   : pointLine
-      action  : renameTag
-      tag     : foo
-      newtag  : baz
-
-    - rule    : test-inconsistent-action-4
-      scope   : pointLine
-      action  : addTagIfNotExists
-      tag     : newtagkey
-      value   : "1"
-
-    - rule    : test-inconsistent-action-1a
-      scope   : metricName
-      action  : dropTag
-      tag     : dc1
-
-    - rule    : test-inconsistent-action-2a
-      scope   : metricName
-      action  : addTag
-      tag     : newtagkey
-      value   : "1"
-
-    - rule    : test-inconsistent-action-3a
-      scope   : metricName
-      action  : renameTag
-      tag     : foo
-      newtag  : baz
-
-    - rule    : test-inconsistent-action-4a
-      scope   : metricName
-      action  : addTagIfNotExists
-      tag     : newtagkey
-      value   : "1"
-
-    - rule    : test-inconsistent-action-1b
-      scope   : sourceName
-      action  : dropTag
-      tag     : dc1
-
-    - rule    : test-inconsistent-action-2b
-      scope   : sourceName
-      action  : addTag
-      tag     : newtagkey
-      value   : "1"
-
-    - rule    : test-inconsistent-action-3b
-      scope   : sourceName
-      action  : renameTag
-      tag     : foo
-      newtag  : baz
-
-    - rule    : test-inconsistent-action-4b
-      scope   : sourceName
-      action  : addTagIfNotExists
-      tag     : newtagkey
-      value   : "1"
-
-    - rule    : test-inconsistent-action-1c
-      scope   : anytag
-      action  : dropTag
-      tag     : dc1
-
-    - rule    : test-inconsistent-action-2c
-      scope   : anytag
-      action  : addTag
-      tag     : newtagkey
-      value   : "1"
-
-    - rule    : test-inconsistent-action-3c
-      scope   : anytag
-      action  : renameTag
-      tag     : foo
-      newtag  : baz
-
-    - rule    : test-inconsistent-action-4c
-      scope   : anytag
-      action  : addTagIfNotExists
-      tag     : newtagkey
-      value   : "1"
-
-    - rule    : test-inconsistent-action-1c
-      scope   : anytag
-      action  : dropTag
-      tag     : dc1
-
-    - rule    : test-inconsistent-action-2c
-      scope   : anytag
-      action  : addTag
-      tag     : newtagkey
-      value   : "1"
-
-    - rule    : test-inconsistent-action-3c
-      scope   : anytag
-      action  : renameTag
-      tag     : foo
-      newtag  : baz
-
-    - rule    : test-inconsistent-action-4c
-      scope   : anytag
-      action  : addTagIfNotExists
-      tag     : newtagkey
-      value   : "1"
-
-
-    # test replaceRegex
-
-    # test replaceRegex: missing parameters
-
-    # missing scope
-    - rule    : test-replaceRegex-1
-      action  : replaceRegex
-      search   : "foo\\..*"
-      replace : ""
-
-    # missing search
-    - rule    : test-replaceRegex-2
-      action  : replaceRegex
-      scope   : pointLine
-      replace : ""
-
-    # missing search
-    - rule    : test-replaceRegex-3
-      action  : replaceRegex
-      scope   : metricName
-      replace : ""
-
-    # null search
-    - rule    : test-replaceRegex-4
-      action  : replaceRegex
-      scope   : pointLine
-      search  :
-      replace : ""
-
-    # empty search
-    - rule    : test-replaceRegex-5
-      action  : replaceRegex
-      scope   : pointLine
-      search   : ""
-      replace : ""
-
-    # test replaceRegex: non-applicable parameters
-
-    # tag does not apply
-    - rule    : test-replaceRegex-6
-      action  : replaceRegex
-      scope   : pointLine
-      search  : "foo"
-      replace : ""
-      tag     : tag
-
-    # newtag does not apply
-    - rule    : test-replaceRegex-8
-      action  : replaceRegex
-      scope   : pointLine
-      search  : "foo"
-      replace : ""
-      newtag  : newtag
-
-    # value does not apply
-    - rule    : test-replaceRegex-9
-      action  : replaceRegex
-      scope   : pointLine
-      search  : "foo"
-      replace : ""
-      value   : "value"
-
-
-
-    # test blacklistRegex
-
-    # test blacklistRegex: missing parameters
-
-    # missing scope
-    - rule    : test-blacklistRegex-1
-      action  : blacklistRegex
-      match   : "foo\\..*"
-
-    # missing match
-    - rule    : test-blacklistRegex-2
-      action  : blacklistRegex
-      scope   : pointLine
-
-    # missing match
-    - rule    : test-blacklistRegex-3
-      action  : blacklistRegex
-      scope   : metricName
-
-    # null match
-    - rule    : test-blacklistRegex-4
-      action  : blacklistRegex
-      scope   : pointLine
-      match   :
-
-    # empty match
-    - rule    : test-blacklistRegex-5
-      action  : blacklistRegex
-      scope   : pointLine
-      match   : ""
-
-    # test blacklistRegex: non-applicable parameters
-
-    # tag does not apply
-    - rule    : test-blacklistRegex-6
-      action  : blacklistRegex
-      scope   : pointLine
-      match   : "foo"
-      tag     : tag
-
-    # replace does not apply
-    - rule    : test-blacklistRegex-7
-      action  : blacklistRegex
-      scope   : pointLine
-      match   : "foo"
-      replace : replace
-
-    # search does not apply
-    - rule    : test-blacklistRegex-8
-      action  : blacklistRegex
-      scope   : pointLine
-      match   : "foo"
-      search  : search
-
-    # newtag does not apply
-    - rule    : test-blacklistRegex-9
-      action  : blacklistRegex
-      scope   : pointLine
-      match   : "foo"
-      newtag  : newtag
-
-    # value does not apply
-    - rule    : test-blacklistRegex-10
-      action  : blacklistRegex
-      scope   : pointLine
-      match   : "foo"
-      value   : "value"
-
-
-    # test whitelistRegex
-
-    # test whitelistRegex: missing parameters
-
-    # missing scope
-    - rule    : test-whitelistRegex-1
-      action  : whitelistRegex
-      match   : "foo\\..*"
-
-    # missing match
-    - rule    : test-whitelistRegex-2
-      action  : whitelistRegex
-      scope   : pointLine
-
-    # missing match
-    - rule    : test-whitelistRegex-3
-      action  : whitelistRegex
-      scope   : metricName
-
-    # null match
-    - rule    : test-whitelistRegex-4
-      action  : whitelistRegex
-      scope   : pointLine
-      match   :
-
-    # empty match
-    - rule    : test-whitelistRegex-5
-      action  : whitelistRegex
-      scope   : pointLine
-      match   : ""
-
-    # test whitelistRegex: non-applicable parameters
-
-    # tag does not apply
-    - rule    : test-whitelistRegex-6
-      action  : whitelistRegex
-      scope   : pointLine
-      match   : "foo"
-      tag     : tag
-
-    # replace does not apply
-    - rule    : test-whitelistRegex-7
-      action  : whitelistRegex
-      scope   : pointLine
-      match   : "foo"
-      replace : replace
-
-    # search does not apply
-    - rule    : test-whitelistRegex-8
-      action  : whitelistRegex
-      scope   : pointLine
-      match   : "foo"
-      search  : search
-
-    # newtag does not apply
-    - rule    : test-whitelistRegex-9
-      action  : whitelistRegex
-      scope   : pointLine
-      match   : "foo"
-      newtag  : newtag
-
-    # value does not apply
-    - rule    : test-whitelistRegex-10
-      action  : whitelistRegex
-      scope   : pointLine
-      match   : "foo"
-      value   : "value"
-
-
-    # test dropTag
-
-    # missing tag
-    - rule    : test-dropTag-1
-      action  : dropTag
-
-    # search does not apply
-    - rule    : test-dropTag-2
-      action  : dropTag
-      tag     : tag
-      search  : search
-
-    # replace does not apply
-    - rule    : test-dropTag-3
-      action  : dropTag
-      tag     : tag
-      replace : replace
-
-    # newtag does not apply
-    - rule    : test-dropTag-4
-      action  : dropTag
-      tag     : tag
-      newtag  : newtag
-
-    # value does not apply
-    - rule    : test-dropTag-5
-      action  : dropTag
-      tag     : tag
-      value   : value
-
-
-    # test addTag
-
-    # missing tag
-    - rule    : test-addTag-1
-      action  : addTag
-      value   : "1"
-
-    # null tag
-    - rule    : test-addTag-2
-      action  : addTag
-      tag     :
-      value   : "1"
-
-    # empty tag
-    - rule    : test-addTag-3
-      action  : addTag
-      tag     : ""
-      value   : "1"
-
-    # null value
-    - rule    : test-addTag-4
-      action  : addTag
-      tag     : tag
-      value   :
-
-    # empty value
-    - rule    : test-addTag-5
-      action  : addTag
-      tag     : tag
-      value   : ""
-
-    # missing value
-    - rule    : test-addTag-6
-      action  : addTag
-      tag     : tag
-
-    # search does not apply
-    - rule    : test-addTag-7
-      action  : addTag
-      tag     : tag
-      value   : "1"
-      search  : search
-
-    # replace does not apply
-    - rule    : test-addTag-8
-      action  : addTag
-      tag     : tag
-      value   : "1"
-      replace : replace
-
-    # newtag does not apply
-    - rule    : test-addTag-9
-      action  : addTag
-      tag     : tag
-      value   : "1"
-      newtag  : newtag
-
-    # match does not apply
-    - rule    : test-addTag-10
-      action  : addTag
-      tag     : tag
-      value   : "1"
-      match   : match
-
-
-    # test addTagIfNotExists
-
-    # missing tag
-    - rule    : test-addTagIfNotExists-1
-      action  : addTagIfNotExists
-      value   : "1"
-
-    # null tag
-    - rule    : test-addTagIfNotExists-2
-      action  : addTagIfNotExists
-      tag     :
-      value   : "1"
-
-    # empty tag
-    - rule    : test-addTagIfNotExists-3
-      action  : addTagIfNotExists
-      tag     : ""
-      value   : "1"
-
-    # null value
-    - rule    : test-addTagIfNotExists-4
-      action  : addTagIfNotExists
-      tag     : tag
-      value   :
-
-    # empty value
-    - rule    : test-addTagIfNotExists-5
-      action  : addTagIfNotExists
-      tag     : tag
-      value   : ""
-
-    # missing value
-    - rule    : test-addTagIfNotExists-6
-      action  : addTagIfNotExists
-      tag     : tag
-
-    # search does not apply
-    - rule    : test-addTagIfNotExists-7
-      action  : addTagIfNotExists
-      tag     : tag
-      value   : "1"
-      search  : search
-
-    # replace does not apply
-    - rule    : test-addTagIfNotExists-8
-      action  : addTagIfNotExists
-      tag     : tag
-      value   : "1"
-      replace : replace
-
-    # newtag does not apply
-    - rule    : test-addTagIfNotExists-9
-      action  : addTagIfNotExists
-      tag     : tag
-      value   : "1"
-      newtag  : newtag
-
-    # match does not apply
-    - rule    : test-addTagIfNotExists-10
-      action  : addTagIfNotExists
-      tag     : tag
-      value   : "1"
-      match   : match
-
-    # test renameTag
-
-    # missing tag
-    - rule    : test-renameTag-1
-      action  : renameTag
-      match   : tag
-      newtag  : newtag
-
-    # null tag
-    - rule    : test-renameTag-2
-      action  : renameTag
-      tag     :
-      match   : tag
-      newtag  : newtag
-
-    # empty tag
-    - rule    : test-renameTag-3
-      action  : renameTag
-      tag     : ""
-      newtag  : newtag
-
-    # missing newtag
-    - rule    : test-renameTag-4
-      action  : renameTag
-      tag     : tag
-
-    # null newtag
-    - rule    : test-renameTag-5
-      action  : renameTag
-      tag     : tag
-      newtag  :
-
-    # empty newtag
-    - rule    : test-renameTag-6
-      action  : renameTag
-      tag     : tag
-      newtag  : ""
-
-    # search does not apply
-    - rule    : test-renameTag-7
-      action  : renameTag
-      tag     : tag
-      match   : match
-      newtag  : newtag
-      search  : search
-
-    # replace does not apply
-    - rule    : test-renameTag-8
-      action  : renameTag
-      tag     : tag
-      match   : match
-      newtag  : newtag
-      replace : replace
-
-    # value does not apply
-    - rule    : test-renameTag-9
-      action  : renameTag
-      tag     : tag
-      match   : match
-      newtag  : newtag
-      value   : "1"
-
-
-    # test extractTag
-
-    # missing tag
-    - rule    : test-extractTag-1
-      action  : extractTag
-      match   : tag
-      source  : metricName
-      search  : tag
-      replace : replace
-
-    # null tag
-    - rule    : test-extractTag-2
-      action  : extractTag
-      tag     :
-      match   : tag
-      source  : metricName
-      search  : tag
-      replace : replace
-
-    # empty tag
-    - rule    : test-extractTag-3
-      action  : extractTag
-      tag     : ""
-      match   : match
-      source  : metricName
-      search  : tag
-      replace : replace
-
-    # missing source
-    - rule    : test-extractTag-4
-      action  : extractTag
-      tag     : tag
-      match   : match
-      search  : tag
-      replace : replace
-
-    # empty source
-    - rule    : test-extractTag-5
-      action  : extractTag
-      tag     : tag
-      source  : ""
-      match   : match
-      search  : tag
-      replace : replace
-
-    # missing search
-    - rule    : test-extractTag-6
-      action  : extractTag
-      tag     : tag
-      source  : metricName
-      match   : match
-      replace : replace
-
-    # empty search
-    - rule    : test-extractTag-7
-      action  : extractTag
-      tag     : tag
-      source  : metricName
-      match   : match
-      search  : ""
-      replace : replace
-
-    # missing replace
-    - rule    : test-extractTag-8
-      action  : extractTag
-      tag     : tag
-      source  : metricName
-      match   : match
-      search  : tag
-
-    # null replace
-    - rule    : test-extractTag-9
-      action  : extractTag
-      tag     : tag
-      source  : metricName
-      match   : match
-      search  : tag
-      replace :
-
-    # scope does not apply
-    - rule    : test-extractTag-10
-      action  : extractTag
-      scope   : tag
-      tag     : tag
-      source  : metricName
-      match   : match
-      search  : tag
-      replace : ""
-
-    # value does not apply
-    - rule    : test-extractTag-11
-      action  : extractTag
-      tag     : tag
-      source  : metricName
-      match   : match
-      search  : tag
-      replace : ""
-      value   : "1"
-
-    # test extractTagIfNotExists
-
-    # missing tag
-    - rule    : test-extractTagIfNotExists-1
-      action  : extractTagIfNotExists
-      match   : tag
-      source  : metricName
-      search  : tag
-      replace : replace
-
-    # null tag
-    - rule    : test-extractTagIfNotExists-2
-      action  : extractTagIfNotExists
-      tag     :
-      match   : tag
-      source  : metricName
-      search  : tag
-      replace : replace
-
-    # empty tag
-    - rule    : test-extractTagIfNotExists-3
-      action  : extractTagIfNotExists
-      tag     : ""
-      match   : match
-      source  : metricName
-      search  : tag
-      replace : replace
-
-    # missing source
-    - rule    : test-extractTagIfNotExists-4
-      action  : extractTagIfNotExists
-      tag     : tag
-      match   : match
-      search  : tag
-      replace : replace
-
-    # empty source
-    - rule    : test-extractTagIfNotExists-5
-      action  : extractTagIfNotExists
-      tag     : tag
-      source  : ""
-      match   : match
-      search  : tag
-      replace : replace
-
-    # missing search
-    - rule    : test-extractTagIfNotExists-6
-      action  : extractTagIfNotExists
-      tag     : tag
-      source  : metricName
-      match   : match
-      replace : replace
-
-    # empty search
-    - rule    : test-extractTagIfNotExists-7
-      action  : extractTagIfNotExists
-      tag     : tag
-      source  : metricName
-      match   : match
-      search  : ""
-      replace : replace
-
-    # missing replace
-    - rule    : test-extractTagIfNotExists-8
-      action  : extractTagIfNotExists
-      tag     : tag
-      source  : metricName
-      match   : match
-      search  : tag
-
-    # null replace
-    - rule    : test-extractTagIfNotExists-9
-      action  : extractTagIfNotExists
-      tag     : tag
-      source  : metricName
-      match   : match
-      search  : tag
-      replace :
-
-    # scope does not apply
-    - rule    : test-extractTagIfNotExists-10
-      action  : extractTagIfNotExists
-      scope   : tag
-      tag     : tag
-      source  : metricName
-      match   : match
-      search  : tag
-      replace : ""
-
-    # value does not apply
-    - rule    : test-extractTagIfNotExists-11
-      action  : extractTagIfNotExists
-      tag     : tag
-      source  : metricName
-      match   : match
-      search  : tag
-      replace : ""
-      value   : "1"
+  # completely empty rule
+  -
+
+  # missing rule name
+  - action: dropTag
+    tag: dc1
+
+  # empty rule name
+  - rule:
+    action: dropTag
+    tag: dc1
+
+  # rule name contains only invalid characters
+  - rule: "$%^&*()!@/.,"
+    action: dropTag
+    tag: dc1
+
+  # missing action
+  - rule: test-missing-action
+    tag: dc1
+
+  # invalid action
+  - rule: test-invalid-action
+    action: nonexistentAction
+
+  # invalid argument
+  - rule: test-invalid-argument
+    action: dropTag
+    tag: dc1
+    invalid: argument
+
+  # "scope" cannot be used with actions relevant to tags
+  - rule: test-inconsistent-action-1
+    scope: pointLine
+    action: dropTag
+    tag: dc1
+
+  - rule: test-inconsistent-action-2
+    scope: pointLine
+    action: addTag
+    tag: newtagkey
+    value: "1"
+
+  - rule: test-inconsistent-action-3
+    scope: pointLine
+    action: renameTag
+    tag: foo
+    newtag: baz
+
+  - rule: test-inconsistent-action-4
+    scope: pointLine
+    action: addTagIfNotExists
+    tag: newtagkey
+    value: "1"
+
+  - rule: test-inconsistent-action-1a
+    scope: metricName
+    action: dropTag
+    tag: dc1
+
+  - rule: test-inconsistent-action-2a
+    scope: metricName
+    action: addTag
+    tag: newtagkey
+    value: "1"
+
+  - rule: test-inconsistent-action-3a
+    scope: metricName
+    action: renameTag
+    tag: foo
+    newtag: baz
+
+  - rule: test-inconsistent-action-4a
+    scope: metricName
+    action: addTagIfNotExists
+    tag: newtagkey
+    value: "1"
+
+  - rule: test-inconsistent-action-1b
+    scope: sourceName
+    action: dropTag
+    tag: dc1
+
+  - rule: test-inconsistent-action-2b
+    scope: sourceName
+    action: addTag
+    tag: newtagkey
+    value: "1"
+
+  - rule: test-inconsistent-action-3b
+    scope: sourceName
+    action: renameTag
+    tag: foo
+    newtag: baz
+
+  - rule: test-inconsistent-action-4b
+    scope: sourceName
+    action: addTagIfNotExists
+    tag: newtagkey
+    value: "1"
+
+  - rule: test-inconsistent-action-1c
+    scope: anytag
+    action: dropTag
+    tag: dc1
+
+  - rule: test-inconsistent-action-2c
+    scope: anytag
+    action: addTag
+    tag: newtagkey
+    value: "1"
+
+  - rule: test-inconsistent-action-3c
+    scope: anytag
+    action: renameTag
+    tag: foo
+    newtag: baz
+
+  - rule: test-inconsistent-action-4c
+    scope: anytag
+    action: addTagIfNotExists
+    tag: newtagkey
+    value: "1"
+
+  - rule: test-inconsistent-action-1c
+    scope: anytag
+    action: dropTag
+    tag: dc1
+
+  - rule: test-inconsistent-action-2c
+    scope: anytag
+    action: addTag
+    tag: newtagkey
+    value: "1"
+
+  - rule: test-inconsistent-action-3c
+    scope: anytag
+    action: renameTag
+    tag: foo
+    newtag: baz
+
+  - rule: test-inconsistent-action-4c
+    scope: anytag
+    action: addTagIfNotExists
+    tag: newtagkey
+    value: "1"
+
+
+  # test replaceRegex
+
+  # test replaceRegex: missing parameters
+
+  # missing scope
+  - rule: test-replaceRegex-1
+    action: replaceRegex
+    search: "foo\\..*"
+    replace: ""
+
+  # missing search
+  - rule: test-replaceRegex-2
+    action: replaceRegex
+    scope: pointLine
+    replace: ""
+
+  # missing search
+  - rule: test-replaceRegex-3
+    action: replaceRegex
+    scope: metricName
+    replace: ""
+
+  # null search
+  - rule: test-replaceRegex-4
+    action: replaceRegex
+    scope: pointLine
+    search:
+    replace: ""
+
+  # empty search
+  - rule: test-replaceRegex-5
+    action: replaceRegex
+    scope: pointLine
+    search: ""
+    replace: ""
+
+  # test replaceRegex: non-applicable parameters
+
+  # tag does not apply
+  - rule: test-replaceRegex-6
+    action: replaceRegex
+    scope: pointLine
+    search: "foo"
+    replace: ""
+    tag: tag
+
+  # newtag does not apply
+  - rule: test-replaceRegex-8
+    action: replaceRegex
+    scope: pointLine
+    search: "foo"
+    replace: ""
+    newtag: newtag
+
+  # value does not apply
+  - rule: test-replaceRegex-9
+    action: replaceRegex
+    scope: pointLine
+    search: "foo"
+    replace: ""
+    value: "value"
+
+
+
+  # test block
+
+  # test block: missing parameters
+
+  # missing scope
+  - rule: test-block-1
+    action: block
+    match: "foo\\..*"
+
+  # missing match
+  - rule: test-block-2
+    action: block
+    scope: pointLine
+
+  # missing match
+  - rule: test-block-3
+    action: block
+    scope: metricName
+
+  # null match
+  - rule: test-block-4
+    action: block
+    scope: pointLine
+    match:
+
+  # empty match
+  - rule: test-block-5
+    action: block
+    scope: pointLine
+    match: ""
+
+  # test block: non-applicable parameters
+
+  # tag does not apply
+  - rule: test-block-6
+    action: block
+    scope: pointLine
+    match: "foo"
+    tag: tag
+
+  # replace does not apply
+  - rule: test-block-7
+    action: block
+    scope: pointLine
+    match: "foo"
+    replace: replace
+
+  # search does not apply
+  - rule: test-block-8
+    action: block
+    scope: pointLine
+    match: "foo"
+    search: search
+
+  # newtag does not apply
+  - rule: test-block-9
+    action: block
+    scope: pointLine
+    match: "foo"
+    newtag: newtag
+
+  # value does not apply
+  - rule: test-block-10
+    action: block
+    scope: pointLine
+    match: "foo"
+    value: "value"
+
+
+  # test allow
+
+  # test allow: missing parameters
+
+  # missing scope
+  - rule: test-allow-1
+    action: allow
+    match: "foo\\..*"
+
+  # missing match
+  - rule: test-allow-2
+    action: allow
+    scope: pointLine
+
+  # missing match
+  - rule: test-allow-3
+    action: allow
+    scope: metricName
+
+  # null match
+  - rule: test-allow-4
+    action: allow
+    scope: pointLine
+    match:
+
+  # empty match
+  - rule: test-allow-5
+    action: allow
+    scope: pointLine
+    match: ""
+
+  # test allow: non-applicable parameters
+
+  # tag does not apply
+  - rule: test-allow-6
+    action: allow
+    scope: pointLine
+    match: "foo"
+    tag: tag
+
+  # replace does not apply
+  - rule: test-allow-7
+    action: allow
+    scope: pointLine
+    match: "foo"
+    replace: replace
+
+  # search does not apply
+  - rule: test-allow-8
+    action: allow
+    scope: pointLine
+    match: "foo"
+    search: search
+
+  # newtag does not apply
+  - rule: test-allow-9
+    action: allow
+    scope: pointLine
+    match: "foo"
+    newtag: newtag
+
+  # value does not apply
+  - rule: test-allow-10
+    action: allow
+    scope: pointLine
+    match: "foo"
+    value: "value"
+
+
+  # test dropTag
+
+  # missing tag
+  - rule: test-dropTag-1
+    action: dropTag
+
+  # search does not apply
+  - rule: test-dropTag-2
+    action: dropTag
+    tag: tag
+    search: search
+
+  # replace does not apply
+  - rule: test-dropTag-3
+    action: dropTag
+    tag: tag
+    replace: replace
+
+  # newtag does not apply
+  - rule: test-dropTag-4
+    action: dropTag
+    tag: tag
+    newtag: newtag
+
+  # value does not apply
+  - rule: test-dropTag-5
+    action: dropTag
+    tag: tag
+    value: value
+
+
+  # test addTag
+
+  # missing tag
+  - rule: test-addTag-1
+    action: addTag
+    value: "1"
+
+  # null tag
+  - rule: test-addTag-2
+    action: addTag
+    tag:
+    value: "1"
+
+  # empty tag
+  - rule: test-addTag-3
+    action: addTag
+    tag: ""
+    value: "1"
+
+  # null value
+  - rule: test-addTag-4
+    action: addTag
+    tag: tag
+    value:
+
+  # empty value
+  - rule: test-addTag-5
+    action: addTag
+    tag: tag
+    value: ""
+
+  # missing value
+  - rule: test-addTag-6
+    action: addTag
+    tag: tag
+
+  # search does not apply
+  - rule: test-addTag-7
+    action: addTag
+    tag: tag
+    value: "1"
+    search: search
+
+  # replace does not apply
+  - rule: test-addTag-8
+    action: addTag
+    tag: tag
+    value: "1"
+    replace: replace
+
+  # newtag does not apply
+  - rule: test-addTag-9
+    action: addTag
+    tag: tag
+    value: "1"
+    newtag: newtag
+
+  # match does not apply
+  - rule: test-addTag-10
+    action: addTag
+    tag: tag
+    value: "1"
+    match: match
+
+
+  # test addTagIfNotExists
+
+  # missing tag
+  - rule: test-addTagIfNotExists-1
+    action: addTagIfNotExists
+    value: "1"
+
+  # null tag
+  - rule: test-addTagIfNotExists-2
+    action: addTagIfNotExists
+    tag:
+    value: "1"
+
+  # empty tag
+  - rule: test-addTagIfNotExists-3
+    action: addTagIfNotExists
+    tag: ""
+    value: "1"
+
+  # null value
+  - rule: test-addTagIfNotExists-4
+    action: addTagIfNotExists
+    tag: tag
+    value:
+
+  # empty value
+  - rule: test-addTagIfNotExists-5
+    action: addTagIfNotExists
+    tag: tag
+    value: ""
+
+  # missing value
+  - rule: test-addTagIfNotExists-6
+    action: addTagIfNotExists
+    tag: tag
+
+  # search does not apply
+  - rule: test-addTagIfNotExists-7
+    action: addTagIfNotExists
+    tag: tag
+    value: "1"
+    search: search
+
+  # replace does not apply
+  - rule: test-addTagIfNotExists-8
+    action: addTagIfNotExists
+    tag: tag
+    value: "1"
+    replace: replace
+
+  # newtag does not apply
+  - rule: test-addTagIfNotExists-9
+    action: addTagIfNotExists
+    tag: tag
+    value: "1"
+    newtag: newtag
+
+  # match does not apply
+  - rule: test-addTagIfNotExists-10
+    action: addTagIfNotExists
+    tag: tag
+    value: "1"
+    match: match
+
+  # test renameTag
+
+  # missing tag
+  - rule: test-renameTag-1
+    action: renameTag
+    match: tag
+    newtag: newtag
+
+  # null tag
+  - rule: test-renameTag-2
+    action: renameTag
+    tag:
+    match: tag
+    newtag: newtag
+
+  # empty tag
+  - rule: test-renameTag-3
+    action: renameTag
+    tag: ""
+    newtag: newtag
+
+  # missing newtag
+  - rule: test-renameTag-4
+    action: renameTag
+    tag: tag
+
+  # null newtag
+  - rule: test-renameTag-5
+    action: renameTag
+    tag: tag
+    newtag:
+
+  # empty newtag
+  - rule: test-renameTag-6
+    action: renameTag
+    tag: tag
+    newtag: ""
+
+  # search does not apply
+  - rule: test-renameTag-7
+    action: renameTag
+    tag: tag
+    match: match
+    newtag: newtag
+    search: search
+
+  # replace does not apply
+  - rule: test-renameTag-8
+    action: renameTag
+    tag: tag
+    match: match
+    newtag: newtag
+    replace: replace
+
+  # value does not apply
+  - rule: test-renameTag-9
+    action: renameTag
+    tag: tag
+    match: match
+    newtag: newtag
+    value: "1"
+
+
+  # test extractTag
+
+  # missing tag
+  - rule: test-extractTag-1
+    action: extractTag
+    match: tag
+    source: metricName
+    search: tag
+    replace: replace
+
+  # null tag
+  - rule: test-extractTag-2
+    action: extractTag
+    tag:
+    match: tag
+    source: metricName
+    search: tag
+    replace: replace
+
+  # empty tag
+  - rule: test-extractTag-3
+    action: extractTag
+    tag: ""
+    match: match
+    source: metricName
+    search: tag
+    replace: replace
+
+  # missing source
+  - rule: test-extractTag-4
+    action: extractTag
+    tag: tag
+    match: match
+    search: tag
+    replace: replace
+
+  # empty source
+  - rule: test-extractTag-5
+    action: extractTag
+    tag: tag
+    source: ""
+    match: match
+    search: tag
+    replace: replace
+
+  # missing search
+  - rule: test-extractTag-6
+    action: extractTag
+    tag: tag
+    source: metricName
+    match: match
+    replace: replace
+
+  # empty search
+  - rule: test-extractTag-7
+    action: extractTag
+    tag: tag
+    source: metricName
+    match: match
+    search: ""
+    replace: replace
+
+  # missing replace
+  - rule: test-extractTag-8
+    action: extractTag
+    tag: tag
+    source: metricName
+    match: match
+    search: tag
+
+  # null replace
+  - rule: test-extractTag-9
+    action: extractTag
+    tag: tag
+    source: metricName
+    match: match
+    search: tag
+    replace:
+
+  # scope does not apply
+  - rule: test-extractTag-10
+    action: extractTag
+    scope: tag
+    tag: tag
+    source: metricName
+    match: match
+    search: tag
+    replace: ""
+
+  # value does not apply
+  - rule: test-extractTag-11
+    action: extractTag
+    tag: tag
+    source: metricName
+    match: match
+    search: tag
+    replace: ""
+    value: "1"
+
+  # test extractTagIfNotExists
+
+  # missing tag
+  - rule: test-extractTagIfNotExists-1
+    action: extractTagIfNotExists
+    match: tag
+    source: metricName
+    search: tag
+    replace: replace
+
+  # null tag
+  - rule: test-extractTagIfNotExists-2
+    action: extractTagIfNotExists
+    tag:
+    match: tag
+    source: metricName
+    search: tag
+    replace: replace
+
+  # empty tag
+  - rule: test-extractTagIfNotExists-3
+    action: extractTagIfNotExists
+    tag: ""
+    match: match
+    source: metricName
+    search: tag
+    replace: replace
+
+  # missing source
+  - rule: test-extractTagIfNotExists-4
+    action: extractTagIfNotExists
+    tag: tag
+    match: match
+    search: tag
+    replace: replace
+
+  # empty source
+  - rule: test-extractTagIfNotExists-5
+    action: extractTagIfNotExists
+    tag: tag
+    source: ""
+    match: match
+    search: tag
+    replace: replace
+
+  # missing search
+  - rule: test-extractTagIfNotExists-6
+    action: extractTagIfNotExists
+    tag: tag
+    source: metricName
+    match: match
+    replace: replace
+
+  # empty search
+  - rule: test-extractTagIfNotExists-7
+    action: extractTagIfNotExists
+    tag: tag
+    source: metricName
+    match: match
+    search: ""
+    replace: replace
+
+  # missing replace
+  - rule: test-extractTagIfNotExists-8
+    action: extractTagIfNotExists
+    tag: tag
+    source: metricName
+    match: match
+    search: tag
+
+  # null replace
+  - rule: test-extractTagIfNotExists-9
+    action: extractTagIfNotExists
+    tag: tag
+    source: metricName
+    match: match
+    search: tag
+    replace:
+
+  # scope does not apply
+  - rule: test-extractTagIfNotExists-10
+    action: extractTagIfNotExists
+    scope: tag
+    tag: tag
+    source: metricName
+    match: match
+    search: tag
+    replace: ""
+
+  # value does not apply
+  - rule: test-extractTagIfNotExists-11
+    action: extractTagIfNotExists
+    tag: tag
+    source: metricName
+    match: match
+    search: tag
+    replace: ""
+    value: "1"
+
+  # test limitLength rule
+  # invalid subtype
+  - rule: test-limitLength-1
+    action: limitLength
+    actionSubtype: invalidsubtype
+    maxLength: "10"
+
+  # "drop" can't be used with metricName scope
+  - rule: test-limitLength-2
+    action: limitLength
+    scope: metricName
+    actionSubtype: drop
+    maxLength: "5"
+
+  # "drop" can't be used with sourceName scope
+  - rule: test-limitLength-3
+    action: limitLength
+    scope: sourceName
+    actionSubtype: drop
+    maxLength: "5"
+
+  # maxLength should be >= 3 for truncateWithEllipsis
+  - rule: test-limitLength-4
+    action: limitLength
+    scope: metricName
+    actionSubtype: truncateWithEllipsis
+    maxLength: "2"
+
+  # maxLength should be > 0
+  - rule: test-limitLength-5
+    action: limitLength
+    scope: metricName
+    actionSubtype: truncate
+    maxLength: "0"
+
+  # test spanLimitLength rule
+
+  # invalid subtype
+  - rule: test-spanLimitLength-1
+    action: spanLimitLength
+    actionSubtype: invalidsubtype
+    maxLength: "10"
+
+  # "drop" can't be used with spanName scope
+  - rule: test-spanLimitLength-2
+    action: spanLimitLength
+    scope: spanName
+    actionSubtype: drop
+    maxLength: "5"
+
+  # "drop" can't be used with sourceName scope
+  - rule: test-spanLimitLength-3
+    action: spanLimitLength
+    scope: sourceName
+    actionSubtype: drop
+    maxLength: "5"
+
+  # maxLength should be >= 3 for truncateWithEllipsis
+  - rule: test-spanLimitLength-4
+    action: spanLimitLength
+    scope: metricName
+    actionSubtype: truncateWithEllipsis
+    maxLength: "2"
+
+  # maxLength should be > 0
+  - rule: test-spanLimitLength-5
+    action: spanLimitLength
+    scope: metricName
+    actionSubtype: truncate
+    maxLength: "0"
+
+  # test spanRenameTag
+  # missing key
+  - rule: test-spanrenametag-1
+    action: spanRenameTag
+    newkey: device
+    match: "^\\d*$"
+
+  # missing newkey
+  - rule: test-spanrenametag-2
+    action: spanRenameTag
+    key: myDevice
+    match: "^\\d*$"
+
+  # null key
+  - rule: test-spanrenametag-3
+    action: spanRenameTag
+    key:
+    newkey: device
+    match: "^\\d*$"
+
+  # empty key
+  - rule: test-spanrenametag-4
+    action: spanRenameTag
+    key: ""
+    newkey: device
+    match: "^\\d*$"
+
+  # null newkey
+  - rule: test-spanrenametag-5
+    action: spanRenameTag
+    key: myDevice
+    newkey:
+    match: "^\\d*$"
+
+  # empty newkey
+  - rule: test-spanrenametag-6
+    action: spanRenameTag
+    key: myDevice
+    newkey: ""
+    match: "^\\d*$"
+
+  # wrong action - case doesn't match
+  - rule: test-spanrenametag-7
+    action: spanrenametag
+    key: myDevice
+    newkey: device
+
+  # Invalid v2 Predicate. Multiple top-level predicates.
+  - rule: test-invalidV2Pred-1
+    action: spanAllow
+    if:
+      all:
+        - equals:
+            scope: key2
+            value: "val2"
+        - contains:
+            scope: sourceName
+            value: "prod"
+      any:
+        - equals:
+            scope: key1
+            value: "val1"
+        - contains:
+            scope: metricName
+            value: "foometric"
+
+  # Invalid v2 Predicate due to Not specifying both v1 Predicates [scope, match].
+  - rule: test-invalidV2Pred-2
+    action: spanAllow
+    scope: metricName
+    if:
+      all:
+        - equals:
+            scope: key2
+            value: "val2"
+        - contains:
+            scope: sourceName
+            value: "prod"
+
+  # Invalid v2 Predicate due to Not specifying both v1 Predicates [scope, match].
+  - rule: test-invalidV2Pred-3
+    action: spanBlock
+    match: "^prod$"
+    if:
+      all:
+        - equals:
+            scope: key2
+            value: "val2"
+        - contains:
+            scope: sourceName
+            value: "prod"
+
+  # Invalid v2 Predicate due to Invalid scope.
+  - rule: test-invalidV2Pred-4
+    action: spanAllow
+    scope: pointline
+    if:
+      all:
+        - equals:
+            scope: key2
+            value: "val2"
+        - contains:
+            scope: sourceName
+            value: "prod"
+
+  # Invalid v2 Predicate due to blank value.
+  - rule: test-invalidV2Pred-5
+    action: spanAllow
+    if:
+      contains:
+        scope: sourceName
+        value:
+
+  # Invalid v2 Predicate due to no scope.
+  - rule: test-invalidV2Pred-6
+    action: spanAllow
+    if:
+      equals:
+        value: "val2"
+
+  # Invalid v2 Predicate due to invalid comparison function.
+  - rule: test-invalidV2Pred-7
+    action: allow
+    if:
+      invalidComparisonFunction:
+        scope: key1
+        value: "val1"
+
+  # Invalid v2 Predicate due to invalid comparison function.
+  - rule: test-invalidV2Pred-8
+    action: spanAllow
+    if:
+      invalidComparisonFunction:
+        scope: key1
+        value: "val1"
+
+'9997':
+  - rule: testFilter
+    action: metricsFilter
+    function: allow
+    names:
+      - "/metrics.2.*/"
+      - "/.*.ok$/"
+      - "metrics.ok.*"
+  - rule: testFilter
+    action: metricsFilter
+    function: allow
+    names:
+      - "metrics.1"
diff --git a/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_multiport.yaml b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_multiport.yaml
new file mode 100644
index 000000000..c9b17d686
--- /dev/null
+++ b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_multiport.yaml
@@ -0,0 +1,20 @@
+'2879':
+  - rule    : test-replace-foobar
+    action  : replaceRegex
+    scope   : metricName
+    search  : "foo"
+    replace : "bar"
+
+
+'2879, 1111':
+  - rule    : test-replace-foobar
+    action  : replaceRegex
+    scope   : metricName
+    search  : "bar"
+    replace : "bar1"
+
+'global':
+  - rule    : example-tag-all-metrics
+    action  : addTag
+    tag     : multiPortTagKey
+    value   : multiTagVal
diff --git a/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_order_test.yaml b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_order_test.yaml
new file mode 100644
index 000000000..0d75c7637
--- /dev/null
+++ b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_order_test.yaml
@@ -0,0 +1,12 @@
+'2878':
+  - rule: test-replace-foobar
+    action: replaceRegex
+    scope: metricName
+    search: "foo"
+    replace: "bar"
+
+  - rule: test-replace-somethingelse
+    action: replaceRegex
+    scope: metricName
+    search: "something"
+    replace: "else"
diff --git a/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_predicates.yaml b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_predicates.yaml
new file mode 100644
index 000000000..e5d160a20
--- /dev/null
+++ b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_predicates.yaml
@@ -0,0 +1,179 @@
+# Comparison Operator predicates.
+'equals-reportpoint':
+  - if:
+      equals:
+        scope: metricName
+        value: "foometric.1"
+
+'startsWith-reportpoint':
+  - if:
+      startsWith:
+        scope: metricName
+        value: "foometric."
+
+'endsWith-reportpoint':
+  - if:
+      endsWith:
+        scope: sourceName
+        value: "prod"
+
+'regexMatch-reportpoint':
+  - if:
+      regexMatch:
+        scope: sourceName
+        value: "^host$"
+
+'contains-reportpoint':
+  - if:
+      contains:
+        scope: sourceName
+        value: "prod"
+
+'equals-span':
+  - if:
+      equals:
+        scope: spanName
+        value: "testSpanName.1"
+
+'startsWith-span':
+  - if:
+      startsWith:
+        scope: spanName
+        value: "testSpanName."
+
+'endsWith-span':
+  - if:
+      endsWith:
+        scope: sourceName
+        value: "prod"
+
+'regexMatch-span':
+  - if:
+      regexMatch:
+        scope: sourceName
+        value: "^host$"
+
+'contains-span':
+  - if:
+      contains:
+        scope: sourceName
+        value: "prod"
+
+# Comparison Operator predicates with list values.
+'equals-list-reportpoint':
+  - if:
+      equals:
+        scope: metricName
+        value: ["foometric.1"]
+
+'startsWith-list-reportpoint':
+  - if:
+      startsWith:
+        scope: metricName
+        value: ["foometric."]
+
+'endsWith-list-reportpoint':
+  - if:
+      endsWith:
+        scope: sourceName
+        value: ["prod"]
+
+'regexMatch-list-reportpoint':
+  - if:
+      regexMatch:
+        scope: sourceName
+        value: ["^host$"]
+
+'contains-list-reportpoint':
+  - if:
+      contains:
+        scope: sourceName
+        value: ["prod"]
+
+'equals-list-span':
+  - if:
+      equals:
+        scope: spanName
+        value: ["testSpanName.1"]
+
+'startsWith-list-span':
+  - if:
+      startsWith:
+        scope: spanName
+        value: ["testSpanName."]
+
+'endsWith-list-span':
+  - if:
+      endsWith:
+        scope: sourceName
+        value: ["prod"]
+
+'regexMatch-list-span':
+  - if:
+      regexMatch:
+        scope: sourceName
+        value: ["^host$"]
+
+'contains-list-span':
+  - if:
+      contains:
+        scope: sourceName
+        value: ["prod"]
+
+# Logical Operator predicates.
+
+'logicalop-reportpoint':
+  - if:
+      all:
+        - any:
+          - startsWith:
+              scope: metricName
+              value: "foometric."
+          - endsWith:
+              scope: key1
+              value: "val11"
+        - none:
+          - contains:
+              scope: debug
+              value: "true"
+          - equals:
+              scope: key3
+              value: "val3"
+        - ignore:
+          - regexMatch:
+            scope: key2
+            value: "^val$"
+          - equals:
+            scope: key4
+            value: "val4"
+        - equals:
+            scope: key2
+            value: "val2"
+
+'logicalop-span':
+  - if:
+      all:
+        - any:
+          - startsWith:
+              scope: spanName
+              value: "testSpanName."
+          - endsWith:
+              scope: key1
+              value: "val11"
+        - none:
+          - contains:
+              scope: debug
+              value: "true"
+          - equals:
+              scope: key3
+              value: "val3"
+        - ignore:
+          - regexMatch:
+            scope: key2
+            value: "^val$"
+          - equals:
+            scope: key4
+            value: "val4"
+        - equals:
+            scope: key2
+            value: "val2"
diff --git a/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_reload.yaml b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_reload.yaml
new file mode 100644
index 000000000..597026632
--- /dev/null
+++ b/proxy/src/test/resources/com/wavefront/agent/preprocessor/preprocessor_rules_reload.yaml
@@ -0,0 +1,46 @@
+'2878':
+  - rule    : test-block-sourcename
+    action  : block
+    scope   : sourceName
+    match   : "bar.*"
+
+  # replace bad characters ("&", "$", "!") with underscores in the entire point line string
+  - rule    : test-replace-badchars
+    action  : replaceRegex
+    scope   : pointLine
+    search  : "[&\\$#!]"
+    replace : "_"
+
+  - rule    : test-dupe-2
+    action  : replaceRegex
+    scope   : pointLine
+    search  : "a"
+    replace : "b"
+
+  # remove "metrictest." from the metric name
+  - rule    : test-replace-metric-name
+    action  : replaceRegex
+    scope   : metricName
+    search  : "metrictest\\."
+    replace : ""
+
+  # for "bar" point tag replace all "-" characters with dots
+  - rule    : test-replace-tag-dash
+    action  : replaceRegex
+    scope   : bar
+    search  : "-"
+    replace : "."
+
+  - rule    : example-extract-metric-prefix
+    action  : extractTag
+    source  : metricName
+    tag     : prefix
+    search  : "^(some).*"
+    replace : "$1"
+
+'9999':
+  - rule: testFilter
+    action: metricsFilter
+    function: drop
+    names:
+      - "metrics.1"
diff --git a/proxy/src/test/resources/demo.cert b/proxy/src/test/resources/demo.cert
new file mode 100644
index 000000000..ae900c405
--- /dev/null
+++ b/proxy/src/test/resources/demo.cert
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDizCCAnOgAwIBAgIEBjJFdDANBgkqhkiG9w0BAQsFADB2MQswCQYDVQQGEwJV
+UzETMBEGA1UECBMKQ2FsaWZvcm5pYTESMBAGA1UEBxMJUGFsbyBBbHRvMQ8wDQYD
+VQQKEwZWTVdhcmUxEjAQBgNVBAsTCVdhdmVmcm9udDEZMBcGA1UEAxMQU3VyZXNo
+IFJlbmdhc2FteTAeFw0yMDA1MTMyMzMxMzZaFw0yMTA1MDgyMzMxMzZaMHYxCzAJ
+BgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRIwEAYDVQQHEwlQYWxvIEFs
+dG8xDzANBgNVBAoTBlZNV2FyZTESMBAGA1UECxMJV2F2ZWZyb250MRkwFwYDVQQD
+ExBTdXJlc2ggUmVuZ2FzYW15MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEAkZL/0pBEu6wh5bDW0+PKB3Jf+86HNYrIXJWHElpfo7T0T+Z6r8WTpKemzyVP
+hqmr7S+sGrIKvKR6wY/awcdvEbPlrKSChZMz1xy2RSvn0Wcpx5ESfNzHTcUqnfiA
+beKqdYy+98D8A3PdLdjmY+MzYqTlGu82eqeyHzf8/ZYfEP8D6imvOxr2xVpbR7vT
+gShVAfOFDQ7BPv3MUfQAgG2U6phVwtiba4/VjlDigw+ABTjy2FKiEdx/MvQgjMCz
+GLWyxMiFEWQP1q0PddDTZ2sLUMcWK7dkT3FE5DaZCh/26Rm1b3mn+Z/FnvXf+/bl
+gBi/6YbMo3XdO86r6yu9kcJ71wIDAQABoyEwHzAdBgNVHQ4EFgQUEWQt9iRqQMRP
+ThCrR6fNkFRe6XowDQYJKoZIhvcNAQELBQADggEBAI2Erp7yhSdG2CQ/Hx95FB06
+vNOc8hWuJqu7f/CsETjHk0+wk4bWS4ue6Fch2ZITV1ouJMWfLFaLT+OocW0kypMa
+y62kQhTk5qF5/HYbmTNelrmGCQjeJIJbTON7HEFO/sZHEZ8hCzWsqN/ZZR7AtsFg
+QYQ3w7LDRyO+M9tntGEfOZfcGnyR/7MeH7Td5MP/iPuIq1Unogjrl4PQ88HnYqDL
+4RVtANo59JdvxjOek50rK3DInagPD8c11jm0s/qd/r87Y2LbDtvjYhMp5ukm7uYZ
+iltQkHHDkNeXpsqXqNQc/CWXngU1hDKQopiXvmSNB7F8qPgxP0RH0WCOfaeDx8w=
+-----END CERTIFICATE-----
\ No newline at end of file
diff --git a/proxy/src/test/resources/demo.key b/proxy/src/test/resources/demo.key
new file mode 100644
index 000000000..0ef9fa195
--- /dev/null
+++ b/proxy/src/test/resources/demo.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCRkv/SkES7rCHl
+sNbT48oHcl/7zoc1ishclYcSWl+jtPRP5nqvxZOkp6bPJU+GqavtL6wasgq8pHrB
+j9rBx28Rs+WspIKFkzPXHLZFK+fRZynHkRJ83MdNxSqd+IBt4qp1jL73wPwDc90t
+2OZj4zNipOUa7zZ6p7IfN/z9lh8Q/wPqKa87GvbFWltHu9OBKFUB84UNDsE+/cxR
+9ACAbZTqmFXC2Jtrj9WOUOKDD4AFOPLYUqIR3H8y9CCMwLMYtbLEyIURZA/WrQ91
+0NNnawtQxxYrt2RPcUTkNpkKH/bpGbVveaf5n8We9d/79uWAGL/phsyjdd07zqvr
+K72RwnvXAgMBAAECggEANnei4lz6aqHQGQneh29eYwTnZubybhUcPI/x9ur7h9wn
+4VFiLCwnvt6/qhfStpb7bgZ9RYvCOqzsBUpW1lRReXUvBTaUY3gdWGo0xJLV7OLF
+nhborPFKXQ3dkTeuje7WSp87wKVjZcNPSV0zbsJOsqTx1+8TGjdujQG81gD6ZLgB
+Ypi8ewVr1IbEUS7cfJj46j98UUgYRvz3uq4QqvviZ+8gEw+/AH5VqSQmQmsQLz83
+WWjKVjjyP9oRhQg7VwT/tMEFV5rDtON9j5MBev4jikQLfRp/Kjlt0QUaWk8GkNSR
+dE48yvYU/ukrNko6zabviuZnkW/TJyff65BQY2J0sQKBgQD08qOkNjuvLzoF3bIM
+t3KY5Chum220J4YWta/Xl04OrMMWshp5zHg2yjbyRZ8yR0PQuPzfO/uIc3nAHqrt
+q60IW1nKvGopfN+kYF2TQIMHjJis0GdvUJnwmhP2HbW0TIrkddcDrkWTo4hI2sOS
+5PumOW5oIWOApTLU57NH4UT/PwKBgQCYJIIolnWeHSwclEbVUM8+sq9SXJPkD71L
+kHNFpB6EoA7FVSslPdxZu0gO7YGmEmccf8dPO50pT6bLjdtNmmG+LWpnmT4PPNGe
+9aoyJuOv3bN46SlDcdz6FUHj1db75xkqJQ8CHeh+SdS8kBcCIqjJMmsys0rYtcWe
+v2YrVrJ1aQKBgQCpZKs6Qq8fxV8w81HQbYT4qsAzTZWeQr7+MYN7ao12pI79wQmC
+NZ7k9Q7umKsxUAtb6rIlhwu6H3GRJSQ73L96ygHcrFQWgN8AhAvya2ix7c8fo7gE
+SQ9MTqGDUKR6HXzn5X5ec0R2h18WUwNxMJ2/JHRv2rc/Hf97MQjQqr7WbQKBgFZE
+tU0ga5b5Qa7+4N9KEAmkNkeEWRODXTnAsaw2cFuRim6YaXuXhR+YUzars80gODlv
+tusViXsIQDLBwC1TscKta91MhmULfm0dLaF8bbSmCIMx6oTkxoFDlnYDJgD2PE2q
+b8Uqgk9BvBAjv/glAQH8xc4c3f7dqy3lp6BBa7WpAoGAQmkF4lU/NPX837uMEWta
+Dzm07y8Wp+uLQM4Zs5/Q+6hygUDfUwUDWPRIYmKVusl2dfKYFgMGCAl0+hjet4EE
+XjyVZlCSZYeAp4m2msbzAq7OwpKstYorOfh0w0ZfKbY+mWnmjGAaa3D5Eurvx6wg
+MGPIZJCkprPpz5PE8S0m46Q=
+-----END PRIVATE KEY-----
\ No newline at end of file
diff --git a/proxy/src/test/resources/test-non-delta.yml b/proxy/src/test/resources/test-non-delta.yml
new file mode 100644
index 000000000..cf1d6d16c
--- /dev/null
+++ b/proxy/src/test/resources/test-non-delta.yml
@@ -0,0 +1,7 @@
+useDeltaCounters: false
+
+counters:
+  - pattern: "counterWithValue %{NUMBER:value}%{GREEDYDATA:extra}"
+    metricName: "counterWithValue"
+  - pattern: "plainCounter"
+    metricName: "plainCounter"
diff --git a/proxy/src/test/resources/test.yml b/proxy/src/test/resources/test.yml
index 0a6c45ee1..0dd58394f 100644
--- a/proxy/src/test/resources/test.yml
+++ b/proxy/src/test/resources/test.yml
@@ -27,6 +27,13 @@ counters:
       - "az-%{az}"
       - "value"
       - "aa%{q}bb"
+  - pattern: "operation %{WORD:op} on host %{WORD:hostname} took %{NUMBER:value} seconds"
+    metricName: "Host.%{op}.totalSeconds"
+    hostName: "%{hostname}.acme.corp"
+    tagKeys:
+      - "static"
+    tagValues:
+      - "value"
   - pattern: "%{COMBINEDAPACHELOG}"
     metricName: "apacheBytes"
     valueLabel: "bytes"
@@ -43,11 +50,37 @@ gauges:
   - pattern: '%{LOGLEVEL}: \[%{NUMBER:port}\] %{GREEDYDATA} points attempted: %{NUMBER:pointsAttempted}'
     metricName: "wavefrontPointsSent.%{port}"
     valueLabel: "pointsAttempted"
+  - pattern: 'pingSSO\|(\s*%{DATA:datetime}\s*)\|(\s*%{DATA:event}\s*)\|(\s*%{DATA:subject}\s*)\|(\s*%{DATA:ip}\s*)\|(\s*%{DATA:app}\s*)\|(\s*%{DATA:connectionid}\s*)\|(\s*%{DATA:protocol}\s*)\|(\s*%{DATA:host}\s*)\|(\s*%{DATA:role}\s*)\|(\s*%{DATA:status}\s*)\|(\s*%{DATA:adapterid}\s*)\|(\s*%{DATA:description}\s*)\|(\s*%{NUMBER:responsetime}\s*)'
+    metricName: 'pingSSO'
+    valueLabel: 'responsetime'
+    tagKeys:
+      - "event"
+      - "subject"
+      - "ip"
+      - "app"
+      - "connectionid"
+      - "protocol"
+      - "sso_host"
+      - "role"
+      - "status"
+      - "adapterid"
+      - "description"
+    tagValueLabels:
+      - "event"
+      - "subject"
+      - "ip"
+      - "app"
+      - "connectionid"
+      - "protocol"
+      - "host"
+      - "role"
+      - "status"
+      - "adapterid"
+      - "description"
 
 histograms:
   - pattern: "histo %{NUMBER:value}"
     metricName: "myHisto"
 
-
 additionalPatterns:
   - "MYPATTERN %{WORD:myword} and %{NUMBER:value}"
\ No newline at end of file
diff --git a/test/runner.py b/test/runner.py
index bc3abf02b..29782da48 100755
--- a/test/runner.py
+++ b/test/runner.py
@@ -695,7 +695,7 @@ def test_host_name_too_long(self):
         self.assertTrue(pushdata.empty())
 
         # check log file for blocked point
-        self.assert_blocked_point_in_log('WF-301:.*Host.*too long.*' +
+        self.assert_blocked_point_in_log('WF-407:.*Host.*too long.*' +
                                          hostname, False)
 
     def test_metric_name_too_long(self):
@@ -721,7 +721,7 @@ def test_metric_name_too_long(self):
         self.assertTrue(pushdata.empty())
 
         # check log file for blocked point
-        self.assert_blocked_point_in_log('WF-301: Metric name is too long.*' +
+        self.assert_blocked_point_in_log('WF-408: Metric name is too long.*' +
                                          metric_name, False)
 
     def test_metric_invalid_characters(self):
diff --git a/test/wavefront.conf b/test/wavefront.conf
index 48a918d61..53089b0fb 100644
--- a/test/wavefront.conf
+++ b/test/wavefront.conf
@@ -101,11 +101,11 @@ idFile=./.wavefront_id
 
 ## Regex pattern (java.util.regex) that input lines must match to be accepted.
 ## Input lines are checked against the pattern before the prefix is prepended.
-#whitelistRegex=^(production|stage).*
+#allow=^(production|stage).*
 
 ## Regex pattern (java.util.regex) that input lines must NOT match to be accepted.
 ## Input lines are checked against the pattern before the prefix is prepended.
-#blacklistRegex=^(qa|development|test).*
+#block=^(qa|development|test).*
 
 ## Whether to split the push batch size when the push is rejected by Wavefront due to rate limit.  Default false.
 #splitPushWhenRateLimited=false
diff --git a/tests/buffer-lock/Makefile b/tests/buffer-lock/Makefile
new file mode 100644
index 000000000..2bb3ec860
--- /dev/null
+++ b/tests/buffer-lock/Makefile
@@ -0,0 +1,36 @@
+tmp_dir := $(shell mktemp -d -t ci-XXXXXXXXXX)
+
+all: test-buffer-lock
+
+.check-env:
+ifndef WF_URL
+	$(error WF_URL is undefined)
+endif
+ifndef WF_TOKEN
+	$(error WF_TOKEN is undefined)
+endif
+
+test-buffer-lock: .check-env
+	@[ -d ${tmp_dir} ] || exit -1
+	WF_URL=${WF_URL} WF_TOKEN=${WF_TOKEN} docker-compose up --build -d
+	sleep 10
+	docker-compose kill
+	docker-compose logs --no-color | tee ${tmp_dir}/out.txt
+	docker-compose rm -f -v
+	echo ${tmp_dir}
+
+	grep OverlappingFileLockException $(tmp_dir)/out.txt || $(MAKE) .error
+	$(MAKE) .clean
+
+.clean:
+	@rm -rf ${tmp_dir}
+
+.error: .clean
+	@echo
+	@echo ERROR !!
+	@exit 1
+
+.ok: .clean
+	@echo
+	@echo OK !!
+	@exit 0
diff --git a/tests/buffer-lock/docker-compose.yml b/tests/buffer-lock/docker-compose.yml
new file mode 100644
index 000000000..c84fa551d
--- /dev/null
+++ b/tests/buffer-lock/docker-compose.yml
@@ -0,0 +1,25 @@
+volumes:
+  tmp: {}
+
+services:
+  proxy-1:
+    build: ../../docker
+    environment:
+      WAVEFRONT_URL: http://host.docker.internal:8080
+      WAVEFRONT_TOKEN: dhgjfdhgsjlkdf22340007-8fc6-4fc6-affa-b000ffa590ef
+      WAVEFRONT_PROXY_ARGS: --ephemeral false --idFile /var/spool/wavefront-proxy/id-1
+    volumes:
+      - tmp:/var/spool/wavefront-proxy
+    ports:
+      - "2878:2878"
+
+  proxy-2:
+    build: ../../docker
+    environment:
+      WAVEFRONT_URL: http://host.docker.internal:8080
+      WAVEFRONT_TOKEN: dhgjfdhgsjlkdf22340007-8fc6-4fc6-affa-b000ffa590ef
+      WAVEFRONT_PROXY_ARGS: --ephemeral false --idFile /var/spool/wavefront-proxy/id-2
+    volumes:
+      - tmp:/var/spool/wavefront-proxy
+    ports:
+      - "2879:2878"
diff --git a/tests/chain-checking/Makefile b/tests/chain-checking/Makefile
new file mode 100644
index 000000000..38049e4eb
--- /dev/null
+++ b/tests/chain-checking/Makefile
@@ -0,0 +1,28 @@
+UUID_E := $(shell uuidgen)
+UUID_C := $(shell uuidgen)
+
+all: test-chain-checking
+
+.check-env:
+ifndef WF_URL
+	$(error WF_URL is undefined)
+endif
+ifndef WF_TOKEN
+	$(error WF_TOKEN is undefined)
+endif
+
+test-chain-checking: .check-env
+	UUID_E=${UUID_E} UUID_C=${UUID_C} WF_URL=${WF_URL} WF_TOKEN=${WF_TOKEN} docker compose up --build -d --remove-orphans
+	sleep 30
+	docker compose kill
+	docker compose logs 
+	docker compose rm -f -v
+
+	curl -f -H 'Authorization: Bearer ${WF_TOKEN}' \
+			-H 'Content-Type: application/json' \
+			"https://${WF_URL}/api/v2/proxy/${UUID_E}" 
+
+	curl -f -H 'Authorization: Bearer ${WF_TOKEN}' \
+			-H 'Content-Type: application/json' \
+			"https://${WF_URL}/api/v2/proxy/${UUID_C}" 
+
diff --git a/tests/chain-checking/docker-compose.yml b/tests/chain-checking/docker-compose.yml
new file mode 100644
index 000000000..15ae0fd15
--- /dev/null
+++ b/tests/chain-checking/docker-compose.yml
@@ -0,0 +1,47 @@
+services:
+
+  proxy-edge:
+    hostname: proxy-edge
+    build: ../../docker
+    environment:
+      WAVEFRONT_URL: https://${WF_URL}/api/
+      WAVEFRONT_TOKEN: ${WF_TOKEN}
+      WAVEFRONT_PROXY_ARGS: --ephemeral false --idFile /var/spool/wavefront-proxy/id --pushRelayListenerPorts 2879
+    ports:
+      - "2878:2878"
+      - "2879:2879"
+    user: root
+    command:
+      [
+        "/bin/bash",
+        "-c",
+        "echo ${UUID_E} > /var/spool/wavefront-proxy/id && bash /opt/wavefront/wavefront-proxy/run.sh"
+      ]
+    healthcheck:
+      test: curl http://localhost:2879
+      interval: 3s
+      retries: 5
+
+  proxy-chained:
+    hostname: proxy-chained
+    build: ../../docker
+    environment:
+      WAVEFRONT_URL: http://proxy-edge:2879
+      WAVEFRONT_TOKEN: XXXX
+      WAVEFRONT_PROXY_ARGS: --ephemeral false --idFile /var/spool/wavefront-proxy/id
+    ports:
+      - "2978:2878"
+    user: root
+    command:
+      [
+        "/bin/bash",
+        "-c",
+        "echo ${UUID_C} > /var/spool/wavefront-proxy/id && bash /opt/wavefront/wavefront-proxy/run.sh"
+      ]
+    depends_on:
+      proxy-edge:
+        condition: service_healthy
+    healthcheck:
+      test: curl http://localhost:2879
+      interval: 3s
+      retries: 5
diff --git a/tests/datadog/Makefile b/tests/datadog/Makefile
new file mode 100644
index 000000000..e3a7ecc6d
--- /dev/null
+++ b/tests/datadog/Makefile
@@ -0,0 +1,15 @@
+all: testDD
+
+.check-env:
+ifndef WF_SERVER
+	$(error WF_SERVER is undefined)
+endif
+ifndef WF_TOKEN
+	$(error WF_TOKEN is undefined)
+endif
+ifndef DD_API_KEY
+	$(error DD_API_KEY is undefined)
+endif
+
+testDD: .check-env
+	WF_SERVER=${WF_SERVER} WF_TOKEN=${WF_TOKEN} DD_API_KEY=${DD_API_KEY} docker compose up --build --attach wf-proxy
diff --git a/tests/datadog/README.md b/tests/datadog/README.md
new file mode 100644
index 000000000..31f4e2780
--- /dev/null
+++ b/tests/datadog/README.md
@@ -0,0 +1,25 @@
+# "DataDog Agent -> WFProxy -> DataDog" Tests
+
+## Build Proxy
+
+On Proxy repo home run:
+
+```
+MVN_ARGS="-DskipTests" make build-jar docker
+```
+
+## Run test
+
+On `tests/datadog/` run:
+
+```
+WF_SERVER=nimba \
+WF_TOKEN=XXXXX \
+DD_API_KEY=XXXX \
+make
+```
+
+## Test if is working
+
+1. Go to you WF server, and serach for a metric `docker.cpu.usage`, you shoul get some series with a `dd_agent_version=7` tag, and other with a `dd_agent_version=6` tag.
+2. Do the same on your Datadog acount (the `dd_agent_version` will not be available)
diff --git a/tests/datadog/compose.yaml b/tests/datadog/compose.yaml
new file mode 100644
index 000000000..24f6c7a0a
--- /dev/null
+++ b/tests/datadog/compose.yaml
@@ -0,0 +1,42 @@
+services:
+  wf-proxy:
+    hostname: wf-proxy
+    build: ../../docker
+    environment:
+      WAVEFRONT_URL: https://${WF_SERVER}.wavefront.com/api/
+      WAVEFRONT_TOKEN: ${WF_TOKEN}
+      WAVEFRONT_PROXY_ARGS: >
+        --dataDogJsonPorts 2879,2880 
+        --dataDogProcessSystemMetrics true 
+        --dataDogProcessServiceChecks true 
+        --dataDogRequestRelayTarget https://api.datadoghq.com
+        --preprocessorConfigFile /tmp/preprocessor_rules.yaml
+    volumes:
+      - ${PWD}/preprocessor_rules.yaml:/tmp/preprocessor_rules.yaml
+
+    ports:
+      - "2878:2878"
+      - "2879:2879"
+      - "2880:2880"
+
+  dd-agent-7:
+    hostname: dd-agent-7
+    image: gcr.io/datadoghq/agent:7
+    environment:
+      DD_DD_URL: http://host.docker.internal:2879
+      DD_API_KEY: ${DD_API_KEY}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /proc/:/host/proc/:ro
+      - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
+
+  dd-agent-6:
+    hostname: dd-agent-6
+    image: gcr.io/datadoghq/agent:6
+    environment:
+      DD_DD_URL: http://host.docker.internal:2880
+      DD_API_KEY: ${DD_API_KEY}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /proc/:/host/proc/:ro
+      - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
diff --git a/tests/datadog/preprocessor_rules.yaml b/tests/datadog/preprocessor_rules.yaml
new file mode 100644
index 000000000..3b50e0bf4
--- /dev/null
+++ b/tests/datadog/preprocessor_rules.yaml
@@ -0,0 +1,11 @@
+'2879': 
+  - rule    : ddv7
+    action  : addTag
+    tag     : dd_agent_version
+    value   : "7"
+
+'2880': 
+  - rule    : ddv6
+    action  : addTag
+    tag     : dd_agent_version
+    value   : "6"
diff --git a/yammer-metrics/pom.xml b/yammer-metrics/pom.xml
deleted file mode 100644
index 660247dd0..000000000
--- a/yammer-metrics/pom.xml
+++ /dev/null
@@ -1,47 +0,0 @@
-
-
-    
-        wavefront
-        com.wavefront
-        4.35-SNAPSHOT
-    
-    4.0.0
-
-    yammer-metrics
-    Wavefront Yammer Metrics
-    
-        
-            com.yammer.metrics
-            metrics-core
-            2.2.0
-        
-        
-            junit
-            junit
-            4.12
-            test
-        
-        
-            com.wavefront
-            java-lib
-        
-        
-            com.google.guava
-            guava
-        
-        
-            org.hamcrest
-            hamcrest-all
-            1.3
-        
-        
-        
-            org.hamcrest
-            java-hamcrest
-            2.0.0.0
-            test
-        
-
-    
-
-
\ No newline at end of file
diff --git a/yammer-metrics/src/main/java/com/wavefront/integrations/metrics/Main.java b/yammer-metrics/src/main/java/com/wavefront/integrations/metrics/Main.java
deleted file mode 100644
index 0c8a3b294..000000000
--- a/yammer-metrics/src/main/java/com/wavefront/integrations/metrics/Main.java
+++ /dev/null
@@ -1,51 +0,0 @@
-package com.wavefront.integrations.metrics;
-
-import com.google.common.base.Joiner;
-
-import com.wavefront.common.TaggedMetricName;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.Histogram;
-import com.yammer.metrics.core.MetricsRegistry;
-import com.yammer.metrics.core.WavefrontHistogram;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Driver for basic experimentation with a {@link com.wavefront.integrations.metrics.WavefrontYammerMetricsReporter}
- *
- * @author Mori Bellamy (mori@wavefront.com)
- */
-public class Main {
-
-  public static void main(String[] args) throws IOException, InterruptedException {
-    // Parse inputs.
-    System.out.println("Args: " + Joiner.on(", ").join(args));
-    if (args.length != 2) {
-      System.out.println("Usage: java -jar this.jar  ");
-      return;
-    }
-    int port = Integer.parseInt(args[0]);
-    int histoPort = Integer.parseInt(args[1]);
-
-    // Set up periodic reporting.
-    MetricsRegistry metricsRegistry = new MetricsRegistry();
-    WavefrontYammerMetricsReporter wavefrontYammerMetricsReporter = new WavefrontYammerMetricsReporter(metricsRegistry,
-        "wavefrontYammerMetrics", "localhost", port, histoPort, System::currentTimeMillis);
-    wavefrontYammerMetricsReporter.start(5, TimeUnit.SECONDS);
-
-    // Populate test metrics.
-    Counter counter = metricsRegistry.newCounter(new TaggedMetricName("group", "mycounter", "tag1", "value1"));
-    Histogram histogram = metricsRegistry.newHistogram(new TaggedMetricName("group2", "myhisto"), false);
-    WavefrontHistogram wavefrontHistogram = WavefrontHistogram.get(metricsRegistry,
-        new TaggedMetricName("group", "mywavefronthisto", "tag2", "value2"));
-
-    while (true) {
-      counter.inc();
-      histogram.update(counter.count());
-      wavefrontHistogram.update(counter.count());
-      Thread.sleep(1000);
-    }
-  }
-
-}
diff --git a/yammer-metrics/src/main/java/com/wavefront/integrations/metrics/SocketMetricsProcessor.java b/yammer-metrics/src/main/java/com/wavefront/integrations/metrics/SocketMetricsProcessor.java
deleted file mode 100644
index e72410200..000000000
--- a/yammer-metrics/src/main/java/com/wavefront/integrations/metrics/SocketMetricsProcessor.java
+++ /dev/null
@@ -1,209 +0,0 @@
-package com.wavefront.integrations.metrics;
-
-import com.tdunning.math.stats.Centroid;
-import com.wavefront.common.MetricsToTimeseries;
-import com.wavefront.common.TaggedMetricName;
-import com.wavefront.metrics.ReconnectingSocket;
-import com.yammer.metrics.core.*;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Supplier;
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-import javax.net.SocketFactory;
-
-/**
- * Yammer MetricProcessor that sends metrics to a TCP Socket in Wavefront-format.
- *
- * This sends a DIFFERENT metrics taxonomy than the Wavefront "dropwizard" metrics reporters.
- *
- * @author Mori Bellamy (mori@wavefront.com)
- */
-public class SocketMetricsProcessor implements MetricProcessor {
-
-  private ReconnectingSocket metricsSocket, histogramsSocket;
-  private final Supplier timeSupplier;
-  private final boolean sendZeroCounters;
-  private final boolean sendEmptyHistograms;
-  private final boolean prependGroupName;
-  private final boolean clear;
-
-  private static final Pattern SIMPLE_NAMES = Pattern.compile("[^a-zA-Z0-9_.\\-~]");
-
-  /**
-   * @param hostname               Host of the WF-graphite telemetry sink.
-   * @param port                   Port of the WF-graphite telemetry sink.
-   * @param wavefrontHistogramPort Port of the WF histogram sink.
-   * @param timeSupplier           Gets the epoch timestamp in milliseconds.
-   * @param prependGroupName       If true, metrics have their group name prepended when flushed.
-   * @param clear                  If true, clear histograms and timers after flush.
-   */
-  SocketMetricsProcessor(String hostname, int port, int wavefrontHistogramPort, Supplier timeSupplier,
-                         boolean prependGroupName, boolean clear, boolean sendZeroCounters, boolean sendEmptyHistograms)
-      throws IOException {
-    this(hostname, port, wavefrontHistogramPort, timeSupplier, prependGroupName, clear, sendZeroCounters,
-        sendEmptyHistograms, null);
-  }
-
-  /**
-   * @param hostname                    Host of the WF-graphite telemetry sink.
-   * @param port                        Port of the WF-graphite telemetry sink.
-   * @param wavefrontHistogramPort      Port of the WF histogram sink.
-   * @param timeSupplier                Gets the epoch timestamp in milliseconds.
-   * @param prependGroupName            If true, metrics have their group name prepended when flushed.
-   * @param clear                       If true, clear histograms and timers after flush.
-   * @param connectionTimeToLiveMillis  Connection TTL, with expiration checked after each flush. When null,
-   *                                    TTL is not enforced.
-   */
-  SocketMetricsProcessor(String hostname, int port, int wavefrontHistogramPort, Supplier timeSupplier,
-                         boolean prependGroupName, boolean clear, boolean sendZeroCounters, boolean sendEmptyHistograms,
-                         @Nullable Long connectionTimeToLiveMillis)
-      throws IOException {
-    this.timeSupplier = timeSupplier;
-    this.sendZeroCounters = sendZeroCounters;
-    this.sendEmptyHistograms = sendEmptyHistograms;
-    this.metricsSocket = new ReconnectingSocket(hostname, port, SocketFactory.getDefault(), connectionTimeToLiveMillis,
-        timeSupplier);
-    this.histogramsSocket = new ReconnectingSocket(hostname, wavefrontHistogramPort, SocketFactory.getDefault(),
-        connectionTimeToLiveMillis, timeSupplier);
-    this.prependGroupName = prependGroupName;
-    this.clear = clear;
-  }
-
-  private String getName(MetricName name) {
-    if (prependGroupName && name.getGroup() != null && !name.getGroup().equals("")) {
-      return sanitize(name.getGroup() + "." + name.getName());
-    }
-    return sanitize(name.getName());
-  }
-
-  private static String sanitize(String name) {
-    return SIMPLE_NAMES.matcher(name).replaceAll("_");
-  }
-
-  /**
-   * @return " k1=v1 k2=v2 ..." if metricName is an instance of TaggedMetricName. "" otherwise.
-   */
-  private String tagsForMetricName(MetricName metricName) {
-    if (metricName instanceof TaggedMetricName) {
-      TaggedMetricName taggedMetricName = (TaggedMetricName) metricName;
-      StringBuilder sb = new StringBuilder();
-      for (Map.Entry entry : taggedMetricName.getTags().entrySet()) {
-        sb.append(" ").append(entry.getKey()).append("=\"").append(entry.getValue()).append("\"");
-      }
-      return sb.toString();
-    } else {
-      return "";
-    }
-  }
-
-  private void writeMetric(MetricName metricName, String nameSuffix, double value) throws Exception {
-    StringBuilder sb = new StringBuilder();
-    sb.append("\"").append(getName(metricName));
-    if (nameSuffix != null && !nameSuffix.equals("")) {
-      sb.append(".").append(nameSuffix);
-    }
-    sb.append("\" ").append(value).append(" ").append(timeSupplier.get() / 1000).append(tagsForMetricName(metricName));
-    metricsSocket.write(sb.append("\n").toString());
-  }
-
-  private void writeMetered(MetricName name, Metered metered) throws Exception {
-    for (Map.Entry entry : MetricsToTimeseries.explodeMetered(metered).entrySet()) {
-      writeMetric(name, entry.getKey(), entry.getValue());
-    }
-  }
-
-  private void writeSummarizable(MetricName name, Summarizable summarizable) throws Exception {
-    for (Map.Entry entry : MetricsToTimeseries.explodeSummarizable(summarizable).entrySet()) {
-      writeMetric(name, entry.getKey(), entry.getValue());
-    }
-  }
-
-  private void writeSampling(MetricName name, Sampling sampling) throws Exception {
-    for (Map.Entry entry : MetricsToTimeseries.explodeSampling(sampling).entrySet()) {
-      writeMetric(name, entry.getKey(), entry.getValue());
-    }
-  }
-
-  @Override
-  public void processMeter(MetricName name, Metered meter, Void context) throws Exception {
-    writeMetered(name, meter);
-  }
-
-  @Override
-  public void processCounter(MetricName name, Counter counter, Void context) throws Exception {
-    if (!sendZeroCounters && counter.count() == 0) return;
-
-    // handle delta counters
-    if (counter instanceof DeltaCounter) {
-      long count = DeltaCounter.processDeltaCounter((DeltaCounter) counter);
-      writeMetric(name, null, count);
-    } else {
-      writeMetric(name, null, counter.count());
-    }
-  }
-
-  @Override
-  public void processHistogram(MetricName name, Histogram histogram, Void context) throws Exception {
-    if (histogram instanceof WavefrontHistogram) {
-      WavefrontHistogram wavefrontHistogram = (WavefrontHistogram) histogram;
-      List bins = wavefrontHistogram.bins(clear);
-      if (bins.isEmpty()) return; // don't send empty histograms.
-      for (WavefrontHistogram.MinuteBin minuteBin : bins) {
-        StringBuilder sb = new StringBuilder();
-        sb.append("!M ").append(minuteBin.getMinMillis() / 1000);
-        for (Centroid c : minuteBin.getDist().centroids()) {
-          sb.append(" #").append(c.count()).append(" ").append(c.mean());
-        }
-        sb.append(" \"").append(getName(name)).append("\"").append(tagsForMetricName(name)).append("\n");
-        histogramsSocket.write(sb.toString());
-      }
-    } else {
-      if (!sendEmptyHistograms && histogram.count() == 0) {
-        // send count still but skip the others.
-        writeMetric(name, "count", 0);
-      } else {
-        writeMetric(name, "count", histogram.count());
-        writeSampling(name, histogram);
-        writeSummarizable(name, histogram);
-        if (clear) histogram.clear();
-      }
-    }
-  }
-
-  @Override
-  public void processTimer(MetricName name, Timer timer, Void context) throws Exception {
-    MetricName samplingName, rateName;
-    if (name instanceof TaggedMetricName) {
-      TaggedMetricName taggedMetricName = (TaggedMetricName) name;
-      samplingName = new TaggedMetricName(
-          taggedMetricName.getGroup(), taggedMetricName.getName() + ".duration", taggedMetricName.getTags());
-      rateName = new TaggedMetricName(
-          taggedMetricName.getGroup(), taggedMetricName.getName() + ".rate", taggedMetricName.getTags());
-    } else {
-      samplingName = new MetricName(name.getGroup(), name.getType(), name.getName() + ".duration");
-      rateName = new MetricName(name.getGroup(), name.getType(), name.getName() + ".rate");
-    }
-
-    writeSummarizable(samplingName, timer);
-    writeSampling(samplingName, timer);
-    writeMetered(rateName, timer);
-
-    if (clear) timer.clear();
-  }
-
-  @Override
-  public void processGauge(MetricName name, Gauge gauge, Void context) throws Exception {
-    if (gauge.value() != null) {
-      writeMetric(name, null, Double.valueOf(gauge.value().toString()));
-    }
-  }
-
-  public void flush() throws IOException {
-    metricsSocket.flush();
-    histogramsSocket.flush();
-  }
-}
diff --git a/yammer-metrics/src/main/java/com/wavefront/integrations/metrics/WavefrontYammerMetricsReporter.java b/yammer-metrics/src/main/java/com/wavefront/integrations/metrics/WavefrontYammerMetricsReporter.java
deleted file mode 100644
index d94bfabae..000000000
--- a/yammer-metrics/src/main/java/com/wavefront/integrations/metrics/WavefrontYammerMetricsReporter.java
+++ /dev/null
@@ -1,252 +0,0 @@
-package com.wavefront.integrations.metrics;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import com.wavefront.common.MetricsToTimeseries;
-import com.wavefront.common.Pair;
-import com.wavefront.metrics.MetricTranslator;
-import com.yammer.metrics.core.Clock;
-import com.yammer.metrics.core.Gauge;
-import com.yammer.metrics.core.Metric;
-import com.yammer.metrics.core.MetricName;
-import com.yammer.metrics.core.MetricsRegistry;
-import com.yammer.metrics.core.SafeVirtualMachineMetrics;
-import com.yammer.metrics.core.VirtualMachineMetrics;
-import com.yammer.metrics.core.WavefrontHistogram;
-import com.yammer.metrics.reporting.AbstractReporter;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Supplier;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
-public class WavefrontYammerMetricsReporter extends AbstractReporter implements Runnable {
-
-  protected static final Logger logger = Logger.getLogger(WavefrontYammerMetricsReporter.class.getCanonicalName());
-
-  private static final Clock clock = Clock.defaultClock();
-  private static final VirtualMachineMetrics vm = SafeVirtualMachineMetrics.getInstance();
-  private final ScheduledExecutorService executor;
-
-  private final boolean includeJvmMetrics;
-  private final ConcurrentHashMap gaugeMap;
-  private final SocketMetricsProcessor socketMetricProcessor;
-  private final MetricTranslator metricTranslator;
-
-  /**
-   *  How many metrics were emitted in the last call to run()
-   */
-  private AtomicInteger metricsGeneratedLastPass = new AtomicInteger();
-
-  public WavefrontYammerMetricsReporter(MetricsRegistry metricsRegistry, String name, String hostname, int port,
-                                        int wavefrontHistogramPort, Supplier timeSupplier) throws IOException {
-    this(metricsRegistry, name, hostname, port, wavefrontHistogramPort, timeSupplier, false, null, false, false);
-  }
-
-  public WavefrontYammerMetricsReporter(MetricsRegistry metricsRegistry, String name, String hostname, int port,
-                                        int wavefrontHistogramPort, Supplier timeSupplier,
-                                        boolean prependGroupName,
-                                        @Nullable MetricTranslator metricTranslator,
-                                        boolean includeJvmMetrics,
-                                        boolean clearMetrics) throws IOException {
-    this(metricsRegistry, name, hostname, port, wavefrontHistogramPort, timeSupplier, prependGroupName,
-        metricTranslator, includeJvmMetrics, clearMetrics, true, true);
-  }
-
-  /**
-   * Reporter of a Yammer metrics registry to Wavefront.
-   *
-   * @param metricsRegistry        The registry to scan-and-report
-   * @param name                   A human readable name for this reporter
-   * @param hostname               The remote host where the wavefront proxy resides
-   * @param port                   Listening port on Wavefront proxy of graphite-like telemetry data
-   * @param wavefrontHistogramPort Listening port for Wavefront histogram data
-   * @param timeSupplier           Get current timestamp, stubbed for testing
-   * @param prependGroupName       If true, outgoing telemetry is of the form "group.name" rather than "name".
-   * @param metricTranslator       If present, applied to each MetricName/Metric pair before flushing to Wavefront. This
-   *                               is useful for adding point tags. Warning: this is called once per metric per scan, so
-   *                               it should probably be performant. May be null.
-   * @param clearMetrics           If true, clear histograms and timers per flush.
-   * @param sendZeroCounters       Whether counters with a value of zero is sent across.
-   * @param sendEmptyHistograms    Whether empty histograms are sent across the wire.
-   * @param includeJvmMetrics      Whether JVM metrics are automatically included.
-   * @throws IOException When we can't remotely connect to Wavefront.
-   */
-  public WavefrontYammerMetricsReporter(MetricsRegistry metricsRegistry, String name, String hostname, int port,
-                                        int wavefrontHistogramPort, Supplier timeSupplier,
-                                        boolean prependGroupName,
-                                        @Nullable MetricTranslator metricTranslator,
-                                        boolean includeJvmMetrics,
-                                        boolean clearMetrics,
-                                        boolean sendZeroCounters,
-                                        boolean sendEmptyHistograms) throws IOException {
-    this(metricsRegistry, name, hostname, port, wavefrontHistogramPort, timeSupplier, prependGroupName,
-        metricTranslator, includeJvmMetrics, clearMetrics, sendZeroCounters, sendEmptyHistograms, null);
-  }
-
-  /**
-   * Reporter of a Yammer metrics registry to Wavefront.
-   *
-   * @param metricsRegistry             The registry to scan-and-report
-   * @param name                        A human readable name for this reporter
-   * @param hostname                    The remote host where the wavefront proxy resides
-   * @param port                        Listening port on Wavefront proxy of graphite-like telemetry data
-   * @param wavefrontHistogramPort      Listening port for Wavefront histogram data
-   * @param timeSupplier                Get current timestamp, stubbed for testing
-   * @param prependGroupName            If true, outgoing telemetry is of the form "group.name" rather than "name".
-   * @param metricTranslator            If present, applied to each MetricName/Metric pair before flushing to Wavefront.
-   *                                    This is useful for adding point tags. Warning: this is called once per metric
-   *                                    per scan, so it should probably be performant. May be null.
-   * @param clearMetrics                If true, clear histograms and timers per flush.
-   * @param sendZeroCounters            Whether counters with a value of zero is sent across.
-   * @param sendEmptyHistograms         Whether empty histograms are sent across the wire.
-   * @param includeJvmMetrics           Whether JVM metrics are automatically included.
-   * @param connectionTimeToLiveMillis  Connection TTL, with expiration checked after each flush. When null,
-   *                                    TTL is not enforced.
-   * @throws IOException When we can't remotely connect to Wavefront.
-   */
-  public WavefrontYammerMetricsReporter(MetricsRegistry metricsRegistry, String name, String hostname, int port,
-                                        int wavefrontHistogramPort, Supplier timeSupplier,
-                                        boolean prependGroupName,
-                                        @Nullable MetricTranslator metricTranslator,
-                                        boolean includeJvmMetrics,
-                                        boolean clearMetrics,
-                                        boolean sendZeroCounters,
-                                        boolean sendEmptyHistograms,
-                                        @Nullable Long connectionTimeToLiveMillis) throws IOException {
-    super(metricsRegistry);
-    this.executor = metricsRegistry.newScheduledThreadPool(1, name);
-    this.metricTranslator = metricTranslator;
-    this.socketMetricProcessor = new SocketMetricsProcessor(hostname, port, wavefrontHistogramPort, timeSupplier,
-        prependGroupName, clearMetrics, sendZeroCounters, sendEmptyHistograms, connectionTimeToLiveMillis);
-    this.includeJvmMetrics = includeJvmMetrics;
-    this.gaugeMap = new ConcurrentHashMap<>();
-  }
-
-  private void upsertGauges(String metricName, Double t) {
-    gaugeMap.put(metricName, t);
-
-    // This call to newGauge only mutates the metrics registry the first time through. Thats why it's important
-    // to access gaugeMap indirectly, as opposed to counting on new calls to newGauage to replace the underlying
-    // double supplier.
-    getMetricsRegistry().newGauge(
-        new MetricName("", "", MetricsToTimeseries.sanitize(metricName)),
-        new Gauge() {
-          @Override
-          public Double value() {
-            return gaugeMap.get(metricName);
-          }
-        });
-  }
-
-  private void upsertGauges(String base, Map metrics) {
-    for (Map.Entry entry : metrics.entrySet()) {
-      upsertGauges(base + "." + entry.getKey(), entry.getValue());
-    }
-  }
-
-  private void upsertJavaMetrics() {
-    upsertGauges("jvm.memory", MetricsToTimeseries.memoryMetrics(vm));
-    upsertGauges("jvm.buffers.direct", MetricsToTimeseries.buffersMetrics(vm.getBufferPoolStats().get("direct")));
-    upsertGauges("jvm.buffers.mapped", MetricsToTimeseries.buffersMetrics(vm.getBufferPoolStats().get("mapped")));
-    upsertGauges("jvm.thread-states", MetricsToTimeseries.threadStateMetrics(vm));
-    upsertGauges("jvm", MetricsToTimeseries.vmMetrics(vm));
-    upsertGauges("current_time", (double) clock.time());
-    for (Map.Entry entry : vm.garbageCollectors().entrySet()) {
-      upsertGauges("jvm.garbage-collectors." + entry.getKey(), MetricsToTimeseries.gcMetrics(entry.getValue()));
-    }
-  }
-
-  /**
-   * @return How many metrics were processed during the last call to {@link #run()}.
-   */
-  @VisibleForTesting
-  int getMetricsGeneratedLastPass() {
-    return metricsGeneratedLastPass.get();
-  }
-
-  /**
-   * Starts the reporter polling at the given period.
-   *
-   * @param period    the amount of time between polls
-   * @param unit      the unit for {@code period}
-   */
-  public void start(long period, TimeUnit unit) {
-    executor.scheduleAtFixedRate(this, period, period, unit);
-  }
-
-  /**
-   * Starts the reporter polling at the given period with specified initial delay
-   *
-   * @param initialDelay the amount of time before first execution
-   * @param period       the amount of time between polls
-   * @param unit         the unit for {@code initialDelay} and {@code period}
-   */
-  public void start(long initialDelay, long period, TimeUnit unit) {
-    executor.scheduleAtFixedRate(this, initialDelay, period, unit);
-  }
-
-  /**
-   * Shuts down the reporter polling, waiting the specific amount of time for any current polls to
-   * complete.
-   *
-   * @param timeout    the maximum time to wait
-   * @param unit       the unit for {@code timeout}
-   * @throws InterruptedException if interrupted while waiting
-   */
-  public void shutdown(long timeout, TimeUnit unit) throws InterruptedException {
-    executor.shutdown();
-    executor.awaitTermination(timeout, unit);
-  }
-
-  @Override
-  public void shutdown() {
-    executor.shutdown();
-    super.shutdown();
-  }
-
-  @Override
-  public void run() {
-    metricsGeneratedLastPass.set(0);
-    try {
-      if (includeJvmMetrics) upsertJavaMetrics();
-
-      // non-histograms go first
-      getMetricsRegistry().allMetrics().entrySet().stream().filter(m -> !(m.getValue() instanceof WavefrontHistogram)).
-          forEach(this::processEntry);
-      // histograms go last
-      getMetricsRegistry().allMetrics().entrySet().stream().filter(m -> m.getValue() instanceof WavefrontHistogram).
-          forEach(this::processEntry);
-      socketMetricProcessor.flush();
-    } catch (Exception e) {
-      logger.log(Level.SEVERE, "Cannot report point to Wavefront! Trying again next iteration.", e);
-    }
-  }
-
-  private void processEntry(Map.Entry entry) {
-    try {
-      MetricName metricName = entry.getKey();
-      Metric metric = entry.getValue();
-      if (metricTranslator != null) {
-        Pair pair = metricTranslator.apply(Pair.of(metricName, metric));
-        if (pair == null) return;
-        metricName = pair._1;
-        metric = pair._2;
-      }
-      metric.processWith(socketMetricProcessor, metricName, null);
-      metricsGeneratedLastPass.incrementAndGet();
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-}
diff --git a/yammer-metrics/src/test/java/com/wavefront/integrations/metrics/WavefrontYammerMetricsReporterTest.java b/yammer-metrics/src/test/java/com/wavefront/integrations/metrics/WavefrontYammerMetricsReporterTest.java
deleted file mode 100644
index 3b90a92b1..000000000
--- a/yammer-metrics/src/test/java/com/wavefront/integrations/metrics/WavefrontYammerMetricsReporterTest.java
+++ /dev/null
@@ -1,379 +0,0 @@
-package com.wavefront.integrations.metrics;
-
-import com.google.common.collect.Lists;
-
-import com.wavefront.common.Pair;
-import com.wavefront.common.TaggedMetricName;
-import com.wavefront.metrics.MetricTranslator;
-import com.yammer.metrics.core.Counter;
-import com.yammer.metrics.core.Gauge;
-import com.yammer.metrics.core.Histogram;
-import com.yammer.metrics.core.Meter;
-import com.yammer.metrics.core.MetricName;
-import com.yammer.metrics.core.MetricsRegistry;
-import com.yammer.metrics.core.Timer;
-import com.yammer.metrics.core.WavefrontHistogram;
-
-import org.hamcrest.text.MatchesPattern;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.startsWith;
-import static org.hamcrest.core.IsCollectionContaining.hasItem;
-
-/**
- * @author Mori Bellamy (mori@wavefront.com)
- */
-public class WavefrontYammerMetricsReporterTest {
-
-  private MetricsRegistry metricsRegistry;
-  private WavefrontYammerMetricsReporter wavefrontYammerMetricsReporter;
-  private Socket metricsSocket, histogramsSocket;
-  private ServerSocket metricsServer, histogramsServer;
-  private BufferedInputStream fromMetrics, fromHistograms;
-  private Long stubbedTime = 1485224035000L;
-
-  private void innerSetUp(boolean prependGroupName, MetricTranslator metricTranslator,
-                          boolean includeJvmMetrics, boolean clear)
-      throws Exception {
-    metricsRegistry = new MetricsRegistry();
-    metricsServer = new ServerSocket(0);
-    histogramsServer = new ServerSocket(0);
-    wavefrontYammerMetricsReporter = new WavefrontYammerMetricsReporter(
-        metricsRegistry, "test", "localhost", metricsServer.getLocalPort(), histogramsServer.getLocalPort(),
-        () -> stubbedTime, prependGroupName, metricTranslator, includeJvmMetrics, clear);
-    metricsSocket = metricsServer.accept();
-    histogramsSocket = histogramsServer.accept();
-    fromMetrics = new BufferedInputStream(metricsSocket.getInputStream());
-    fromHistograms = new BufferedInputStream(histogramsSocket.getInputStream());
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    innerSetUp(false, null, false, false);
-  }
-
-  @After
-  public void tearDown() throws IOException {
-    metricsSocket.close();
-    histogramsSocket.close();
-    metricsServer.close();
-    histogramsServer.close();
-  }
-
-  List receiveFromSocket(int numMetrics, InputStream stream) throws IOException {
-    List received = Lists.newArrayListWithCapacity(numMetrics);
-    // Read N metrics, which are produced from the prepared registry. If N is too high, we will time out. If
-    // N is too low, the asserts later should fail.
-    for (int i = 0; i < numMetrics; i++) {
-      ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-      int c;
-      while ((c = stream.read()) != '\n') {
-        byteArrayOutputStream.write(c);
-      }
-      received.add(new String(byteArrayOutputStream.toByteArray(), "UTF-8"));
-    }
-    return received;
-  }
-
-  @Test(timeout = 1000)
-  public void testJvmMetrics() throws Exception {
-    innerSetUp(true, null, true, false);
-    wavefrontYammerMetricsReporter.run();
-    List metrics = receiveFromSocket(
-        wavefrontYammerMetricsReporter.getMetricsGeneratedLastPass(), fromMetrics);
-    assertThat(metrics, not(hasItem(MatchesPattern.matchesPattern("\".* .*\".*"))));
-    assertThat(metrics, hasItem(startsWith("\"jvm.memory.heapCommitted\"")));
-    assertThat(metrics, hasItem(startsWith("\"jvm.fd_usage\"")));
-    assertThat(metrics, hasItem(startsWith("\"jvm.buffers.mapped.totalCapacity\"")));
-    assertThat(metrics, hasItem(startsWith("\"jvm.buffers.direct.totalCapacity\"")));
-    assertThat(metrics, hasItem(startsWith("\"jvm.thread-states.runnable\"")));
-  }
-
-
-  @Test(timeout = 1000)
-  public void testPlainCounter() throws Exception {
-    Counter counter = metricsRegistry.newCounter(WavefrontYammerMetricsReporterTest.class, "mycount");
-    counter.inc();
-    counter.inc();
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(1, fromMetrics), contains(equalTo("\"mycount\" 2.0 1485224035")));
-  }
-
-  @Test(timeout = 1000)
-  public void testTransformer() throws Exception {
-    innerSetUp(false, pair -> Pair.of(new TaggedMetricName(
-        pair._1.getGroup(), pair._1.getName(), "tagA", "valueA"), pair._2), false, false);
-    TaggedMetricName taggedMetricName = new TaggedMetricName("group", "mycounter",
-        "tag1", "value1", "tag2", "value2");
-    Counter counter = metricsRegistry.newCounter(taggedMetricName);
-    counter.inc();
-    counter.inc();
-    wavefrontYammerMetricsReporter.run();
-    assertThat(
-        receiveFromSocket(1, fromMetrics),
-        contains(equalTo("\"mycounter\" 2.0 1485224035 tagA=\"valueA\"")));
-  }
-
-  @Test(timeout = 1000)
-  public void testTaggedCounter() throws Exception {
-    TaggedMetricName taggedMetricName = new TaggedMetricName("group", "mycounter",
-        "tag1", "value1", "tag2", "value2");
-    Counter counter = metricsRegistry.newCounter(taggedMetricName);
-    counter.inc();
-    counter.inc();
-    wavefrontYammerMetricsReporter.run();
-    assertThat(
-        receiveFromSocket(1, fromMetrics),
-        contains(equalTo("\"mycounter\" 2.0 1485224035 tag1=\"value1\" tag2=\"value2\"")));
-  }
-
-  @Test(timeout = 1000)
-  public void testPlainHistogramWithClear() throws Exception {
-    innerSetUp(false, null, false, true /* clear */);
-    Histogram histogram = metricsRegistry.newHistogram(WavefrontYammerMetricsReporterTest.class, "myhisto");
-    histogram.update(1);
-    histogram.update(10);
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(11, fromMetrics), containsInAnyOrder(
-        equalTo("\"myhisto.count\" 2.0 1485224035"),
-        equalTo("\"myhisto.min\" 1.0 1485224035"),
-        equalTo("\"myhisto.max\" 10.0 1485224035"),
-        equalTo("\"myhisto.mean\" 5.5 1485224035"),
-        equalTo("\"myhisto.sum\" 11.0 1485224035"),
-        startsWith("\"myhisto.stddev\""),
-        equalTo("\"myhisto.median\" 5.5 1485224035"),
-        equalTo("\"myhisto.p75\" 10.0 1485224035"),
-        equalTo("\"myhisto.p95\" 10.0 1485224035"),
-        equalTo("\"myhisto.p99\" 10.0 1485224035"),
-        equalTo("\"myhisto.p999\" 10.0 1485224035")
-    ));
-    // Second run should clear data.
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(11, fromMetrics), hasItem("\"myhisto.count\" 0.0 1485224035"));
-  }
-
-  @Test(timeout = 1000)
-  public void testPlainHistogramWithoutClear() throws Exception {
-    innerSetUp(false, null, false, false /* clear */);
-    Histogram histogram = metricsRegistry.newHistogram(WavefrontYammerMetricsReporterTest.class, "myhisto");
-    histogram.update(1);
-    histogram.update(10);
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(11, fromMetrics), containsInAnyOrder(
-        equalTo("\"myhisto.count\" 2.0 1485224035"),
-        equalTo("\"myhisto.min\" 1.0 1485224035"),
-        equalTo("\"myhisto.max\" 10.0 1485224035"),
-        equalTo("\"myhisto.mean\" 5.5 1485224035"),
-        equalTo("\"myhisto.sum\" 11.0 1485224035"),
-        startsWith("\"myhisto.stddev\""),
-        equalTo("\"myhisto.median\" 5.5 1485224035"),
-        equalTo("\"myhisto.p75\" 10.0 1485224035"),
-        equalTo("\"myhisto.p95\" 10.0 1485224035"),
-        equalTo("\"myhisto.p99\" 10.0 1485224035"),
-        equalTo("\"myhisto.p999\" 10.0 1485224035")
-    ));
-    // Second run should be the same.
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(11, fromMetrics), containsInAnyOrder(
-        equalTo("\"myhisto.count\" 2.0 1485224035"),
-        equalTo("\"myhisto.min\" 1.0 1485224035"),
-        equalTo("\"myhisto.max\" 10.0 1485224035"),
-        equalTo("\"myhisto.mean\" 5.5 1485224035"),
-        equalTo("\"myhisto.sum\" 11.0 1485224035"),
-        startsWith("\"myhisto.stddev\""),
-        equalTo("\"myhisto.median\" 5.5 1485224035"),
-        equalTo("\"myhisto.p75\" 10.0 1485224035"),
-        equalTo("\"myhisto.p95\" 10.0 1485224035"),
-        equalTo("\"myhisto.p99\" 10.0 1485224035"),
-        equalTo("\"myhisto.p999\" 10.0 1485224035")
-    ));
-  }
-
-  @Test(timeout = 1000)
-  public void testWavefrontHistogram() throws Exception {
-    AtomicLong clock = new AtomicLong(System.currentTimeMillis());
-    long timeBin = (clock.get() / 60000 * 60);
-    WavefrontHistogram wavefrontHistogram = WavefrontHistogram.get(metricsRegistry, new TaggedMetricName(
-        "group", "myhisto", "tag1", "value1", "tag2", "value2"), clock::get);
-    for (int i = 0; i < 101; i++) {
-      wavefrontHistogram.update(i);
-    }
-
-    // Advance the clock by 1 min ...
-    clock.addAndGet(60000L + 1);
-
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(1, fromHistograms), contains(equalTo(
-        "!M " + timeBin + " #1 0.0 #1 1.0 #1 2.0 #1 3.0 #1 4.0 #1 5.0 #1 6.0 #1 7.0 #1 8.0 #1 9.0 #1 10.0 #1 11.0 #1 12.0 #1 13.0 #1 14.0 #1 15.0 #1 16.0 #1 17.0 #1 18.0 #1 19.0 #1 20.0 #1 21.0 #1 22.0 #1 23.0 #1 24.0 #1 25.0 #1 26.0 #1 27.0 #1 28.0 #1 29.0 #1 30.0 #1 31.0 #1 32.0 #1 33.0 #1 34.0 #1 35.0 #1 36.0 #1 37.0 #1 38.0 #1 39.0 #1 40.0 #1 41.0 #1 42.0 #1 43.0 #1 44.0 #1 45.0 #1 46.0 #1 47.0 #1 48.0 #1 49.0 #1 50.0 #1 51.0 #1 52.0 #1 53.0 #1 54.0 #1 55.0 #1 56.0 #1 57.0 #1 58.0 #1 59.0 #1 60.0 #1 61.0 #1 62.0 #1 63.0 #1 64.0 #1 65.0 #1 66.0 #1 67.0 #1 68.0 #1 69.0 #1 70.0 #1 71.0 #1 72.0 #1 73.0 #1 74.0 #1 75.0 #1 76.0 #1 77.0 #1 78.0 #1 79.0 #1 80.0 #1 81.0 #1 82.0 #1 83.0 #1 84.0 #1 85.0 #1 86.0 #1 87.0 #1 88.0 #1 89.0 #1 90.0 #1 91.0 #1 92.0 #1 93.0 #1 94.0 #1 95.0 #1 96.0 #1 97.0 #1 98.0 #1 99.0 #1 100.0 \"myhisto\" tag1=\"value1\" tag2=\"value2\"")));
-  }
-
-  @Test(timeout = 1000)
-  public void testPlainMeter() throws Exception {
-    Meter meter = metricsRegistry.newMeter(WavefrontYammerMetricsReporterTest.class, "mymeter", "requests",
-        TimeUnit.SECONDS);
-    meter.mark(42);
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(5, fromMetrics), containsInAnyOrder(
-        equalTo("\"mymeter.count\" 42.0 1485224035"),
-        startsWith("\"mymeter.mean\""),
-        startsWith("\"mymeter.m1\""),
-        startsWith("\"mymeter.m5\""),
-        startsWith("\"mymeter.m15\"")
-    ));
-  }
-
-  @Test(timeout = 1000)
-  public void testPlainGauge() throws Exception {
-    Gauge gauge = metricsRegistry.newGauge(
-        WavefrontYammerMetricsReporterTest.class, "mygauge", new Gauge() {
-          @Override
-          public Double value() {
-            return 13.0;
-          }
-        });
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(1, fromMetrics), contains(equalTo("\"mygauge\" 13.0 1485224035")));
-  }
-
-  @Test(timeout = 1000)
-  public void testTimerWithClear() throws Exception {
-    innerSetUp(false, null, false, true /* clear */);
-    Timer timer = metricsRegistry.newTimer(new TaggedMetricName("", "mytimer", "foo", "bar"),
-        TimeUnit.SECONDS, TimeUnit.SECONDS);
-    timer.time().stop();
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(15, fromMetrics), containsInAnyOrder(
-        equalTo("\"mytimer.rate.count\" 1.0 1485224035 foo=\"bar\""),
-        startsWith("\"mytimer.duration.min\""),
-        startsWith("\"mytimer.duration.max\""),
-        startsWith("\"mytimer.duration.mean\""),
-        startsWith("\"mytimer.duration.sum\""),
-        startsWith("\"mytimer.duration.stddev\""),
-        startsWith("\"mytimer.duration.median\""),
-        startsWith("\"mytimer.duration.p75\""),
-        startsWith("\"mytimer.duration.p95\""),
-        startsWith("\"mytimer.duration.p99\""),
-        startsWith("\"mytimer.duration.p999\""),
-        startsWith("\"mytimer.rate.m1\""),
-        startsWith("\"mytimer.rate.m5\""),
-        startsWith("\"mytimer.rate.m15\""),
-        startsWith("\"mytimer.rate.mean\"")
-    ));
-
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(15, fromMetrics), hasItem("\"mytimer.rate.count\" 0.0 1485224035 foo=\"bar\""));
-  }
-
-  @Test(timeout = 1000)
-  public void testPlainTimerWithoutClear() throws Exception {
-    innerSetUp(false, null, false, false /* clear */);
-    Timer timer = metricsRegistry.newTimer(WavefrontYammerMetricsReporterTest.class, "mytimer");
-    timer.time().stop();
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(15, fromMetrics), containsInAnyOrder(
-        equalTo("\"mytimer.rate.count\" 1.0 1485224035"),
-        startsWith("\"mytimer.duration.min\""),
-        startsWith("\"mytimer.duration.max\""),
-        startsWith("\"mytimer.duration.mean\""),
-        startsWith("\"mytimer.duration.sum\""),
-        startsWith("\"mytimer.duration.stddev\""),
-        startsWith("\"mytimer.duration.median\""),
-        startsWith("\"mytimer.duration.p75\""),
-        startsWith("\"mytimer.duration.p95\""),
-        startsWith("\"mytimer.duration.p99\""),
-        startsWith("\"mytimer.duration.p999\""),
-        startsWith("\"mytimer.rate.m1\""),
-        startsWith("\"mytimer.rate.m5\""),
-        startsWith("\"mytimer.rate.m15\""),
-        startsWith("\"mytimer.rate.mean\"")
-    ));
-
-    // No changes.
-    wavefrontYammerMetricsReporter.run();
-    assertThat(receiveFromSocket(15, fromMetrics), containsInAnyOrder(
-        equalTo("\"mytimer.rate.count\" 1.0 1485224035"),
-        startsWith("\"mytimer.duration.min\""),
-        startsWith("\"mytimer.duration.max\""),
-        startsWith("\"mytimer.duration.mean\""),
-        startsWith("\"mytimer.duration.sum\""),
-        startsWith("\"mytimer.duration.stddev\""),
-        startsWith("\"mytimer.duration.median\""),
-        startsWith("\"mytimer.duration.p75\""),
-        startsWith("\"mytimer.duration.p95\""),
-        startsWith("\"mytimer.duration.p99\""),
-        startsWith("\"mytimer.duration.p999\""),
-        startsWith("\"mytimer.rate.m1\""),
-        startsWith("\"mytimer.rate.m5\""),
-        startsWith("\"mytimer.rate.m15\""),
-        startsWith("\"mytimer.rate.mean\"")
-    ));
-  }
-
-  @Test(timeout = 1000)
-  public void testPrependGroupName() throws Exception {
-    innerSetUp(true, null, false, false);
-
-    // Counter
-    TaggedMetricName taggedMetricName = new TaggedMetricName("group", "mycounter",
-        "tag1", "value1", "tag2", "value2");
-    Counter counter = metricsRegistry.newCounter(taggedMetricName);
-    counter.inc();
-    counter.inc();
-
-    AtomicLong clock = new AtomicLong(System.currentTimeMillis());
-    long timeBin = (clock.get() / 60000 * 60);
-    // Wavefront Histo
-    WavefrontHistogram wavefrontHistogram = WavefrontHistogram.get(metricsRegistry, new TaggedMetricName(
-        "group3", "myhisto", "tag1", "value1", "tag2", "value2"), clock::get);
-    for (int i = 0; i < 101; i++) {
-      wavefrontHistogram.update(i);
-    }
-
-    // Exploded Histo
-    Histogram histogram = metricsRegistry.newHistogram(new MetricName("group2", "", "myhisto"), false);
-    histogram.update(1);
-    histogram.update(10);
-
-    // Advance the clock by 1 min ...
-    clock.addAndGet(60000L + 1);
-
-    wavefrontYammerMetricsReporter.run();
-    assertThat(
-        receiveFromSocket(12, fromMetrics),
-        containsInAnyOrder(
-            equalTo("\"group.mycounter\" 2.0 1485224035 tag1=\"value1\" tag2=\"value2\""),
-            equalTo("\"group2.myhisto.count\" 2.0 1485224035"),
-            equalTo("\"group2.myhisto.min\" 1.0 1485224035"),
-            equalTo("\"group2.myhisto.max\" 10.0 1485224035"),
-            equalTo("\"group2.myhisto.mean\" 5.5 1485224035"),
-            equalTo("\"group2.myhisto.sum\" 11.0 1485224035"),
-            startsWith("\"group2.myhisto.stddev\""),
-            equalTo("\"group2.myhisto.median\" 5.5 1485224035"),
-            equalTo("\"group2.myhisto.p75\" 10.0 1485224035"),
-            equalTo("\"group2.myhisto.p95\" 10.0 1485224035"),
-            equalTo("\"group2.myhisto.p99\" 10.0 1485224035"),
-            equalTo("\"group2.myhisto.p999\" 10.0 1485224035")));
-
-    assertThat(
-        receiveFromSocket(1, fromHistograms),
-        contains(equalTo("!M " + timeBin + " #1 0.0 #1 1.0 #1 2.0 #1 3.0 #1 4.0 #1 5.0 #1 6.0 #1 7.0 #1 8.0 #1 9.0 #1 10.0 #1 11.0 #1 12.0 #1 13.0 #1 14.0 #1 15.0 #1 16.0 #1 17.0 #1 18.0 #1 19.0 #1 20.0 #1 21.0 #1 22.0 #1 23.0 #1 24.0 #1 25.0 #1 26.0 #1 27.0 #1 28.0 #1 29.0 #1 30.0 #1 31.0 #1 32.0 #1 33.0 #1 34.0 #1 35.0 #1 36.0 #1 37.0 #1 38.0 #1 39.0 #1 40.0 #1 41.0 #1 42.0 #1 43.0 #1 44.0 #1 45.0 #1 46.0 #1 47.0 #1 48.0 #1 49.0 #1 50.0 #1 51.0 #1 52.0 #1 53.0 #1 54.0 #1 55.0 #1 56.0 #1 57.0 #1 58.0 #1 59.0 #1 60.0 #1 61.0 #1 62.0 #1 63.0 #1 64.0 #1 65.0 #1 66.0 #1 67.0 #1 68.0 #1 69.0 #1 70.0 #1 71.0 #1 72.0 #1 73.0 #1 74.0 #1 75.0 #1 76.0 #1 77.0 #1 78.0 #1 79.0 #1 80.0 #1 81.0 #1 82.0 #1 83.0 #1 84.0 #1 85.0 #1 86.0 #1 87.0 #1 88.0 #1 89.0 #1 90.0 #1 91.0 #1 92.0 #1 93.0 #1 94.0 #1 95.0 #1 96.0 #1 97.0 #1 98.0 #1 99.0 #1 100.0 \"group3.myhisto\" tag1=\"value1\" tag2=\"value2\"")));
-  }
-
-}